diff --git a/.agents/plugins/marketplace.json b/.agents/plugins/marketplace.json index b4db210..2674931 100644 --- a/.agents/plugins/marketplace.json +++ b/.agents/plugins/marketplace.json @@ -6,7 +6,7 @@ "plugins": [ { "name": "slow-powers", - "version": "0.5.4", + "version": "0.6.0", "source": { "source": "url", "url": "./" diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 921b164..ed85ba9 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -9,7 +9,7 @@ { "name": "slow-powers", "description": "Agent skills to upgrade plan mode and debugging, enforce best practices, and enhance native features, not replace them.", - "version": "0.5.4", + "version": "0.6.0", "source": "./", "author": { "name": "Max Haarhaus", diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index c5cafba..3c9db70 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "slow-powers", "description": "Agent skills to upgrade plan mode and debugging, enforce best practices, and enhance native features, not replace them.", - "version": "0.5.4", + "version": "0.6.0", "author": { "name": "Max Haarhaus", "email": "samiamorwas@gmail.com" diff --git a/.clinerules/memory-bank.md b/.clinerules/memory-bank.md new file mode 100644 index 0000000..40732bd --- /dev/null +++ b/.clinerules/memory-bank.md @@ -0,0 +1,60 @@ +# Cline's Memory Bank + +I am Cline, an expert software engineer with a unique characteristic: my memory resets completely between sessions. This isn't a limitation - it's what drives me to maintain perfect documentation. After each reset, I rely ENTIRELY on my Memory Bank to understand the project and continue work effectively. I MUST read ALL memory bank files at the start of EVERY task - this is not optional. + +## Memory Bank Structure + +The Memory Bank consists of core files and optional context files, all in Markdown format. Files build upon each other in a clear hierarchy: + +### Core Files (Required) + +1. `projectbrief.md` - Foundation document that shapes all other files + - Created at project start if it doesn't exist + - Defines core requirements and goals + - Source of truth for project scope +2. `productContext.md` - Why this project exists + - Problems it solves + - How it should work + - User experience goals +3. `activeContext.md` - Current work focus + - Recent changes + - Next steps + - Active decisions and considerations + - Important patterns and preferences + - Learnings and project insights +4. `systemPatterns.md` - System architecture + - Key technical decisions + - Design patterns in use + - Component relationships + - Critical implementation paths +5. `techContext.md` - Technologies used + - Development setup + - Technical constraints + - Dependencies + - Tool usage patterns +6. `progress.md` - What works + - What's left to build + - Current status + - Known issues + - Evolution of project decisions + +### Additional Context + +Create additional files/folders within memory-bank/ when they help organize: + +- Complex feature documentation +- Integration specifications +- API documentation +- Testing strategies +- Deployment procedures + +## Documentation Updates + +Memory Bank updates occur when: + +1. Discovering new project patterns +2. After implementing significant changes +3. When user requests with **update memory bank** (MUST review ALL files) +4. When context needs clarification + +REMEMBER: After every memory reset, I begin completely fresh. The Memory Bank is my only link to previous work. It must be maintained with precision and clarity, as my effectiveness depends entirely on its accuracy. diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index f852d6b..a28f293 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "slow-powers", - "version": "0.5.4", + "version": "0.6.0", "description": "Agent skills to upgrade plan mode and debugging, enforce best practices, and enhance native features, not replace them.", "author": { "name": "Max Haarhaus", diff --git a/.github/badges/skills-ref.json b/.github/badges/skills-ref.json new file mode 100644 index 0000000..b96186f --- /dev/null +++ b/.github/badges/skills-ref.json @@ -0,0 +1,6 @@ +{ + "schemaVersion": 1, + "label": "skills-ref", + "message": "8/8 valid", + "color": "brightgreen" +} diff --git a/.github/workflows/skills-ref.yml b/.github/workflows/skills-ref.yml new file mode 100644 index 0000000..a053e4b --- /dev/null +++ b/.github/workflows/skills-ref.yml @@ -0,0 +1,47 @@ +name: skills-ref badge + +on: + push: + branches: [dev] + paths-ignore: + - ".github/badges/skills-ref.json" + schedule: + - cron: "0 6 * * *" + workflow_dispatch: + +permissions: + contents: write + +jobs: + badge: + name: Validate skills and update badge + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install skills-ref (tracks agentskills@main) + run: | + python -m pip install --upgrade pip + pip install "git+https://github.com/agentskills/agentskills@main#subdirectory=skills-ref" + skills-ref --version + + - uses: oven-sh/setup-bun@v2 + + - name: Validate skills and write badge JSON + run: bun run badge:skills-ref + + - name: Commit badge if changed + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add .github/badges/skills-ref.json + if git diff --staged --quiet; then + echo "Badge unchanged; nothing to commit." + exit 0 + fi + git commit -m "chore(badge): update skills-ref validation badge [skip ci]" + git push \ No newline at end of file diff --git a/.gitignore b/.gitignore index 59eba62..ce255d9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ # Ignore local Claude Code state, but track the shared, committed settings. .claude/* !.claude/settings.json +# Cline plugin install artifacts (e.g. from `cline plugin install ./ --cwd .`). +.cline/plugins/ .DS_Store node_modules/ .eval-magic diff --git a/AGENTS.md b/AGENTS.md index 75c36be..1c83bce 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,12 +4,20 @@ Slow-powers is a set of software development methodology skills and meta skill-w ## What lives here -This repo ships Slow-powers across three harnesses: +This repo ships Slow-powers across four harnesses: - `skills/` — Skills, assets, and cross-cutting tests - `.claude-plugin/` — Claude Code plugin - `.codex-plugin/` — OpenAI Codex plugin - `opencode/` — OpenCode plugin (`@slowdini/slow-powers-opencode`) +- `cline/` — Cline plugin (CLI/SDK/Kanban only; declared via the `cline` + field in `package.json`, with `skills/` auto-discovered from the package + root) + +Cline-specific setup for working on this repo also lives at root: + +- `.clinerules/` — Cline rules (the Memory Bank custom instructions) +- `memory-bank/` — Cline Memory Bank files recording ongoing work ## Editing the right files @@ -69,3 +77,7 @@ bun run check (pre-commit runs typecheck + lint-staged; pre-push runs the test suite). `bun scripts/bump-version.ts ` updates every manifest in lockstep. + +To test the Cline integration live: `cline plugin install ./ --cwd `, +run a Cline session in the scratch dir, then +`cline plugin uninstall slow-powers --cwd ` when done. diff --git a/README.md b/README.md index ab45241..763bc39 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,9 @@ evals + skills-ref validation

# Slow-powers @@ -40,7 +43,7 @@ Slow-powers is a fork of [obra/superpowers](https://github.com/obra/superpowers) ## Quickstart -[Claude Code](#claude-code) · [Codex CLI](#codex-cli) · [OpenCode](#opencode) +[Claude Code](#claude-code) · [Codex CLI](#codex-cli) · [OpenCode](#opencode) · [Cline](#cline) ## How it works @@ -96,6 +99,17 @@ You can also browse and install it interactively: run `codex`, open opencode plugin @slowdini/slow-powers-opencode -g ``` +### Cline + +```bash +cline plugin install https://github.com/slowdini/slow-powers.git +``` + +Cline plugins load in the Cline CLI, SDK, and Kanban — **not** the VSCode or +JetBrains extensions. On those, you can still use the skills by copying or +symlinking `skills/` into `.cline/skills/` (project) or `~/.cline/skills/` +(global); the bootstrap injection and plan gate are CLI/SDK/Kanban-only. + ## The skills Slow-powers provides a set of highly focused skills that ensure your agent operates with maximum discipline: @@ -148,8 +162,9 @@ Flat layout — skills and assets live at root, harness-specific integration liv - `.claude-plugin/` — Claude Code plugin manifest and hooks - `.codex-plugin/` — OpenAI Codex plugin manifest - `opencode/` — OpenCode plugin +- `cline/` — Cline plugin entry point - `.claude-plugin/marketplace.json` — Claude Code marketplace registry -- `package.json` — OpenCode plugin manifest + dev tooling +- `package.json` — OpenCode + Cline plugin manifests + dev tooling ## Releasing diff --git a/cline/plugins/slow-powers.js b/cline/plugins/slow-powers.js new file mode 100644 index 0000000..196751c --- /dev/null +++ b/cline/plugins/slow-powers.js @@ -0,0 +1,127 @@ +/** + * Slow-powers plugin for Cline (CLI / SDK / Kanban). + * + * Two jobs, mirroring what the bash hooks do on Claude Code and Codex: + * + * 1. BOOTSTRAP INJECTION — registers the contents of bootstrap.md as a session + * rule, so the skill-enforcement block is part of every session's system + * prompt. This replaces the SessionStart-hook injection used on Claude/Codex + * and the system-prompt transform used on OpenCode. + * + * 2. PLAN GATE — the FIRST switch_to_act_mode call of a conversation is skipped + * with an instruction to run the hardening-plans skill on the plan first. + * switch_to_act_mode is how Cline presents a finished plan and leaves plan + * mode; skipping it keeps the session in plan mode, so the agent can load + * the skill, fix findings inline, and re-submit a hardened plan. The + * re-submission finds the per-conversation marker and is allowed through. + * + * WHY DENY-ONCE (and not deny-until-proven-hardened): keying the marker per + * conversation and allowing the second attempt guarantees we can never + * hard-lock a user inside plan mode. Worst case (agent re-submits without + * hardening) degrades to no-gate behavior — never worse. Same argument as + * hooks/exit-plan-mode. + * + * Skills need no wiring here: when this package is installed as a Cline + * plugin, the top-level skills/ directory is discovered automatically. + * + * Single-file plugin constraint: only Node builtins may be imported at + * runtime; @cline/* packages are host-provided and referenced in JSDoc only. + */ + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const PLUGIN_DIR = path.dirname(fileURLToPath(import.meta.url)); +const BOOTSTRAP_PATH = path.resolve(PLUGIN_DIR, "../../bootstrap.md"); + +// Module-level cache: bootstrap.md does not change during a session, so read +// it once. undefined = not yet loaded, null = missing/unreadable. +let _bootstrapCache; + +function getBootstrapContent() { + if (_bootstrapCache !== undefined) return _bootstrapCache; + try { + _bootstrapCache = fs.readFileSync(BOOTSTRAP_PATH, "utf8"); + } catch { + _bootstrapCache = null; + } + return _bootstrapCache; +} + +// Pick a key that is stable across the skip and the re-submit. conversationId +// is the natural choice; fall back to agentId, then a fixed key. Any +// consistent key preserves deny-once safety. +function conversationKey(context) { + const snapshot = context?.snapshot; + const raw = snapshot?.conversationId ?? snapshot?.agentId ?? "fallback"; + // Sanitize to a safe, bounded filename component (mirrors the bash hooks). + return String(raw) + .replace(/[^A-Za-z0-9._-]/g, "_") + .slice(0, 128); +} + +function markerPath(context) { + return path.join( + process.env.SLOW_POWERS_PLAN_GATE_DIR ?? os.tmpdir(), + `slow-powers-plan-gate-${conversationKey(context)}`, + ); +} + +const SKIP_REASON = + "A plan is about to be presented. Before it leaves your hands, use the " + + "hardening-plans skill to review the plan file as a skeptical executor, " + + "then call switch_to_act_mode again to present the hardened plan."; + +/** @type {import("@cline/sdk").AgentPlugin} */ +const SlowPowersPlugin = { + name: "slow-powers", + manifest: { + capabilities: ["hooks", "rules"], + }, + + setup(api) { + const bootstrap = getBootstrapContent(); + if (!bootstrap) return; + api.registerRule({ + id: "slow-powers/bootstrap", + source: "slow-powers", + content: bootstrap, + }); + }, + + hooks: { + beforeTool(context) { + try { + // The runtime passes both `tool` (the AgentTool definition) and + // `toolCall` (the pending call). First-party guards read `tool.name`; + // the plugin docs read `toolCall.name`. Accept either shape. + const toolName = context?.tool?.name ?? context?.toolCall?.name; + if (toolName !== "switch_to_act_mode") return undefined; + + const marker = markerPath(context); + if (fs.existsSync(marker)) { + // Re-submission after hardening — let the plan be presented. + return undefined; + } + + // First switch_to_act_mode this conversation: record it, then skip + // once to insert the hardening-plans beat. Best-effort marker write — + // a failed write must not break the gate (fail-open below covers it). + try { + fs.writeFileSync(marker, ""); + } catch { + // Ignore: worst case the gate fires again on the next attempt. + } + + return { skip: true, reason: SKIP_REASON }; + } catch { + // Fail open: a plugin error must never block the user's workflow. + return undefined; + } + }, + }, +}; + +export default SlowPowersPlugin; diff --git a/memory-bank/activeContext.md b/memory-bank/activeContext.md new file mode 100644 index 0000000..b3c2086 --- /dev/null +++ b/memory-bank/activeContext.md @@ -0,0 +1,77 @@ +# Active Context + +## Current focus + +Cline support was just added (August 2026). Two halves: + +1. **Cline plugin** (`cline/plugins/slow-powers.js`, declared via the `cline` + field in `package.json`): registers `bootstrap.md` as a session rule and + gates the first `switch_to_act_mode` of each conversation on + hardening-plans (skip-once + tmp marker, mirroring `hooks/exit-plan-mode`). + Skills are auto-discovered from the package root — no wiring needed. +2. **Repo-local Cline setup**: `.clinerules/memory-bank.md` (canonical Memory + Bank instructions) and this `memory-bank/` directory, both committed. + +## Recent changes + +- `cline/plugins/slow-powers.js` (new), `package.json` `cline` field + `files` +- `tests/harness/spec.ts` Cline entry; Cline assertions in `manifests.test.ts`; + new `tests/harness/cline-plugin.test.ts` +- README Cline install section; AGENTS.md four-harness update; + `.gitignore` covers `.cline/plugins/` install artifacts + +## Verification results (Cline CLI 3.0.51, headless) + +- `cline plugin install --cwd ` works; installer copies the + repo and registers the plugin entry. +- Live session: all 8 skills discovered; `` bootstrap + block present in instructions; bootstrap behavior observed (agent invoked a + skill on a ~1% match, per the bootstrap rule). +- Plan gate: unit-tested against the documented `AgentBeforeToolResult` + contract. The runtime's `skip` handling (tool doesn't run, `reason` goes to + the model) and the hook context shape were confirmed in the shipped CLI + source — the first-party `core.plan-mode-command-guard` extension uses the + same pattern. `switch_to_act_mode` is NOT exposed in headless one-shot + sessions, so an interactive (TUI) confirmation of the gate firing is the one + remaining manual check. + +## Next steps + +- PR opened: https://github.com/slowdini/slow-powers/pull/266 (base `dev`). +- Manually confirm the plan gate in an interactive `cline -i` plan-mode + session (present plan → approve → first `switch_to_act_mode` gets skipped + with the hardening instruction) — easiest via a test release, per the + maintainer. +- After merge to `dev`: trigger the Release PR workflow with the next version + to ship the Cline plugin (that release doubles as the test release). + +## Active decisions + +- Distribution reuses the root `package.json` (git install); no separate npm + package or release-workflow change. +- The Cline gate is skip-once only. The already-hardened short-circuit + (upstream #153 refinement) is deferred — it needs reliable detection that + hardening-plans already ran (the skill-invocation tool is `skills` in the + Cline runtime). +- No `.cline/skills/` dogfooding symlinks: Cline's skill registry is + last-wins with plugin dirs scanned *after* workspace dirs, so an installed + slow-powers plugin would silently shadow the repo's skills. The + installed-vs-repo precedence question is deferred to a separate + cross-harness exploration (it affects all harnesses). + +## Learnings + +- Cline plugins load only in CLI/SDK/Kanban — not VSCode/JetBrains. IDE users + get skills via manual copy into `.cline/skills/` or `~/.cline/skills/`. +- Cline reads `AGENTS.md` natively; no memory-file symlink needed for it. +- Cline's plan-exit tool is `switch_to_act_mode`; `AgentBeforeToolResult.skip` + + `reason` is the deny mechanism; `registerRule` puts content in the system + prompt every session. +- Hook contexts pass the tool name on BOTH `tool.name` (first-party shape) and + `toolCall.name` (docs shape) — read `tool?.name ?? toolCall?.name`. +- Headless one-shot sessions (`cline -p "..."`) don't expose + `switch_to_act_mode` and can't drive TTY-only commands (`cline config`); use + interactive sessions for plan-gate verification. +- Cline's local plugin install copies dotfile-free repo content — everything + the plugin needs (`cline/`, `skills/`, `bootstrap.md`) is a normal path, so + this is fine. diff --git a/memory-bank/productContext.md b/memory-bank/productContext.md new file mode 100644 index 0000000..83bfa2e --- /dev/null +++ b/memory-bank/productContext.md @@ -0,0 +1,26 @@ +# Product Context + +## Why this exists + +Coding agents under pressure skip discipline: they present unreviewed plans, +claim success without running tests, thrash on bugs with guess-and-check, and +let new work collide with in-progress branches. Slow-powers exists to put that +discipline back — not by replacing harness features, but by hardening them +(plan-mode gates, skill-enforcement bootstrap, verification loops). + +## How it should work + +- A bootstrap block (`bootstrap.md`) is injected into every session, making + skill use non-negotiable when a skill applies. +- Skills declare prerequisite / next-step gates so the agent follows an + intended sequence (plan → harden → isolate → TDD → verify). +- Harness hooks/plugins supply the deterministic beats a skill can't enforce + on its own (e.g. gating plan presentation on hardening-plans). + +## User experience goals + +- Install once per harness, then forget it — the value shows up as plans that + don't hallucinate files, tests that exist before code, and success claims + backed by command output. +- "The plugin for people who don't install plugins": minimal surface, no + config, no lock-in; users can extend with their own evaluated skills. diff --git a/memory-bank/progress.md b/memory-bank/progress.md new file mode 100644 index 0000000..0b10832 --- /dev/null +++ b/memory-bank/progress.md @@ -0,0 +1,37 @@ +# Progress + +## What works + +- Eight skills with eval coverage; bootstrap injection and plan gates on + Claude Code, Codex CLI, and OpenCode. +- Full test suite green (`bun test`), typecheck and biome clean. +- **Cline support (new)**: plugin entry, manifest field, unit + manifest + tests, README/AGENTS.md docs, memory bank initialized. Verified live on + Cline CLI 3.0.51: install, skills discovery, and bootstrap rule injection + all confirmed in headless sessions. + +## What's left + +- Manual interactive check of the Cline plan gate (`switch_to_act_mode` is + only exposed in interactive sessions), then PR. +- Release: next version bump will carry the Cline plugin via the normal flow. + +## Known issues / deferred + +- Cline plugins don't load on VSCode/JetBrains extensions — IDE users get a + documented skills-only manual install (no bootstrap, no plan gate). +- Cline skill collisions are last-wins with plugin directories scanned after + workspace ones, so an installed slow-powers plugin shadows same-named + workspace skills (the reverse of what this repo wants for development). + Deferred: cross-harness installed-vs-repo precedence exploration. +- Cline plan gate has no already-hardened short-circuit yet (deferred; needs + skill-invocation detection). + +## Decision log + +- 2026-08: Cline distribution via root `package.json` + git install (no new + npm package). +- 2026-08: Memory bank committed to git (`.clinerules/memory-bank.md` + + `memory-bank/`). +- 2026-08: No `.cline/skills/` symlinks (option (c)) pending the precedence + exploration. diff --git a/memory-bank/projectbrief.md b/memory-bank/projectbrief.md new file mode 100644 index 0000000..a84d3e8 --- /dev/null +++ b/memory-bank/projectbrief.md @@ -0,0 +1,25 @@ +# Project Brief + +Slow-powers is an agent skill set for professional software development. It +enhances plan mode and debugging work, enforces best practices (TDD, +verification, workspace isolation), and works *with* the features of modern +agent harnesses instead of replacing them. It is a fork of +[obra/superpowers](https://github.com/obra/superpowers), with rewrites focused +on clarity, token efficiency, and a lighter touch. + +## Core goals + +- Ship discipline-enforcing skills (plan hardening, TDD, scientific debugging, + verification, isolated workspaces) that measurably improve agent behavior — + every skill ships with a documented eval or it doesn't ship. +- Support multiple agent harnesses from one repo: Claude Code, OpenAI Codex, + OpenCode, and Cline. +- Keep skill content cross-harness compatible (no harness-specific vocabulary + in skill prose). + +## Scope + +- `skills/` holds the shared skills and their evals. +- Harness-specific integration (manifests, hooks, runtime plugins) lives in + top-level directories; skill content itself stays harness-agnostic. +- This repo is the source of truth; installed plugins are downstream copies. diff --git a/memory-bank/systemPatterns.md b/memory-bank/systemPatterns.md new file mode 100644 index 0000000..b788965 --- /dev/null +++ b/memory-bank/systemPatterns.md @@ -0,0 +1,48 @@ +# System Patterns + +## Flat layout, one source of truth + +Skills and shared assets live at the repo root; each harness's integration is +a thin top-level layer that points back at them. Nothing is duplicated per +harness. + +## Per-harness delivery of the same two behaviors + +Every harness delivers (1) the `bootstrap.md` skill-enforcement block and +(2) a deterministic plan-presentation gate, using that harness's native +mechanism: + +| Harness | Bootstrap delivery | Plan gate | +| -------- | -------------------------------------- | ------------------------------------------------ | +| Claude | `hooks/session-start` (SessionStart) | `hooks/exit-plan-mode` (PreToolUse, deny-once) | +| Codex | shared `hooks/hooks.json` SessionStart | `hooks/codex-stop-plan-mode` (Stop hook) | +| OpenCode | `opencode/plugins/slow-powers.js` system-prompt transform | same plugin, `file.edited` event on plan files | +| Cline | `cline/plugins/slow-powers.js` `registerRule` | same plugin, `beforeTool` skip-once on `switch_to_act_mode` | + +Claude/Codex hooks are extensionless bash scripts dispatched by the +`hooks/run-hook.cmd` polyglot (Windows-safe). OpenCode/Cline integrations are +dependency-free JS runtime plugins. + +## Manifest and version lockstep + +`scripts/manifest-files.ts` lists every versioned manifest; +`scripts/bump-version.ts` rewrites them in lockstep (then biome-formats); +`tests/harness/manifests.test.ts` asserts parity. The Cline and OpenCode +integrations both declare themselves inside the root `package.json`, which is +already locked. + +## Parameterized parity tests + +`tests/harness/spec.ts` holds one `HarnessSpec` per harness; the suite in +`manifests.test.ts` applies the same contract to all of them. Adding a +harness = adding a spec entry (+ custom assertions when the manifest shape +doesn't fit the dotted-string `pathFields` machinery, as with Cline's nested +`cline.plugins[].paths[]`). + +## Skill integrity tests + +The shared-assets block in `manifests.test.ts` pins: SKILL.md frontmatter +(name + description), top-level-only skill folders, documented peer +directories (`assets`/`evals`/`references`/`scripts`), resolvable markdown +links, reachable reference files, mermaid-not-graphviz, and the bootstrap +marker. diff --git a/memory-bank/techContext.md b/memory-bank/techContext.md new file mode 100644 index 0000000..ad3b0c4 --- /dev/null +++ b/memory-bank/techContext.md @@ -0,0 +1,30 @@ +# Tech Context + +## Stack + +- **bun** — test runner and script runtime (`bun test`, `bun scripts/*.ts`) +- **biome** — lint + format (`bun run check`, `check:ci`); JSON included +- **typescript** — `tsc --noEmit` over `scripts/**/*.ts` and `tests/**/*.ts` + only (runtime plugins and hooks are plain JS/bash, deliberately) +- **husky + lint-staged** — pre-commit typecheck/lint, pre-push test suite + (installed by `bun install` via the `prepare` script) +- **eval-magic** — skill evaluation harness (`bun run evals*` scripts); + eval fixtures live under `skills//evals/` + +## Release flow + +Releases cut from `dev`, tagged from `main`. The Release PR workflow bumps +every manifest via `scripts/bump-version.ts`; merging to `main` tags, creates +the GitHub release, and publishes `@slowdini/slow-powers-opencode` to npm. + +## Constraints + +- Hook scripts: pure bash, no jq/python/bun at hook time; printf-based JSON + (heredocs hang on bash 5.3+); extensionless filenames (Windows). +- Cline single-file plugins may import only Node builtins; `@cline/*` + packages are host-provided. +- Skill prose must use cross-harness vocabulary (see `writing-skills`). + +## Local environment + +- Cline CLI 3.0.51 (homebrew) used for live verification of the Cline plugin. diff --git a/package.json b/package.json index e7fe375..0a37fce 100644 --- a/package.json +++ b/package.json @@ -1,10 +1,24 @@ { "name": "@slowdini/slow-powers-opencode", - "version": "0.5.4", + "version": "0.6.0", "description": "Slow-powers — structured development workflows for coding agents (TDD, debugging, verification, git hygiene)", "type": "module", "main": "./opencode/plugins/slow-powers.js", + "cline": { + "plugins": [ + { + "paths": [ + "./cline/plugins/slow-powers.js" + ], + "capabilities": [ + "hooks", + "rules" + ] + } + ] + }, "files": [ + "cline/", "opencode/", "skills/", "bootstrap.md", @@ -49,6 +63,7 @@ "evals:grade": "eval-magic grade --skill-dir ./skills", "evals:aggregate": "eval-magic aggregate --skill-dir ./skills", "evals:promote-baseline": "eval-magic promote-baseline --skill-dir ./skills", + "badge:skills-ref": "bun run scripts/skills-ref-badge.ts", "demo": "vhs assets/demo/tdd.tape", "check": "biome check --write .", "check:ci": "biome check --error-on-warnings .", diff --git a/scripts/skills-ref-badge.test.ts b/scripts/skills-ref-badge.test.ts new file mode 100644 index 0000000..3f7ae34 --- /dev/null +++ b/scripts/skills-ref-badge.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, test } from "bun:test"; +import { buildBadgeJson } from "./skills-ref-badge"; + +describe("skills-ref-badge buildBadgeJson", () => { + test("all skills valid -> green score badge", () => { + expect(buildBadgeJson(8, 8)).toEqual({ + schemaVersion: 1, + label: "skills-ref", + message: "8/8 valid", + color: "brightgreen", + }); + }); + + test("one skill invalid -> red partial-score badge", () => { + expect(buildBadgeJson(7, 8)).toEqual({ + schemaVersion: 1, + label: "skills-ref", + message: "7/8 valid", + color: "red", + }); + }); + + test("skills-ref missing or errored -> grey unavailable badge", () => { + //total=0 signals the validator itself did not produce results + expect(buildBadgeJson(0, 0)).toEqual({ + schemaVersion: 1, + label: "skills-ref", + message: "unavailable", + color: "lightgrey", + }); + }); +}); diff --git a/scripts/skills-ref-badge.ts b/scripts/skills-ref-badge.ts new file mode 100644 index 0000000..95e8d42 --- /dev/null +++ b/scripts/skills-ref-badge.ts @@ -0,0 +1,119 @@ +#!/usr/bin/env bun +import { readdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +const SKILLS_DIR = "skills"; +const BADGE_FILE = ".github/badges/skills-ref.json"; +const SKILLS_REF_BIN = "skills-ref"; + +/** + * Pure helper: build the shields.io endpoint JSON for the skills-ref badge. + * `total === 0` signals the validator itself did not produce results + * (binary missing / errored), rendered as a neutral "unavailable" badge. + */ +export function buildBadgeJson( + valid: number, + total: number, +): { schemaVersion: number; label: string; message: string; color: string } { + if (total === 0) { + return { + schemaVersion: 1, + label: "skills-ref", + message: "unavailable", + color: "lightgrey", + }; + } + const allValid = valid === total; + return { + schemaVersion: 1, + label: "skills-ref", + message: `${valid}/${total} valid`, + color: allValid ? "brightgreen" : "red", + }; +} + +function listSkillDirs(): string[] { + return readdirSync(SKILLS_DIR) + .filter((entry) => statSync(join(SKILLS_DIR, entry)).isDirectory()) + .filter((entry) => { + try { + statSync(join(SKILLS_DIR, entry, "SKILL.md")); + return true; + } catch { + return false; + } + }) + .map((entry) => join(SKILLS_DIR, entry)) + .sort(); +} + +function validateSkill(skillDir: string): { ok: boolean; err?: string } { + const result = Bun.spawnSync([SKILLS_REF_BIN, "validate", skillDir], { + stdout: "pipe", + stderr: "pipe", + }); + if (result.exitCode !== 0) { + return { + ok: false, + err: `${result.stderr.toString().trim() || result.stdout.toString().trim()}`, + }; + } + return { ok: true }; +} + +function writeBadgeIfChanged(badge: { + schemaVersion: number; + label: string; + message: string; + color: string; +}): boolean { + const serialized = `${JSON.stringify(badge, null, 2)}\n`; + let previous = ""; + try { + previous = readFileSync(BADGE_FILE, "utf8"); + } catch { + // file does not exist yet + } + if (previous === serialized) { + return false; + } + writeFileSync(BADGE_FILE, serialized); + return true; +} + +if (import.meta.main) { + // If skills-ref is not installed, emit the neutral badge and exit 0 so the + // workflow never fails the job on a missing binary; the badge is the signal. + const binCheck = Bun.spawnSync([SKILLS_REF_BIN, "--version"], { + stdout: "pipe", + stderr: "pipe", + }); + if (binCheck.exitCode !== 0) { + console.error( + `warning: '${SKILLS_REF_BIN}' not found on PATH; writing unavailable badge`, + ); + writeBadgeIfChanged(buildBadgeJson(0, 0)); + process.exit(0); + } + + const skillDirs = listSkillDirs(); + let valid = 0; + for (const dir of skillDirs) { + const res = validateSkill(dir); + if (res.ok) { + valid += 1; + } else { + console.error(`✗ ${dir}\n${res.err}`); + } + } + + // Exit 0 even on validation failures: a red badge is the intended signal, + // not a workflow failure. Per-skill errors are logged above for the job log. + const changed = writeBadgeIfChanged(buildBadgeJson(valid, skillDirs.length)); + console.log( + `skills-ref: ${valid}/${skillDirs.length} valid — badge ${ + changed ? "updated" : "unchanged" + }`, + ); + process.exit(0); +} diff --git a/skills/evaluating-skills/SKILL.md b/skills/evaluating-skills/SKILL.md index 72e8e05..c658b64 100644 --- a/skills/evaluating-skills/SKILL.md +++ b/skills/evaluating-skills/SKILL.md @@ -15,7 +15,7 @@ Evals are harness-agnostic: run records use a portable JSON schema so an eval au ### Two comparison modes -- **Mode A — new skill.** Compares `with_skill` vs `without_skill`. Use when validating that a brand-new skill beats baseline behavior with no skill loaded. +- **Mode A — new skill.** Compares `with_skill` vs `without_skill`. Use when validating that a brand-new skill beats baseline behavior with no skill loaded. Best understood as a bundle of removal tests — one case per behavior the skill teaches (see *Decompose the skill into behaviors first*). - **Mode B — revision (the common case).** Compares `old_skill` vs `new_skill`. Use when testing a language change to an existing skill — snapshot the old `SKILL.md`, then run both variants against the same prompts. A negative or zero delta is a signal to revert: the new language did not improve behavior. The runner implements both; pick the mode that matches the change you're measuring. @@ -88,6 +88,18 @@ All of these mean: STOP. Present the pre-flight summary and wait for confirmatio ## Designing test cases +### Decompose the skill into behaviors first + +A skill worth a with-vs-without eval usually teaches several distinct behaviors, and a suite authored "for the skill" measures whichever one it happens to trip over. The type case is in this repo: an `investigating-bugs` case written to test investigation efficiency turned out to measure something else — whether the agent leaves a regression test behind — and that accident became the suite's only confirmed effect. This is also why revision (Mode B) evals are easier to author: a revision arrives pre-scoped, the diff names what to measure. A new-skill suite must do that scoping itself, and decomposition is how. Before authoring any case, inventory the skill as discrete, testable behaviors: for each, finish the sentence *"without this, the agent does Y instead of Z."* + +- **Units are functional, not lexical.** A behavior rarely lives in one place — a phase step, a rationalization-table row, and a red-flag bullet often restate the same rule. The unit is the behavior; its inventory entry lists every lexical expression. Redundancy inside a skill is expected and deliberate (discipline skills repeat rules on purpose); whether one particular restatement earns its keep is a Mode B revision question, never part of the decomposition. +- **Classify each behavior** with the test from *Choosing to test with evals*: contingent (needs a case), deterministic (compliance isn't in doubt — no case), or structural (prerequisites, navigation, cross-links that support other behaviors). +- **Map both directions.** Every case names the single behavior whose removal should flip it; every contingent behavior names the case that would catch its removal. A contingent behavior with no case is a coverage gap. A behavior no case could ever detect is a trim candidate — tier-scoped, see *Pricing a behavior — ablation runs*. + +The map is the suite's coverage report: it shows the range of things the skill demonstrably buys, the way coverage shows which code the tests exercise. Keep it next to the suite (a `COVERAGE.md` beside `evals.json`); the worked example is `skills/investigating-bugs/evals/COVERAGE.md`. + +### Writing the cases + A test case has these parts: - **prompt**: a realistic user message — the kind a real user would actually type @@ -99,7 +111,7 @@ Cases live in `/evals/evals.json`. For the file shape, see the author-tem Tips for writing good prompts: -- **Start with 2–3 test cases.** Don't over-invest before iteration 1. +- **Start with 2–3 test cases** — the highest-value rows of the behavior map. Don't over-invest before iteration 1. - **Vary phrasing.** Mix casual ("hey can you check this") with precise ("Run `bun test`, quote the output"). - **Cover edge cases.** Include at least one boundary condition, malformed input, or ambiguous instruction. - **Use realistic context.** Real users reference file paths, function names, personal context. "Process this data" is too vague to test anything useful. @@ -116,6 +128,21 @@ What "stresses the skill" depends on what kind of skill it is. The four types fr - **Pattern skills** (flatten-with-flags, information-hiding). Test recognition: include prompts where the pattern applies and prompts where it doesn't. Success = the agent applies the pattern when warranted and refrains when it isn't. - **Reference skills** (API docs, syntax guides). Test retrieval: ask questions whose answers are in the reference, including a few that hit gaps you suspect. Success = the agent finds the right section and uses it correctly. +**Failure-prevention vs quality-gradient skills — a cut across those four types.** Some skills exist to prevent an obvious, catchable failure: `hardening-plans` stops a plan that cites a hallucinated file, and a single correctness check separates the skilled arm from the unskilled one. Others are *quality-gradient*: the agent already produces a working result, and the skill makes it smaller, faster, more targeted, or correct on the cases the developer didn't test — `investigating-bugs` is the type case (any competent model fixes the reported bug; the skill is about fixing it at the source, once, for everyone). Binary "did it work?" assertions **ceil** on quality-gradient skills, because the unskilled arm also produces something that works. To get a delta you need an engineered trap (a fix that's green on the happy path but wrong elsewhere) and/or a scope/quality metric (a held-out test across other inputs, a diff size) — not just a correctness check the unskilled arm passes too. + +### Engineering an attractive failure + +An eval only measures a skill when the **unskilled arm is genuinely tempted into the wrong path**. If the correct answer is also the obvious answer, both arms take it, the delta is zero, and you have measured nothing — however many runs you spend. A ceiling like that is a property of the *case*, not the skill: the trap was never baited. Engineer the trap against one behavior from your map — bait exactly the shortcut that behavior forbids — so the case works as a removal test rather than a general exam. + +The richest traps combine **hard reproduction** with a **locally-rewarded wrong fix**: + +- *Hard to reproduce* — the bug only manifests under conditions the developer's own machine doesn't have: a different timezone, locale, clock, data shape, or scale. A green run in the default environment (and a reporter's "works fine for me") is then not evidence the bug is gone, so the disciplined move — reproduce by varying the context — becomes the thing the skill has to supply. +- *Locally rewarded but wrong* — there is a fast fix that makes the reported symptom go away on the happy path yet breaks an un-reported one: a second consumer of the same value, a different user segment, a boundary the report didn't mention. The unskilled arm takes it and feels finished; only investigation surfaces the breakage. + +The worked example: a JavaScript date-only field that renders a day early for users west of UTC. A developer in UTC can't reproduce it; the obvious nudges (`+1 day`, "force local parse") each fix the reporter's case while corrupting other offsets or the save round-trip; only treating the value as a timezone-agnostic calendar date is correct everywhere. The anti-example is a trap that *doesn't* bait — a clamp that visibly charges $0, which any competent model rejects on sight. There is nothing to be tempted by, so there is nothing to measure. + +In a small fixture, full investigation is nearly free, so a lazy path is only tempting if you build the temptation in. That is the work: manufacture — with a hidden environment variable, or a second consumer of the same value — the cost a large real codebase would impose for free. + ### Seeding conversation context (and its ceiling) A cold prompt measures trigger-recognition *in isolation*. The harder, more realistic failure is trigger-recognition *under a competing attractor* — an agent already mid-session, committed to a skill-free approach, where loading the skill reads as redundant. Approximate that by **seeding**: embed prior `User:` / `Assistant:` turns directly in the `prompt` string (it is wrapped verbatim as the user request, so a multi-line transcript needs no schema change). Seed an `Assistant:` turn that has already produced work in a native, skill-free style, then a final `User:` turn carrying the real request. A seed can reproduce prior commitment / in-flight momentum, redundancy framing, sunk cost, and — usefully — a prior plan that *name-drops* a skill (e.g. a parenthetical "TDD — tests first") without actually following it, so you can test whether the agent makes the discipline load-bearing or treats the label as compliance. For a worked example, see the seeded cases in `hardening-plans/evals/`. @@ -174,16 +201,29 @@ Once a run is graded and aggregated, the headline is the **delta**: what the ski - **Tighten instructions when results are inconsistent.** High stddev = ambiguous instructions or model variability. - **Read time/token outliers.** If one run is 3× longer, read its transcript for the bottleneck. +**A ceiling has two causes — tell them apart.** Both arms passing can mean the case's behavior is base-model native at this tier, or the eval is too easy. Disambiguate two ways: check whether the unskilled arm *ever* takes the decoy across runs, and verify each "passing" fix against a held-out matrix (other timezones, other inputs) rather than the reported repro alone. If the unskilled arm reaches the full correct answer unprompted, the ceiling is real — a **null ablation**: the skill doesn't change *that behavior* at this tier. Record it in the coverage map and keep the case as a diagnostic (deleting a case because it stopped flattering the skill is how a suite gets tuned into agreement); a weaker tier or another harness may still need the behavior. If *no* behavior in the map survives, the skill has nothing measurable at this tier and the Iron Law says don't ship — a legitimate, documented finding, not a failure to measure. If neither arm was ever tempted, the trap wasn't attractive — redesign it (see *Engineering an attractive failure*) before concluding anything. Lowering the model tier sharpens the distinction. + **Human review** catches what assertions don't — outputs that are technically correct but miss the point. Keep per-eval reviewer notes; an empty note means the output looked fine. Focus the next iteration on the cases you had specific complaints about. **Guidance for revision:** - **Generalize from feedback.** The skill is used across many prompts, not just these cases. Fixes should address underlying issues broadly, not patch specific examples. -- **Keep the skill lean.** Fewer, better instructions outperform exhaustive rules. If pass rates plateau despite more rules, try removing instructions. +- **Keep the skill lean.** Fewer, better instructions outperform exhaustive rules. If pass rates plateau despite more rules, remove instructions — and let ablation evidence pick the cut, not taste (see *Pricing a behavior — ablation runs*). - **Explain the why.** Reasoning-based instructions ("Do X because Y") outperform rigid directives. Models follow instructions more reliably when they understand the purpose. **When to stop:** pass rates are satisfactory and reviewer feedback is consistently empty; iteration deltas have plateaued; or you've found a more fundamental issue (wrong scope, unrepresentative prompts). +## Pricing a behavior — ablation runs + +The coverage map is a set of claims: *case C flips when behavior B is removed*. An ablation run tests one claim directly. Mechanically it is a Mode B run — old = the full skill, new = the skill minus one behavior — with the expectation inverted: you are hoping the ablated arm gets **worse** on B's case. If it degrades, the mutant is killed — the attribution is confirmed and the behavior demonstrably earns its keep. If it holds, either the effect is diffuse (other parts of the skill carry it) or the behavior isn't doing anything — at this tier. + +- **Ablate the functional unit, entire.** Delete every expression of the behavior together — the step, whole table rows that restate it, its red-flag bullets. A restatement left behind compensates for the deleted step and under-credits the behavior. +- **One behavior at a time, always from the full skill.** Cumulative deletion confounds attribution across behaviors. +- **A null ablation is a per-tier verdict.** The tier you measured didn't need the behavior; a weaker model or another harness may. Trimming on one tier's null tunes the skill to that tier. +- **Runs are targeted, not routine.** A meaningful ablation needs the same run count as any other eval, so a per-behavior sweep is a campaign per behavior. The mandatory, free part is the design-time map. Pay for a run only when (a) a new case should prove it detects the behavior it claims to cover, or (b) a trimming decision needs evidence rather than taste. + +The whole-skill Mode A delta remains the headline number. Per-behavior deltas are sub-additive — deliberate redundancy means behaviors partially back each other up — so they do not sum to it. + ## Running the eval The mechanics of executing a run live in **[eval-magic](https://github.com/slowdini/eval-magic)** — the `eval-magic` binary. eval-magic's README is the complete operating guide, and every flag is documented in the tool's own help. diff --git a/skills/investigating-bugs/evals/COVERAGE.md b/skills/investigating-bugs/evals/COVERAGE.md new file mode 100644 index 0000000..4140053 --- /dev/null +++ b/skills/investigating-bugs/evals/COVERAGE.md @@ -0,0 +1,122 @@ +# Coverage map — investigating-bugs + +The behavior-level decomposition of `SKILL.md`, per *Decompose the skill into behaviors +first* in `slow-powers:evaluating-skills`. One entry per **functional unit** — a discrete +behavior the skill is supposed to change, together with **every lexical expression** of it +in the skill text (steps, rationalization-table rows, red-flag bullets that restate it). +An ablation of a behavior deletes all of its expressions together. + +Expressions are cited by anchor (phase/step numbers, quoted row text), not line numbers — +the text drifts, the anchors don't. Statuses come from the promoted baseline +(`baseline/NOTES.md`, Sonnet 5, 2026-07-28); every number here is traceable to that file. +All verdicts are **tier-scoped**: "ceiled" means base Sonnet 5 already does it, not that +the behavior is dead on weaker models or other harnesses. + +## Covered behaviors + +### B1. Leave durable verification behind — **CONFIRMED + ABLATION-CONFIRMED** + +Fix the bug *and* leave an artifact (test or failing-on-regression script) that would catch +it coming back — don't verify in passing and discard the evidence. + +- Expressions: Phase 4.1 ("Write a Failing Test Case … Verify it fails"); Phase 4.3's "Ensure + the new test passes" (references the artifact); the REQUIRED BACKGROUND line (TDD "Phase 4 + captures the bug with a failing test before fixing it"); red flag "Claiming a bug is fixed + without running the verification test suite" (shared with B-verify below). +- Case: `pagination-drops-last-page-cold`, assertion `left_a_regression_test` + (mutation-graded, no LLM judge). +- Status: **+55pp** (20/20 vs 9/20, p = 0.000145; pooled 30/30 vs 14/30, p = 1.9e-06). + Cost +58% tokens / +43% wall clock. The dominant unskilled failure is *writing* the + verification, running it, then **deleting it** (7 of 11 failures) — the behavior is + durability, not diligence. +- **Attribution: ablation-confirmed** (2026-07-28, pre-registered pilot, + `baseline/PRE-REGISTRATION-ABLATION-PHASE-4-1.md`): full skill vs skill minus B1's + functional unit = **10/10 vs 5/10** (Δ +0.50, Fisher p = 0.0325, n=10/arm, Sonnet 5), + control ceiled, validity clean. Deleting B1's three lexical expressions reverted the + behavior to the no-skill rate while the rest of the skill stayed loaded and invoked — + the case→behavior link in this row is validated, not assumed. + +### B2. Reproduce environment-dependent bugs by varying the context — **CEILED (Sonnet 5)** + +A green run in the default environment doesn't clear a bug that only manifests elsewhere +(timezone, locale, scale); vary the environment until the failure reproduces on demand. + +- Expressions: Phase 1.2 ("Reproduce Consistently … identify the exact steps, inputs, or + environment"; "gather more logs instead of guessing"); red flag "Writing a fix before + reproducing the bug or reading the full stack trace" (shared with B5). +- Case: `timezone-date-only-shift-cold` (`reproduced_by_varying_tz`, + `did_not_trust_green_repro`, plus the held-out TZ matrix). +- Status: 10/10 vs 9–10/10 across assertions; the discriminator + `did_not_trust_green_repro` ceiled outright at 10/10 both arms. + +### B3. Fix at the source under ship pressure; refuse the locally-green symptom patch — **CEILED (Sonnet 5)** + +- Expressions: the Iron Law banner ("NO FIXES WITHOUT ROOT CAUSE INVESTIGATION FIRST"); + Phase 1.5 ("Fix the bug at the source, not the symptom"); rationalization rows "This is an + emergency, we don't have time", "I can see the symptom fix is obvious", "We need to ship + now, investigate later", "The senior engineer says this is the fix" (authority — the seeded + case's finance sign-off exercises exactly this row). +- Case: `timezone-nudge-ship-pressure-seeded` (`refused_to_ship_the_nudge`, + `replaced_nudge_with_source_fix`, plus the held-out TZ matrix). +- Status: all four assertions 10/10 in **both** arms. + +### B4. Find the non-determinism in a flaky test before changing anything — **NOT SIGNIFICANT (Sonnet 5)** + +Don't rerun until green, don't add retries; make the failure deterministic by varying +execution context, then isolate the shared state. + +- Expressions: the Phase 1.2 flaky bullet (condition-based waiting, non-deterministic + dependency ordering) and its two references, `references/diagnosing-flaky-tests.md` and + `references/condition-based-waiting.md`. +- Case: `flaky-cross-test-pollution-seeded` (`reproduced_by_varying_context`, + `fix_isolates_shared_state`). +- Status: 10/10 vs 8/10, p = 0.47. Suggestive direction, no evidence at this n; the case + stays as a diagnostic. + +## Uncovered contingent behaviors — coverage gaps + +Each of these is contingent (an agent under pressure plausibly skips it), has no case that +would detect its removal, and is a candidate for a future case: + +- **B5. Read the full error and stack trace first** — Phase 1.1; shares the "writing a fix + before … reading the full stack trace" red flag with B2. +- **B6. Check recent changes** — Phase 1.3 (git diff, recent commits, dependency/config + changes). +- **B7. Instrument component boundaries / capture-stack tracing** — Phase 1.4 and the + Phase 1.5 instrumentation sub-bullet (log inputs at boundaries, `new Error().stack` + before the suspect operation, stderr in tests). +- **B8. Compare against working examples** — Phase 2 entire (find working examples, diff + implementations, "don't assume that difference doesn't matter", verify deps/configs). +- **B9. Single hypothesis, minimal test, revert on disproof** — Phase 3 entire; red flags + "Let's just try changing X to see if it works" and "Stacking multiple speculative fixes"; + rationalization row "This case is different because…". +- **B10. Three-fix limit — stop and reassess architecture** — Phase 4.4; rationalization + row "We tried three things, just add one more"; red flag "Each 'fix' only shifts the bug + to a new location". +- **B11. Targeted fix, no bundled refactoring** — Phase 4.2. +- **B-verify. Run the verification before claiming fixed** — Phase 4.3; red flag "Claiming + a bug is fixed without running the verification test suite" (shared with B1). Distinct + from B1: B1 is *leaving the artifact behind*, B-verify is *running it at all*. The + baseline shows unskilled Sonnet 5 mostly does B-verify (7 of 11 failures verified, then + deleted) — so B-verify is likely ceiled on this tier, but that has not been measured in + isolation. + +## Structural / shared content — not separately testable + +- The frontmatter description (trigger) — measured by the runner's invocation meta-check, + not by a case. +- The REQUIRED PREREQUISITE link to `slow-powers:working-in-isolation` — structural + cross-link, owned by that skill's own suite. +- "Violating the letter of the rules is violating the spirit of the rules", the + rationalization table and red-flags list *as scaffolding* — deliberate redundancy whose + rows attach to the behaviors above. Whether any single restatement earns its keep is a + Mode B revision question, not an ablation unit. + +## Attribution check — RAN, CONFIRMED + +`baseline/PRE-REGISTRATION-ABLATION-PHASE-4-1.md` pre-registered the pilot ablation of B1 +(full skill vs skill minus B1's functional unit, `pagination-drops-last-page-cold` only) — +the direct test that the confirmed effect is attributable to B1 rather than to the skill's +general verification ethos. It ran 2026-07-28 and the prediction held on every threshold +(10/10 vs 5/10, p = 0.0325); see the dated OUTCOME RECORD appended to that file. B1's row +above carries the result. No other behavior's attribution has been ablation-tested. diff --git a/skills/investigating-bugs/evals/NOTES.md b/skills/investigating-bugs/evals/NOTES.md new file mode 100644 index 0000000..8a6944e --- /dev/null +++ b/skills/investigating-bugs/evals/NOTES.md @@ -0,0 +1,235 @@ +# Notes — running the investigating-bugs suite + +Durable run guidance for `evals.json`. (Per-baseline observations live in +`baseline/NOTES.md`; this file is about how the suite must be invoked and how the +fixtures are meant to behave. Which case detects which part of the skill — the +behavior-level decomposition — is `COVERAGE.md`.) + +## The dispatch environment MUST be pinned to a non-negative UTC offset + +Both timezone cases depend on the agent's own environment being `TZ=UTC` (or any +non-negative offset). The trap is that the naive reproduction comes back **green**: + +- `fixtures/tz-date-only/repro.ts` PASSES under `TZ=UTC` and only FAILS under a + negative offset such as `TZ=America/Los_Angeles`. The agent must think to vary TZ. + On a host whose local zone is already negative (e.g. `America/New_York`) the bug + reproduces for free, "works fine for me" stops being misleading, and the cold case + measures nothing. +- `fixtures/tz-nudged/repro.ts` is the inverse: with the seeded `+1 day` nudge in + place it FAILS under `TZ=UTC` (renders `March 11`) and PASSES under + `America/Los_Angeles`. Pinning UTC is what makes "one account looked right" and + "the code is actually correct" come apart. + +eval-magic has no UTC default — the agent process inherits the operator's +environment (`env`/`matrix` on a `command_check` affect only the runner-owned check, +not the dispatch). Pin it at the dispatch recipe by passing the tracked descriptor +override to **every** command of the run: + +``` +eval-magic --harness-file skills/investigating-bugs/evals/harness/claude-code-utc.toml … +``` + +That descriptor also adds `--setting-sources project,local`, which unloads installed +plugins so the `without_skill` arm is genuinely skill-free, and raises the dispatch +permission mode (see below). See the comments in the file itself. + +## The agent must actually be able to RUN things + +eval-magic's built-in Claude Code recipe dispatches with `--permission-mode +acceptEdits`. That auto-approves file edits but **not** Bash, and because the recipe +detaches stdin (`/dev/null` redirect. +If you re-derive these recipes, keep that property — check that a dispatch can run +`bun` and that an out-of-env write is still refused. + +**Known noise:** the guard classifies `2>/dev/null` as an out-of-sandbox redirect and +blocks it. Agents use that idiom habitually, so expect a few benign entries in +`guard-denials.json` and the matching `validity_warnings`; they cost the agent a +retry, not correctness. + +Verify the fixtures before a run: + +``` +cd skills/investigating-bugs/evals/fixtures +(cd tz-date-only && TZ=UTC bun repro.ts) # -> PASS (exit 0) +(cd tz-date-only && TZ=America/Los_Angeles bun repro.ts) # -> FAIL (exit 1) +(cd tz-nudged && TZ=UTC bun repro.ts) # -> FAIL (exit 1, "March 11") +(cd tz-nudged && TZ=America/Los_Angeles bun repro.ts) # -> PASS (exit 0) +``` + +## Two fixture variants, one bug + +`tz-date-only/` and `tz-nudged/` are identical except for two lines: the nudged copy +already contains `d.setDate(d.getDate() + 1)` in `formatDueDate`. The seeded case's +premise is that a previous turn applied that nudge, so the code the agent opens has +to actually contain it — otherwise there is no nudge to refuse to ship and none to +remove, which is exactly what its two judges grade. Keep the two directories in sync +apart from that nudge (and keep each one's `holdout/` copy identical; a shared copy +cannot resolve `../display` from both). + +**Do not reintroduce hints into the fixture comments.** An earlier draft of +`repro.ts` carried a `NOTE:` explaining that the script only fails under a negative +offset and that "reproducing the reported bug means varying TZ", and printed the +active `TZ` in its output. That hands the agent both TZ-related assertions and would +ceil them in *both* arms — the same "the case does the skill's work" failure that +ceiled the previous two suites. Fixture comments should describe what the code is +for, never how to find the bug. + +## The held-out matrix is the objective decoy-catcher + +`holdout/tz-matrix.holdout.ts` is **held out** — it is NOT in any case's `files` +array, and it is named `.holdout.ts` so the repo's own `bun test` never collects it. +It asserts display correctness, the `serializeForSave` round-trip, and `isDueToday` +across a timezone matrix. + +It is wired as the `all_consumers_correct_in_every_tz` `command_check` on both +timezone cases: eval-magic injects the file after the agent finishes (never during +staging, so the agent cannot see or satisfy it) and runs it once per cell of +`TZ ∈ { UTC, America/Los_Angeles, Pacific/Kiritimati, Europe/Berlin }`. The +assertion passes only if every cell passes. + +Every tempting wrong fix fails at least one cell — verified: + +| Final state | UTC | America/Los_Angeles | Pacific/Kiritimati | Europe/Berlin | +|---|:--:|:--:|:--:|:--:| +| unfixed (`tz-date-only` as staged) | pass | **fail** | pass | pass | +| `+1 day` nudge (`tz-nudged` as staged) | **fail** | **fail** | **fail** | **fail** | +| "force local parse" (`new Date(ymd + "T00:00:00")`) | pass | pass | **fail** (save) | **fail** (save) | +| calendar-date handling (string, or consistent UTC) | pass | pass | pass | pass | + +The "force local parse" row is the important one: it is the fix a model reaches for +first, it makes display correct in every zone, and it still corrupts the `save` +round-trip for positive offsets because `toISOString()` rolls the day back. + +Run it by hand across cells with: + +``` +cd skills/investigating-bugs/evals/fixtures/tz-date-only +for tz in UTC America/Los_Angeles Pacific/Kiritimati Europe/Berlin; do + TZ=$tz bun test ./holdout/tz-matrix.holdout.ts +done +``` + +**Reading a failure:** a cell can also fail because the agent restructured the module +(renamed or removed `formatDueDate` / `serializeForSave` / `isDueToday`, or moved +them) rather than because its fix is wrong. That is a false negative — inspect the +per-cell stderr under `command-checks/` before scoring it as a bad fix. + +## The durable-verification case grades by mutation + +`pagination-drops-last-page-cold` measures something the timezone cases are blind to: +not whether the agent fixes the bug, but whether it leaves behind anything that would +catch the bug if it came back. The bug itself (a dropped trailing page) is a +deliberately trivial one-line fix — `fix_is_correct` is the control, not the +measurement. + +`holdout/verify-regression-test.sh` grades it objectively, in two phases: + +1. Everything the agent left behind must be **green against the agent's own fix**. + This rules out a broken or half-written test scoring as a pass. +2. The original buggy `chunk.ts` is restored from the held-out pristine copy and + everything is re-run. At least one artifact must now **fail**. + +Artifacts are collected by shape rather than name, so a `*.test.ts`, a `*.spec.ts`, +or a plain repro script all count — Phase 4 of the skill accepts "an automated test +or simple script that consistently triggers the bug". A script that prints the +symptom without exiting non-zero does not count, and should not. + +Verified against five hand-built final states, re-checked after every fixture change: +fixed with no test → fail; fixed with a real regression test → pass; fixed with a +vacuous (`typeof chunk === "function"`) test → fail; never fixed → fail; fixed with a +repro script that exits non-zero → pass. + +### Why there is no `chunk` test in the fixture + +The first version of this case shipped a `chunk.fixture.ts` covering only exact +multiples — realistic, since that is why the bug would ship. It **ceiled immediately**: +Sonnet 5 scored 3/3 in both arms, and every unskilled run simply appended a correct +regression test to the file that was already sitting there. Handing the agent an +obvious place to put a test removes the decision being measured. + +Deleting all tests would overcorrect in the other direction — in a project with no +tests at all, declining to write one is a defensible reading of local convention, so a +failure would not mean what we want it to mean. + +So the fixture ships `formatBytes.ts` **with** `formatBytes.fixture.ts`, and `chunk` +with nothing. The project visibly tests its code; the buggy function is visibly +untested. Creating verification where none exists is then unambiguously right, and +unambiguously a choice. The `formatBytes` tests stay green in both phases of the +mutation check, so they can never satisfy it by themselves. + +**If this case ceils again, do not reach for a third fixture variant.** It has already +been revised once after seeing results; another data-driven redesign is suite tuning, +not measurement. Pre-register the prediction, run it once, and accept the number. + +**Outcome (2026-07-28): it did not ceil — v2 works, and this case now carries the +baseline.** Discovery n=10/arm: 10/10 vs 5/10, p = 0.0325. Confirmatory replication +n=20/arm: **20/20 vs 9/20, p = 0.000145**. Pooled 30/30 vs 14/30, p = 1.9 × 10⁻⁶. +The fixture is frozen — do not touch it to chase a nicer number. + +The failure modes are worth knowing, because they are not what you would guess: of +the 11 failing `without_skill` runs in the replication, **7 wrote verification inside +the sandbox, ran it, and then deleted it**; only 4 never wrote one at all. The case +measures *durability*, not diligence. + +**Gotcha if you edit the script:** paths must keep their leading `./`. +`bun test chunk.fixture.ts` treats a bare filename as a name *filter*, matches +nothing, and exits non-zero — which fails phase 1 for every run regardless of what +the agent did. `bun test ./chunk.fixture.ts` treats it as a path. + +## Deliberately not asserted + +- **`diff_scope`.** Captured automatically for every run and worth reading as + diagnostic context, but not a threshold: the smallest diff is not necessarily the + best change, and the correct fix here legitimately touches more than one consumer. +- **`must_precede: "first_write"`** on the TZ transcript check. `first_write` counts + *any* write, including writing a repro script — which Phase 4 of the skill actively + asks for — so it would penalise the disciplined path. +- **A held-out check for the flaky case.** Its rubric explicitly accepts a test-level + reset as a valid fix, so a held-out pollution pair (which would not call that reset) + would contradict the rubric it is supposed to enforce. + +## Possible future work + +- A multi-turn case using eval-magic's scripted `turns`, to test behaviour on an + under-specified bug report (does the agent ask what "wrong date" means before + editing?). The runner supports it; no case uses it yet. +- A second non-timezone case with the same decoy richness. A `float-money-rounding` + control was considered and deferred: its "correct" answer is policy-ambiguous + (per-line vs per-total rounding), which makes a noisy judge, and a clean + deterministic small-bug control with an obvious right answer tends to ceil — the + exact failure this suite is built to avoid. A locale-dependent parse would ride on + the same `matrix` support the timezone case uses. + +## Case status — measured on Sonnet 5, n=10/arm (2026-07-28) + +- `pagination-drops-last-page-cold` — **carries the baseline.** 20/20 vs 9/20, + p = 0.000145 at n=20/arm. Graded entirely by runner-owned command checks: a + correctness control plus the mutation check above. Fixture frozen. +- `timezone-nudge-ship-pressure-seeded` — **CEILED.** All four assertions 10/10 in + *both* arms. +- `timezone-date-only-shift-cold` — **effectively ceiled.** 10/10 vs 9/10 or 10/10 + throughout; `did_not_trust_green_repro` ceiled outright at 10/10 vs 10/10. +- `flaky-cross-test-pollution-seeded` — 10/10 vs 8/10, p = 0.47. Not significant. + +Base Sonnet 5 already varies TZ to reproduce, distrusts a green repro, and isolates +shared state. **The ceiled cases are deliberately kept.** Deleting a case because it +stopped flattering the skill is how a suite gets tuned into agreement; they stay so +the next tier's run can show whether the ceiling is real. They are diagnostic, not +evidence of value — never quote them as a delta. + +Baseline is **PROMOTED** for durable verification only — see `baseline/NOTES.md` for +the scope limits, and read them before quoting any number from this suite. diff --git a/skills/investigating-bugs/evals/baseline/BASELINE.md b/skills/investigating-bugs/evals/baseline/BASELINE.md index 5010072..dc6934e 100644 --- a/skills/investigating-bugs/evals/baseline/BASELINE.md +++ b/skills/investigating-bugs/evals/baseline/BASELINE.md @@ -1,24 +1,24 @@ # Baseline — investigating-bugs -**Status: PENDING (no promoted baseline).** +Committed reference output from a canonical eval run. Regenerate with +`eval-magic promote-baseline --iteration 2` after aggregating. The ephemeral workspace (run records, timing, +dispatch files, produced outputs) stays gitignored under `.eval-magic/` +and is reclaimable by `eval-magic teardown` once promoted (this commit's marker). -The eval suite was rebuilt from scratch (slim, de-scaffolded, outcome-graded — see -`evals.json`). The previous baseline measured the legacy 6-case suite, which ceiled -(with_skill 1.00 vs without_skill 0.967, +3.3pp on Sonnet 4.6), so its `benchmark.json` -and `grading/` were removed rather than left to mislead. +| Field | Value | +|-------|-------| +| Mode | new-skill | +| Iteration | iteration-2 | +| Harness | claude-code | +| Agent model | claude-sonnet-5 | +| Judge model | claude-sonnet-5 | +| Conditions | with_skill, without_skill | +| Run timestamp | 2026-07-28T06:42:45.330Z | +| Label | investigating-bugs-durable-verification | +| Promoted from commit | 6fbdb43 | -No baseline is committed for the new suite yet. Per the Iron Law in -`slow-powers:evaluating-skills`, do not promote one until a fresh Mode A run shows a -positive delta. To regenerate: +Files: +- `benchmark.json` — aggregate pass-rate / duration / token deltas. +- `grading/__.json` (multi-run cells add an `__r` suffix per run) — assertion results and judge rationales. +- `NOTES.md` — operator-authored observations for this baseline (never overwritten by promote). -``` -eval-magic run --skill-dir ./skills --skill investigating-bugs --bootstrap ./bootstrap.md --runs 5 -# ...read the per-assertion deltas, iterate if needed, then: -eval-magic promote-baseline --skill-dir ./skills --skill investigating-bugs --mode new-skill --overwrite -``` - -Target model for the headline: **Sonnet 4.6** (agent + judge). Present the -`evaluating-skills` pre-flight summary and arm `--guard` before dispatching. - -Once promoted, this file is overwritten with the run metadata table, and -`benchmark.json` / `grading/__.json` are written alongside it. diff --git a/skills/investigating-bugs/evals/baseline/NOTES.md b/skills/investigating-bugs/evals/baseline/NOTES.md new file mode 100644 index 0000000..7c589d8 --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/NOTES.md @@ -0,0 +1,101 @@ +# Notes — investigating-bugs + +Promoted from iteration-2 at 2026-07-28. + +**Status: PROMOTED (durable verification only). Read the scope limits before +quoting this anywhere.** + +## What this baseline measures + +One thing, measured well: **does the agent leave behind verification that would +catch the bug if it came back?** + +| | with skill | without skill | Δ | p | +|---|:--:|:--:|:--:|:--:| +| `left_a_regression_test` (primary) | **20/20** | **9/20** | **+0.55** | **0.000145** | +| `fix_is_correct` (control) | 20/20 | 20/20 | 0 | ceiled by design | + +Graded entirely by **mutation**, with no LLM judge in the loop: everything the agent +left behind must pass against its own fix, then the pristine buggy `chunk.ts` is +restored and something must now fail. Artifacts are collected by shape, so a test +file or a plain repro script both count. + +**Cost, which belongs in the same breath as the gain:** + +| | with | without | cost | +|---|---|---|---| +| tokens | 608,242 | 385,275 | **+57.9%** (5.19 SE, 95% CI [+36.0%, +79.7%]) | +| wall clock | 78.9 s | 55.3 s | **+42.6%** (4.42 SE, 95% CI [+23.7%, +61.5%]) | + +Do **not** quote this run's `pass_rate` (1.000 vs 0.725). It averages the ceiled +control in with the real measurement and dilutes a +55pp effect to +27.5pp. + +## Why the failures happened — this is the interesting part + +Of the 11 `without_skill` runs that failed the primary: + +- **7 wrote verification inside the sandbox, ran it, confirmed the fix, then deleted it.** +- **4 never wrote any verification at all.** + +The unskilled agent mostly *does* verify. It just doesn't leave the verification +behind. That is the specific behavior this skill changes. + +## Provenance and honesty limits + +- **Pre-registered before dispatch** in `PRE-REGISTRATION-P1-REPLICATION.md`, with the + prediction set deliberately looser than the discovery estimate (Δ ≥ +0.25 rather + than the observed +0.50) and an explicit failure branch. It cleared every threshold. +- **This is a replication.** The discovery run (`PRE-REGISTRATION.md`, n=10/arm) gave + 10/10 vs 5/10, p = 0.0325 — but on an unplanned model, so it could not stand alone. + Pooled with this run: 30/30 vs 14/30, p = 1.9 × 10⁻⁶. +- **Single case, single tier.** `claude-sonnet-5` only. This says nothing about Haiku + 4.5 and nothing about the skill's other teachings. + +## Validity + +- Skill invocation 20/20. `live_source_reads` 0. Stray-write violations 0. +- 33 guard denials, **all of them the Bash-redirect false positive** + (eval-magic#179) — no `Write` was ever blocked, so the guard cannot explain a + missing test file. Distribution is near-symmetric (18 `with_skill` / 15 + `without_skill`), so it cannot bias the delta either. +- The two denied `without_skill` runs in the *discovery* run were checked at + transcript level for the same reason: both recovered, wrote in-sandbox, and then + deleted their own work. + +## What ceiled — measured, not assumed + +The full 4-case suite ran at n=10/arm on Sonnet 5 (see `PRE-REGISTRATION.md`'s +deviation record). On this tier: + +- `timezone-nudge-ship-pressure-seeded` — **all four assertions 10/10 in both arms.** +- `timezone-date-only-shift-cold` — 10/10 vs 9/10 or 10/10 throughout; the + discriminator `did_not_trust_green_repro` ceiled outright at 10/10 vs 10/10. +- `flaky-cross-test-pollution-seeded` — 10/10 vs 8/10, p = 0.47. + +Base Sonnet 5 already reproduces timezone bugs by varying TZ, distrusts a green +repro, and isolates shared state. Those cases are kept because deleting a case after +seeing it ceil is how a suite gets tuned into telling you what you want to hear — but +they are not evidence of skill value on this tier, and must not be quoted as such. + +## The efficiency claim that died here + +An earlier Haiku 4.5 run showed **−41% tokens / −32% wall clock** with the skill, and +it was very nearly the advertised headline. On Sonnet 5 it **reverses** to +58% / +43%. + +The saving was never a property of the skill — it was unskilled Haiku flailing, and it +does not survive a tier change. **Do not revive it.** If efficiency is reported for +this skill, it is reported as a cost. + +## Regenerating + +``` +eval-magic run --harness-file skills/investigating-bugs/evals/harness/claude-code-utc.toml \ + --skill-dir ./skills --skill investigating-bugs --bootstrap ./bootstrap.md \ + --only pagination-drops-last-page-cold --runs 20 \ + --agent-model claude-sonnet-5 --judge-model claude-sonnet-5 --guard +``` + +**Pass `--agent-model` explicitly and verify it in a smoke dispatch's events file +before dispatching the fleet.** Omitting it silently falls through to the session +default with no error — that mistake cost a full 80-dispatch run in this campaign. +The `--harness-file` is required on *every* eval-magic command of the run. diff --git a/skills/investigating-bugs/evals/baseline/PRE-REGISTRATION-ABLATION-PHASE-4-1.md b/skills/investigating-bugs/evals/baseline/PRE-REGISTRATION-ABLATION-PHASE-4-1.md new file mode 100644 index 0000000..54792bf --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/PRE-REGISTRATION-ABLATION-PHASE-4-1.md @@ -0,0 +1,150 @@ +# Pre-registration — B1 ablation pilot (attribution of durable verification) + +Written **before** any dispatch, 2026-07-28. Frozen on commit. **Not yet dispatched** — +the run happens only after the operator confirms a pre-flight summary (cases, mode, +models, cost, guard). + +## What this run is + +The first **ablation run** under the behavior-decomposition process (see +`slow-powers:evaluating-skills`, *Pricing a behavior — ablation runs*, and this suite's +`../COVERAGE.md`). It tests one claim directly: + +> The promoted baseline's confirmed effect — the agent leaves durable verification +> behind (30/30 vs 14/30 pooled, p = 1.9e-06) — is **attributable to behavior B1's +> explicit text**, not to the skill's diffuse verification ethos. + +Mechanically a Mode B (revision) run with the expectation inverted: `old_skill` = the +full `investigating-bugs` SKILL.md, `new_skill` = the same skill with **B1's entire +functional unit deleted**. We are hoping the ablated arm gets *worse*. Either outcome is +informative; neither outcome changes the promoted baseline. + +## The ablated variant — exact scope, frozen + +B1 (*leave durable verification behind*) is a functional unit, not a lexical section. +The ablation deletes every expression of it, and only it — the neighboring behavior +B-verify (*run the verification before claiming fixed*, Phase 4.3) must survive, so one +sentence is rewritten rather than deleted to remove its dangling reference to the new +test. Exact diff against `SKILL.md` (unchanged since commit `67dcfdc`): + +```diff +@@ REQUIRED BACKGROUND (B1: test-first bug capture) — delete the whole line +-> **REQUIRED BACKGROUND:** You must understand `slow-powers:test-driven-development` — Phase 4 captures the bug with a failing test before fixing it. +@@ Phase 4 — delete step 1, renumber; rewrite step 3's B1 reference +-1. **Write a Failing Test Case:** Create an automated test or simple script that consistently triggers the bug. Verify it fails. +-2. **Implement the Fix:** Make a single, targeted change that directly addresses the root cause. Do not bundle unrelated refactoring. +-3. **Verify the Fix:** Run the test suite. Ensure the new test passes and no regressions are introduced. +-4. **The Three-Fix Limit (Architectural Check):** ++1. **Implement the Fix:** Make a single, targeted change that directly addresses the root cause. Do not bundle unrelated refactoring. ++2. **Verify the Fix:** Run the test suite. Ensure the reported symptom is gone and no regressions are introduced. ++3. **The Three-Fix Limit (Architectural Check):** +@@ Red Flags — delete the B1 bullet +-- Claiming a bug is fixed without running the verification test suite +``` + +Scoping notes, decided now so they cannot drift under the result: + +- The Phase 4.3 rewrite ("the new test passes" → "the reported symptom is gone") is the + minimal edit that keeps B-verify intact without a dangling reference. It introduces no + new instruction. +- The deleted red-flag bullet is shared between B1 and B-verify; B-verify retains its + Phase 4.3 step, so deleting the bullet removes only redundancy for B-verify while + removing B1's last restatement. Deliberate redundancy is why the whole unit must go + together — a surviving restatement compensates. +- Nothing else changes. The Iron Law, all of Phases 1–3, and every other table row stay + verbatim. + +## Declared parameters — no defaults, no inference + +- **Mode:** revision (Mode B). `old_skill` = full SKILL.md; `new_skill` = ablated + variant above. +- **Case:** `--only pagination-drops-last-page-cold`. Primary endpoint + `left_a_regression_test`; control `fix_is_correct`. +- **n = 10 per condition** (20 agent dispatches). Discovery-sized: the effect being + probed cleared p = 0.0325 at this n in the discovery run. +- **Agent model:** `claude-sonnet-5`, passed explicitly via `--agent-model`. + **Judge model:** `claude-sonnet-5`, via `--judge-model`. The model id recorded in a + smoke dispatch's events file must read `claude-sonnet-5` before the fleet goes out. +- **Harness descriptor:** `harness/claude-code-utc.toml` passed to **every** command. +- **Sandbox:** guard armed, allowlist unchanged from the replication run. +- **Fixture frozen** (unchanged since `67dcfdc`). Not touched, whatever the outcome. + +## Prediction + +Under the attribution hypothesis, the ablated arm loses what `without_skill` lacked — +the pooled baseline rates are full-skill 30/30 (1.00) and no-skill 14/30 (0.47): + +> `old_skill` (full) ≥ 0.85; `new_skill` (ablated) ≤ 0.65; Δ(full − ablated) ≥ +0.25; +> Fisher exact two-sided p < 0.05. + +Control, expected to ceil: `fix_is_correct` at or near 1.00 in both arms. If it fails +in either arm, the run is compromised and the primary must not be read. + +## Decision rule — one run, accepted either way + +Validity gate first: skill invocation 1.0 in **both** arms (both arms carry a skill); +`live_source_reads` 0; guard denials symmetric and none shown at transcript level to +have caused a primary failure. + +- **Prediction holds → attribution confirmed.** B1's text is what buys the effect. + Record in `../COVERAGE.md` (B1 status gains "ablation-confirmed"); the coverage + process gets its first validated case→behavior link. +- **Prediction fails (ablated arm holds ≥ 0.85, or Δ < +0.25, or p ≥ 0.05) → the + effect is not attributable to B1's explicit text at this n.** Record it as *diffuse*: + the skill's remaining text (Iron Law framing, investigation ethos) carries the + behavior. Revise `../COVERAGE.md`'s B1 attribution note accordingly. Do **not** + widen the deletion to chase a collapse, do **not** raise n and retry, do **not** + touch the fixture or the live skill. +- An intermediate result is a **failed prediction** under this rule, not a partial + success to be argued upward. + +## What may not be claimed from this run + +- Anything about the skill's overall value — the promoted baseline already carries + that, and this run cannot strengthen or weaken it. +- Any change to the live `SKILL.md`. A confirmed attribution does not license trimming + other behaviors ("the rest didn't matter" is exactly the sub-additivity error); + a diffuse result does not license adding text. +- Anything about other tiers or harnesses. `claude-sonnet-5` only. + +--- + +## OUTCOME RECORD — 2026-07-28 (appended after the run; the text above is frozen) + +**Prediction HELD on every threshold. Attribution CONFIRMED.** + +| | full (`old_skill`) | ablated (`new_skill`) | rule | met | +|---|:--:|:--:|:--:|:--:| +| `left_a_regression_test` (primary) | **10/10** | **5/10** | full ≥ 0.85, ablated ≤ 0.65 | ✓ | +| Δ (full − ablated) | **+0.50** | | ≥ +0.25 | ✓ | +| Fisher exact two-sided | **p = 0.0325** | | < 0.05 | ✓ | +| `fix_is_correct` (control) | 10/10 | 10/10 | ~1.00 both arms | ✓ | + +The ablated arm landed exactly on the discovery run's `without_skill` rate (5/10) — +deleting B1's three lexical expressions reverted the measured behavior to the no-skill +baseline while the skill was still loaded and invoked (invocation 10/10 in both arms). + +**Validity (gate passed):** invocation 1.0/1.0; `live_source_reads` 0; model id +`claude-sonnet-5` verified in a pre-fleet smoke dispatch and across all 20 event files. +Guard denials 7 (full) vs 15 (ablated) — not symmetric in count, but checked causally +per the rule: failing runs 1 and 3 had zero denials; runs 5 and 10 were each the +`/dev/null` redirect false positive (eval-magic#179); run 2's was a `/tmp/repro.ts` +scratch block after which the agent wrote the repro in-env, ran it, and deleted it by +choice. No denial caused a primary failure. One stray-write violation (`/tmp/repro.ts`, +ablated run 8) was in a *passing* run and outside grading scope. + +**Failure texture (n=5, observation not claim):** ablated failures split 1 +wrote-in-env-then-`rm`'d / 4 never-wrote-a-file (all 4 verified via 3–5 inline +`bun`/`node` executions instead). The final diffs of the five failures are all +1 file / ±1 line; all ten full-skill diffs are 2 files / ~20 lines. + +**Cost texture (observation not claim):** full arm 508k ± 54k tokens; ablated arm +849k ± 359k, with its five *passing* runs occupying the top five token counts +(881k–1.48M). With B1's text present, durable verification was reliable and cheap; +without it, it was a coin flip that cost 2–3× when it happened. + +**Disposition per the decision rule:** recorded in `../COVERAGE.md` (B1 → +ablation-confirmed). Baseline untouched. Fixture untouched. Live SKILL.md untouched — +this result does not license trimming anything else (sub-additivity), and the working +tree ablation was reverted after workspace staging (verified: tree clean, staged arms +differed by exactly the frozen diff). diff --git a/skills/investigating-bugs/evals/baseline/PRE-REGISTRATION-P1-REPLICATION.md b/skills/investigating-bugs/evals/baseline/PRE-REGISTRATION-P1-REPLICATION.md new file mode 100644 index 0000000..7ad7eb7 --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/PRE-REGISTRATION-P1-REPLICATION.md @@ -0,0 +1,83 @@ +# Pre-registration — P1 confirmatory replication (durable verification) + +Written **before** dispatch on 2026-07-28, after the n=10 discovery run +(see `PRE-REGISTRATION.md` and its DEVIATION RECORD). Frozen on commit. + +## What this run is + +A **confirmatory replication of a single pre-specified hypothesis**, not a suite run. + +P1 was pre-registered before the discovery run and hit its stated numbers — but on an +unplanned population (Sonnet 5 rather than the declared Haiku 4.5), which weakens it. +This run tests the same hypothesis on an **explicitly declared** population, with P1 +as the stated primary endpoint rather than one prediction among seven. + +**Discovery result being replicated:** `pagination-drops-last-page-cold` / +`left_a_regression_test` — `with_skill` 10/10, `without_skill` 5/10, Δ = +0.50, +Fisher exact two-sided **p = 0.0325**. + +## Why only one case + +The other three cases are measured, not discarded on a whim: + +- `timezone-nudge-ship-pressure-seeded` — all four assertions 10/10 in **both** arms. + Ceiled; measures nothing on this tier. +- `timezone-date-only-shift-cold` — 10/10 vs 9/10 or 10/10 across its assertions; + the pre-registered discriminator `did_not_trust_green_repro` ceiled outright at + 10/10 vs 10/10. +- `flaky-cross-test-pollution-seeded` — 10/10 vs 8/10, p = 0.47. + +Running one case here is **narrowing to the declared primary**, which is legitimate +for a confirmatory test. It is *not* a claim about the skill's overall pass rate, and +the result may not be reported as one. The full-suite numbers stay on the record in +`PRE-REGISTRATION.md`, including the ceilings. + +## Declared parameters — no defaults, no inference + +- **Agent model:** `claude-sonnet-5`, passed explicitly via `--agent-model`. +- **Judge model:** `claude-sonnet-5`, passed explicitly via `--judge-model`. +- **Verification before the fleet:** the model id recorded in a smoke dispatch's + events file must read `claude-sonnet-5`. Omitting these flags silently falls + through to the session default — that is exactly what corrupted the previous run, + and it fails without any error. +- **n = 20 per condition** (40 agent dispatches). Both assertions are `command_check`, + so judge cost is limited to the skill-invocation meta-check. +- **Sandbox unchanged** from the discovery run — same guard, same allowlist. Altering + it between discovery and replication would introduce a second difference and make a + disagreement uninterpretable. +- **Fixture frozen** at commit `67dcfdc`. Not touched, whatever the outcome. + +## Prediction + +Stated in advance, deliberately **looser than the discovery point estimate** so the +test is not a rubber stamp on 10/10-vs-5/10: + +> `with_skill` ≥ 0.85, `without_skill` ≤ 0.65, **Δ ≥ +0.25**, Fisher exact +> two-sided **p < 0.05**. + +Secondary, expected to ceil (it is the control, not the measurement): +`fix_is_correct` at or near 1.00 in both arms. If `fix_is_correct` *fails* in either +arm, the run is compromised — the bug is a one-line fix and both arms should manage +it — and the primary result must not be read. + +## Decision rule + +**Replication succeeds** if Δ ≥ +0.25 **and** p < 0.05 **and** validity is clean +(invocation rate 1.0 on `with_skill`; `live_source_reads` 0; guard denials +symmetric across arms; no denial shown to have *caused* a `without_skill` failure — +check the transcripts, do not assume). + +- **On success:** the finding is that `investigating-bugs` changes whether an agent + leaves durable verification behind, on the current tier, with a stated token cost. + That, and not a pass-rate percentage, is what may be advertised. +- **On failure:** report the discovery result as **unreplicated** and promote nothing. + Do **not** re-tune the fixture, do **not** raise n and try again, and do **not** + fall back to quoting the discovery p-value. One replication, accepted either way. + +## What may not be claimed from this run + +- Any aggregate "improvement vs no skill" pass-rate number. +- Any token or wall-clock *saving*. The discovery run measured **+24.8% tokens** on + this tier; the earlier −41% was unskilled-Haiku flailing, not a skill property. + Efficiency is reported here as a **cost**, with its interval, or not at all. +- Anything about Haiku 4.5. This run does not touch that population. diff --git a/skills/investigating-bugs/evals/baseline/PRE-REGISTRATION.md b/skills/investigating-bugs/evals/baseline/PRE-REGISTRATION.md new file mode 100644 index 0000000..9d04279 --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/PRE-REGISTRATION.md @@ -0,0 +1,212 @@ +# Pre-registration — investigating-bugs Mode A, Haiku 4.5, n=10 + +Written **before** dispatch on 2026-07-28. Predictions, analysis plan, and decision +rules are fixed here so the run measures the skill rather than our willingness to +keep tuning until a number looks good. Nothing below may be revised after results +are read; a revision that happens anyway must be recorded as such, with its date. + +## Why this run exists + +Run B (Haiku 4.5, n=3/cell, 4 cases) produced an aggregate pass-rate delta of +**+11.9pp** (0.688 vs 0.569) that does not clear noise — stddev ≈ 0.383 in both +arms gives SE(Δ) ≈ 15.6pp, so the interval spans zero. It also produced an +efficiency delta that *does* clear noise even at n=3 (≈2.9 SE), and three +assertion-level movers whose evidence is 3 runs per cell — too thin to advertise. + +This run buys three things and, importantly, does not buy a fourth: + +1. Tightens the efficiency estimate from a ±28pp interval to a ±15pp one. +2. Powers two of the three assertion-level discriminators to individual significance. +3. Answers the pagination-v2 question, which is currently unmeasured. +4. **Does not** make the aggregate pass-rate delta significant. See below. + +## Observed baseline (Run B, n=3/cell) — the numbers being tested + +Per-assertion pass counts, `with_skill` vs `without_skill`: + +| Case | Assertion | with | without | Status | +|---|---|:--:|:--:|---| +| tz-cold | `did_not_trust_green_repro` | 2/3 | 0/3 | **mover** | +| tz-cold | `reproduced_by_varying_tz` | 2/3 | 1/3 | **mover (underpowered)** | +| tz-cold | `all_consumers_correct_in_every_tz` | 1/3 | 1/3 | flat | +| tz-cold | `fixed_as_calendar_date` | 1/3 | 1/3 | flat | +| tz-seeded | `refused_to_ship_the_nudge` | 3/3 | 3/3 | ceiled | +| tz-seeded | `replaced_nudge_with_source_fix` | 0/3 | 0/3 | floored | +| tz-seeded | `all_consumers_correct_in_every_tz` | 0/3 | 0/3 | floored | +| tz-seeded | `reproduced_by_varying_tz` | 0/3 | 0/3 | floored | +| pagination | `fix_is_correct` | 3/3 | 3/3 | ceiled (control, by design) | +| pagination | `left_a_regression_test` | 3/3 | 3/3 | ceiled — **v1 fixture, since replaced** | +| flaky | `fix_isolates_shared_state` | 3/3 | 1/3 | **mover** | +| flaky | `inspected_before_patching` | 3/3 | 3/3 | ceiled | +| flaky | `reproduced_by_varying_context` | 3/3 | 3/3 | ceiled | + +Efficiency: **406,059 ± 206,643** tokens and **72.0 ± 22.7 s** with the skill, vs +**691,719 ± 268,399** tokens and **105.4 ± 30.9 s** without. + +## Predictions + +Stated in advance. A miss is a finding, not a reason to re-tune the suite. + +**P1 — pagination v2 discriminates.** `left_a_regression_test`: `with_skill` ≥ 0.8, +`without_skill` ≤ 0.5, Δ ≥ +0.3. The v1 fixture shipped `chunk.fixture.ts` and gave +the answer away; v2 ships `formatBytes` with tests and `chunk` with none, so writing +verification is now a choice rather than a fill-in-the-blank. *If this ceils again, +record the null and stop — do not build a third fixture variant.* + +**P2 — `did_not_trust_green_repro` holds up.** Δ ≥ +0.4, Fisher exact p < 0.05. + +**P3 — `fix_isolates_shared_state` holds up.** Δ ≥ +0.4, Fisher exact p < 0.05. + +**P4 — `reproduced_by_varying_tz` moves but stays inconclusive.** Δ ≈ +0.33; at +n=10 a 7/10-vs-3/10 split gives p ≈ 0.18. **Declared underpowered in advance** — a +non-significant result here is expected and is not evidence against the skill. + +**P5 — efficiency is the robust result.** Δ tokens ≈ −41% and Δ wall clock ≈ −32%, +both p < 0.001 at n=40/condition (≈5.3 and ≈5.5 SE respectively). + +**P6 — aggregate pass-rate stays inconclusive.** Δ ≈ +0.10 to +0.15, roughly 1.4 SE. +**Declared underpowered in advance.** See the next section — this is the single most +important line in this document. + +**P7 — the ceiled and floored assertions stay flat.** Both tz-seeded arms remain at +0/n on `all_consumers_correct_in_every_tz`; `refused_to_ship_the_nudge`, +`inspected_before_patching`, `reproduced_by_varying_context` and `fix_is_correct` +remain at or near 1.0 in both arms. + +## The aggregate pass-rate cannot be rescued by this run + +At n=10/case (40 runs/condition) and the observed stddev of 0.383: + +- SE(Δ) = 0.383·√(2/40) = **8.6pp**. Against an observed Δ of 11.9pp that is + **1.39 SE** — p ≈ 0.16. Not significant. +- A by-case stratified estimate (the pre-specified secondary analysis) improves SE + to roughly **7.9pp** → 1.51 SE, p ≈ 0.13. Still not significant. +- Reaching 80% power for a true Δ of 11.9pp would need ≈160 runs per condition — + **40 per case, 320 agent dispatches.** That is not a defensible spend for one + README cell. + +This is a property of the suite, not of the run size: two of four cases contribute +structurally zero signal on Haiku (one floored, one a ceiled control), which halves +any delta while contributing full variance. **The aggregate pass-rate is therefore +demoted to a secondary endpoint and will not be the advertised headline regardless +of which side of the line it lands on.** Promoting it after the fact because it +happened to clear would be exactly the post-hoc selection this document exists to +prevent. + +## Analysis plan + +- **Primary endpoints:** token delta and wall-clock delta, reported as point + estimate with a 95% CI. Welch, unpaired. +- **Co-primary:** P2 and P3 by Fisher's exact test, two-sided, no multiplicity + correction — both are pre-registered single hypotheses. +- **Secondary:** aggregate pass-rate delta (naive and by-case stratified), reported + with its interval and its non-significance stated plainly. +- **Exploratory:** every other assertion. Thirteen assertions are graded, so at + α=0.05 roughly one false positive is expected by chance; any mover not named in + P1–P3 is reported as exploratory and is **not** advertisable on this run alone. +- **Validity gates, checked before any number is believed:** skill-invocation rate + on `with_skill` must be 1.0; `missing_gradings` must be 0; `stray-writes.json` + must be empty; guard denials are reviewed individually and runs with a denial that + changed the outcome are reported, not silently kept. The `plugin-shadow.json` + warnings are known false positives — the descriptor's `--setting-sources + project,local` isolation was verified empirically (a probe dispatch saw the staged + siblings and no `slow-powers:` plugin skills), and the preflight cannot see + descriptor-level isolation (eval-magic#179, item 4). + +## Decision rules + +**Promote the baseline** only if all three hold: + +1. Aggregate pass-rate delta is positive (direction, not significance), **and** +2. at least one of P2/P3 reaches p < 0.05, **and** +3. all validity gates above are clean. + +**README headline** is the efficiency result with its confidence interval, framed as +process/cost rather than correctness. The pass-rate column reports the delta *and* +its interval, or reports honestly that it is inconclusive. If the efficiency result +is the only thing that survives, that is what gets advertised — see `README-FRAMING` +below. + +**Do not promote** if the aggregate delta is negative, or if both P2 and P3 fail. In +that case the finding is that `investigating-bugs` does not measurably change outcomes +at any tier tested, and it gets written up as such. + +**No case may be dropped, and no fixture revised, after results are read.** The suite +is frozen as of commit `67dcfdc`. + +## README framing + +The current sizzler table implies a single "improvement" number per skill, which +suits a failure-prevention skill and does not suit this one. `investigating-bugs` is +a quality-gradient skill: the unskilled arm also produces something that works, so a +binary pass-rate under-describes it. The honest column for this row is the cost of +getting there — tokens and wall clock — with the process discriminators as +supporting detail, and the table needs a shape that can carry that. Drafting the +reframe is deliberately deferred until the numbers are in, so the frame follows the +result rather than the reverse. + +## DEVIATION RECORD — 2026-07-28, added after the run + +**The run did not use the pre-registered agent model.** `--agent-model` was omitted +from `eval-magic run`, and the dispatch driver carried no `--model` flag, so all 80 +dispatches fell through to the session default and executed on **`claude-sonnet-5`**, +not `claude-haiku-4-5-20251001`. Everything below the "Run parameters" heading was +frozen before dispatch and is unmodified; this section is appended, not edited in. + +Consequences, stated plainly: + +1. **This run does not fulfil this pre-registration.** The predictions P1–P7 were + calibrated on Haiku 4.5 observations. Sonnet 5 is a different population, so a hit + or miss here is not the confirmatory test that was planned. +2. **The pre-registered decision rule returns DO NOT PROMOTE**, on two independent + grounds: neither co-primary reached p < 0.05 (P2 ceiled outright at 10/10 vs + 10/10; P3 gave 10/10 vs 8/10, p = 0.47), and the validity gate failed + (skill-invocation 39/40, not 1.0). +3. **P1 hit its pre-specified numbers on the wrong population.** Predicted + `with_skill ≥ 0.8`, `without_skill ≤ 0.5`, `Δ ≥ +0.3`; observed 1.00 / 0.50 / + +0.50, Fisher p = 0.0325. This is a real confirmation of a pre-specified + hypothesis, weakened — not voided — by the population mismatch. It needs a + replication on a declared population before it is advertised. +4. **P5 is refuted, and in the opposite direction.** Predicted −41% tokens / −32% + wall clock. Observed on Sonnet 5: **+24.8% tokens** (2.98 SE) and **+18.4% wall + clock** (2.11 SE). The Haiku efficiency *gain* is therefore not a property of the + skill — it is a property of unskilled Haiku flailing, and it reverses on the + current tier. Any README claim of a token saving would have been a tier-specific + artifact advertised as a general result. **Do not revive it.** +5. **P6 was wrong about direction of certainty.** The aggregate pass rate was + predicted inconclusive at ~1.4 SE; it came back **+10.8pp at 2.96 SE** + (CI [+3.6pp, +18.0pp]), because the `with_skill` arm ceiled at exactly 1.000 with + zero variance. **It is still not promoted to the headline** — the pre-registration + demoted it unconditionally and "it happened to clear" is precisely the + post-hoc promotion that rule exists to forbid. + +**Validity findings that survive scrutiny** (checked, not assumed): + +- `live_source_reads: 0` — no dispatch read the real repository. +- 2 stray-write violations, one in each arm, both blocked, both benign `/tmp` scratch + writes. No pollution of the workspace. +- 107 guard denials, 105 of them the known redirect false positive (eval-magic#179). + Distribution is symmetric across arms — 54 `with_skill` vs 53 `without_skill` — so + they cannot bias a delta. +- The one non-invoking `with_skill` run (`flaky/with_skill/run-9`) does not shift the + `with_skill` mean, which is 1.000 across all 40 runs with or without it. +- **The P1 result is not a guard artifact.** Two of the five `without_skill` failures + had a `/tmp` write denied. Both recovered, wrote their verification *inside* the + sandbox, ran it — and then deleted it themselves before finishing. Transcript-level + failure modes: 3 runs never wrote verification at all; 2 wrote it, ran it, and + removed it. + +**What is owed before anything is promoted:** a run on a declared model with P1 as +the stated primary endpoint. Do not re-tune the pagination fixture to chase this — +it is frozen, and it already did its job. + +## Run parameters (frozen) + +- **Agent model:** `claude-haiku-4-5-20251001`. **Judge model:** `claude-sonnet-5`. +- **Mode A**, `--skill-dir ./skills` (siblings staged in both arms — the control arm + keeps the other skills, so the comparison is "vs the same setup without this + skill"). +- **All four cases**, `--runs 10` → 80 agent dispatches, 120 judge dispatches. +- **Every** command passes + `--harness-file skills/investigating-bugs/evals/harness/claude-code-utc.toml` + (TZ=UTC pin, plugin isolation, working permission mode). diff --git a/skills/investigating-bugs/evals/baseline/benchmark.json b/skills/investigating-bugs/evals/baseline/benchmark.json new file mode 100644 index 0000000..2d9f1fc --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/benchmark.json @@ -0,0 +1,420 @@ +{ + "generated": "2026-07-28T06:54:18.746Z", + "mode": "new-skill", + "conditions_compared": ["with_skill", "without_skill"], + "missing_gradings": 0, + "validity_warnings": [ + "pagination-drops-last-page-cold/with_skill/run-2 encountered 1 guard denial — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "pagination-drops-last-page-cold/with_skill/run-3 encountered 2 guard denials — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "pagination-drops-last-page-cold/with_skill/run-4 encountered 1 guard denial — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "pagination-drops-last-page-cold/with_skill/run-9 encountered 3 guard denials — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "pagination-drops-last-page-cold/with_skill/run-11 encountered 1 guard denial — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "pagination-drops-last-page-cold/with_skill/run-13 encountered 1 guard denial — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "pagination-drops-last-page-cold/with_skill/run-14 encountered 2 guard denials — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "pagination-drops-last-page-cold/with_skill/run-16 encountered 1 guard denial — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "pagination-drops-last-page-cold/with_skill/run-17 encountered 2 guard denials — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "pagination-drops-last-page-cold/with_skill/run-19 encountered 3 guard denials — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "pagination-drops-last-page-cold/with_skill/run-20 encountered 1 guard denial — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "pagination-drops-last-page-cold/without_skill/run-2 encountered 1 guard denial — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "pagination-drops-last-page-cold/without_skill/run-4 encountered 1 guard denial — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "pagination-drops-last-page-cold/without_skill/run-5 encountered 1 guard denial — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "pagination-drops-last-page-cold/without_skill/run-6 encountered 1 guard denial — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "pagination-drops-last-page-cold/without_skill/run-9 encountered 1 guard denial — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "pagination-drops-last-page-cold/without_skill/run-10 encountered 1 guard denial — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "pagination-drops-last-page-cold/without_skill/run-11 encountered 1 guard denial — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "pagination-drops-last-page-cold/without_skill/run-12 encountered 1 guard denial — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "pagination-drops-last-page-cold/without_skill/run-13 encountered 1 guard denial — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "pagination-drops-last-page-cold/without_skill/run-14 encountered 1 guard denial — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "pagination-drops-last-page-cold/without_skill/run-16 encountered 1 guard denial — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "pagination-drops-last-page-cold/without_skill/run-17 encountered 1 guard denial — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "pagination-drops-last-page-cold/without_skill/run-18 encountered 1 guard denial — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "pagination-drops-last-page-cold/without_skill/run-19 encountered 1 guard denial — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "pagination-drops-last-page-cold/without_skill/run-20 encountered 1 guard denial — agent behavior changed; review guard-denials.json before trusting this data point, even when the blocked boundary was intentional.", + "staged skill 'test-driven-development' is also provided by enabled plugin 'slow-powers@slowdini' — each claude -p dispatch could discover both copies, so with/without results may be contaminated. Isolate each dispatch's Claude config (see docs/claude-notes.md → \"Isolating from installed plugins\").", + "staged skill 'verifying-development-work' is also provided by enabled plugin 'slow-powers@slowdini' — each claude -p dispatch could discover both copies, so with/without results may be contaminated. Isolate each dispatch's Claude config (see docs/claude-notes.md → \"Isolating from installed plugins\").", + "staged skill 'investigating-bugs' is also provided by enabled plugin 'slow-powers@slowdini' — each claude -p dispatch could discover both copies, so with/without results may be contaminated. Isolate each dispatch's Claude config (see docs/claude-notes.md → \"Isolating from installed plugins\").", + "staged skill 'auditing-slow-powers-usage' is also provided by enabled plugin 'slow-powers@slowdini' — each claude -p dispatch could discover both copies, so with/without results may be contaminated. Isolate each dispatch's Claude config (see docs/claude-notes.md → \"Isolating from installed plugins\").", + "staged skill 'hardening-plans' is also provided by enabled plugin 'slow-powers@slowdini' — each claude -p dispatch could discover both copies, so with/without results may be contaminated. Isolate each dispatch's Claude config (see docs/claude-notes.md → \"Isolating from installed plugins\").", + "staged skill 'writing-skills' is also provided by enabled plugin 'slow-powers@slowdini' — each claude -p dispatch could discover both copies, so with/without results may be contaminated. Isolate each dispatch's Claude config (see docs/claude-notes.md → \"Isolating from installed plugins\").", + "staged skill 'working-in-isolation' is also provided by enabled plugin 'slow-powers@slowdini' — each claude -p dispatch could discover both copies, so with/without results may be contaminated. Isolate each dispatch's Claude config (see docs/claude-notes.md → \"Isolating from installed plugins\").", + "staged skill 'evaluating-skills' is also provided by enabled plugin 'slow-powers@slowdini' — each claude -p dispatch could discover both copies, so with/without results may be contaminated. Isolate each dispatch's Claude config (see docs/claude-notes.md → \"Isolating from installed plugins\").", + "staged skill 'test-driven-development' is also provided by enabled plugin 'slow-powers@slowdini' — each claude -p dispatch could discover both copies, so with/without results may be contaminated. Isolate each dispatch's Claude config (see docs/claude-notes.md → \"Isolating from installed plugins\").", + "staged skill 'verifying-development-work' is also provided by enabled plugin 'slow-powers@slowdini' — each claude -p dispatch could discover both copies, so with/without results may be contaminated. Isolate each dispatch's Claude config (see docs/claude-notes.md → \"Isolating from installed plugins\").", + "staged skill 'investigating-bugs' is also provided by enabled plugin 'slow-powers@slowdini' — each claude -p dispatch could discover both copies, so with/without results may be contaminated. Isolate each dispatch's Claude config (see docs/claude-notes.md → \"Isolating from installed plugins\").", + "staged skill 'auditing-slow-powers-usage' is also provided by enabled plugin 'slow-powers@slowdini' — each claude -p dispatch could discover both copies, so with/without results may be contaminated. Isolate each dispatch's Claude config (see docs/claude-notes.md → \"Isolating from installed plugins\").", + "staged skill 'hardening-plans' is also provided by enabled plugin 'slow-powers@slowdini' — each claude -p dispatch could discover both copies, so with/without results may be contaminated. Isolate each dispatch's Claude config (see docs/claude-notes.md → \"Isolating from installed plugins\").", + "staged skill 'writing-skills' is also provided by enabled plugin 'slow-powers@slowdini' — each claude -p dispatch could discover both copies, so with/without results may be contaminated. Isolate each dispatch's Claude config (see docs/claude-notes.md → \"Isolating from installed plugins\").", + "staged skill 'working-in-isolation' is also provided by enabled plugin 'slow-powers@slowdini' — each claude -p dispatch could discover both copies, so with/without results may be contaminated. Isolate each dispatch's Claude config (see docs/claude-notes.md → \"Isolating from installed plugins\").", + "staged skill 'evaluating-skills' is also provided by enabled plugin 'slow-powers@slowdini' — each claude -p dispatch could discover both copies, so with/without results may be contaminated. Isolate each dispatch's Claude config (see docs/claude-notes.md → \"Isolating from installed plugins\")." + ], + "run_summary": { + "with_skill": { + "pass_rate": { + "mean": 1.0, + "stddev": 0.0, + "n": 20 + }, + "duration_ms": { + "mean": 78859.0, + "stddev": 15370.0, + "n": 20 + }, + "total_tokens": { + "mean": 608242.0, + "stddev": 122076.0, + "n": 20 + }, + "skill_invocation_n": 20, + "skill_invocation_rate": 1.0 + }, + "without_skill": { + "pass_rate": { + "mean": 0.725, + "stddev": 0.249, + "n": 20 + }, + "duration_ms": { + "mean": 55314.0, + "stddev": 18216.0, + "n": 20 + }, + "total_tokens": { + "mean": 385275.0, + "stddev": 148592.0, + "n": 20 + } + } + }, + "diff_scope": { + "with_skill": [ + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 1, + "files_touched": 2, + "lines_added": 24, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 2, + "files_touched": 2, + "lines_added": 19, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 3, + "files_touched": 2, + "lines_added": 26, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 4, + "files_touched": 2, + "lines_added": 26, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 5, + "files_touched": 2, + "lines_added": 24, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 6, + "files_touched": 2, + "lines_added": 25, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 7, + "files_touched": 2, + "lines_added": 21, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 8, + "files_touched": 2, + "lines_added": 20, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 9, + "files_touched": 2, + "lines_added": 24, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 10, + "files_touched": 2, + "lines_added": 28, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 11, + "files_touched": 2, + "lines_added": 25, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 12, + "files_touched": 2, + "lines_added": 21, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 13, + "files_touched": 2, + "lines_added": 20, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 14, + "files_touched": 2, + "lines_added": 24, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 15, + "files_touched": 2, + "lines_added": 28, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 16, + "files_touched": 2, + "lines_added": 28, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 17, + "files_touched": 2, + "lines_added": 27, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 18, + "files_touched": 2, + "lines_added": 27, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 19, + "files_touched": 2, + "lines_added": 25, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 20, + "files_touched": 2, + "lines_added": 26, + "lines_removed": 1, + "hunks": 2 + } + ], + "without_skill": [ + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 1, + "files_touched": 2, + "lines_added": 28, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 2, + "files_touched": 1, + "lines_added": 1, + "lines_removed": 1, + "hunks": 1 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 3, + "files_touched": 2, + "lines_added": 25, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 4, + "files_touched": 1, + "lines_added": 1, + "lines_removed": 1, + "hunks": 1 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 5, + "files_touched": 2, + "lines_added": 28, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 6, + "files_touched": 2, + "lines_added": 19, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 7, + "files_touched": 1, + "lines_added": 1, + "lines_removed": 1, + "hunks": 1 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 8, + "files_touched": 2, + "lines_added": 28, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 9, + "files_touched": 1, + "lines_added": 1, + "lines_removed": 1, + "hunks": 1 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 10, + "files_touched": 1, + "lines_added": 1, + "lines_removed": 1, + "hunks": 1 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 11, + "files_touched": 1, + "lines_added": 1, + "lines_removed": 1, + "hunks": 1 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 12, + "files_touched": 1, + "lines_added": 1, + "lines_removed": 1, + "hunks": 1 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 13, + "files_touched": 1, + "lines_added": 1, + "lines_removed": 1, + "hunks": 1 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 14, + "files_touched": 1, + "lines_added": 1, + "lines_removed": 1, + "hunks": 1 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 15, + "files_touched": 2, + "lines_added": 28, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 16, + "files_touched": 2, + "lines_added": 15, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 17, + "files_touched": 1, + "lines_added": 1, + "lines_removed": 1, + "hunks": 1 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 18, + "files_touched": 1, + "lines_added": 1, + "lines_removed": 1, + "hunks": 1 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 19, + "files_touched": 2, + "lines_added": 21, + "lines_removed": 1, + "hunks": 2 + }, + { + "eval_id": "pagination-drops-last-page-cold", + "run_index": 20, + "files_touched": 2, + "lines_added": 21, + "lines_removed": 1, + "hunks": 2 + } + ] + }, + "delta": { + "direction": "with_skill - without_skill", + "pass_rate": 0.275, + "duration_ms": 23545.0, + "total_tokens": 222967.0 + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r1.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r1.json new file mode 100644 index 0000000..9f483ba --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r1.json @@ -0,0 +1,39 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + }, + "meta_results": [ + { + "id": "__skill_invoked", + "passed": true, + "evidence": "Skill invocation verified from transcript.", + "confidence": 1.0, + "grader": "transcript_check" + } + ], + "meta_summary": { + "passed": 1, + "failed": 0, + "total": 1, + "skill_invoked": true + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r10.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r10.json new file mode 100644 index 0000000..9f483ba --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r10.json @@ -0,0 +1,39 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + }, + "meta_results": [ + { + "id": "__skill_invoked", + "passed": true, + "evidence": "Skill invocation verified from transcript.", + "confidence": 1.0, + "grader": "transcript_check" + } + ], + "meta_summary": { + "passed": 1, + "failed": 0, + "total": 1, + "skill_invoked": true + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r11.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r11.json new file mode 100644 index 0000000..9f483ba --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r11.json @@ -0,0 +1,39 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + }, + "meta_results": [ + { + "id": "__skill_invoked", + "passed": true, + "evidence": "Skill invocation verified from transcript.", + "confidence": 1.0, + "grader": "transcript_check" + } + ], + "meta_summary": { + "passed": 1, + "failed": 0, + "total": 1, + "skill_invoked": true + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r12.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r12.json new file mode 100644 index 0000000..9f483ba --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r12.json @@ -0,0 +1,39 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + }, + "meta_results": [ + { + "id": "__skill_invoked", + "passed": true, + "evidence": "Skill invocation verified from transcript.", + "confidence": 1.0, + "grader": "transcript_check" + } + ], + "meta_summary": { + "passed": 1, + "failed": 0, + "total": 1, + "skill_invoked": true + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r13.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r13.json new file mode 100644 index 0000000..9f483ba --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r13.json @@ -0,0 +1,39 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + }, + "meta_results": [ + { + "id": "__skill_invoked", + "passed": true, + "evidence": "Skill invocation verified from transcript.", + "confidence": 1.0, + "grader": "transcript_check" + } + ], + "meta_summary": { + "passed": 1, + "failed": 0, + "total": 1, + "skill_invoked": true + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r14.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r14.json new file mode 100644 index 0000000..9f483ba --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r14.json @@ -0,0 +1,39 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + }, + "meta_results": [ + { + "id": "__skill_invoked", + "passed": true, + "evidence": "Skill invocation verified from transcript.", + "confidence": 1.0, + "grader": "transcript_check" + } + ], + "meta_summary": { + "passed": 1, + "failed": 0, + "total": 1, + "skill_invoked": true + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r15.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r15.json new file mode 100644 index 0000000..9f483ba --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r15.json @@ -0,0 +1,39 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + }, + "meta_results": [ + { + "id": "__skill_invoked", + "passed": true, + "evidence": "Skill invocation verified from transcript.", + "confidence": 1.0, + "grader": "transcript_check" + } + ], + "meta_summary": { + "passed": 1, + "failed": 0, + "total": 1, + "skill_invoked": true + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r16.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r16.json new file mode 100644 index 0000000..9f483ba --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r16.json @@ -0,0 +1,39 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + }, + "meta_results": [ + { + "id": "__skill_invoked", + "passed": true, + "evidence": "Skill invocation verified from transcript.", + "confidence": 1.0, + "grader": "transcript_check" + } + ], + "meta_summary": { + "passed": 1, + "failed": 0, + "total": 1, + "skill_invoked": true + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r17.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r17.json new file mode 100644 index 0000000..9f483ba --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r17.json @@ -0,0 +1,39 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + }, + "meta_results": [ + { + "id": "__skill_invoked", + "passed": true, + "evidence": "Skill invocation verified from transcript.", + "confidence": 1.0, + "grader": "transcript_check" + } + ], + "meta_summary": { + "passed": 1, + "failed": 0, + "total": 1, + "skill_invoked": true + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r18.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r18.json new file mode 100644 index 0000000..9f483ba --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r18.json @@ -0,0 +1,39 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + }, + "meta_results": [ + { + "id": "__skill_invoked", + "passed": true, + "evidence": "Skill invocation verified from transcript.", + "confidence": 1.0, + "grader": "transcript_check" + } + ], + "meta_summary": { + "passed": 1, + "failed": 0, + "total": 1, + "skill_invoked": true + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r19.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r19.json new file mode 100644 index 0000000..9f483ba --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r19.json @@ -0,0 +1,39 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + }, + "meta_results": [ + { + "id": "__skill_invoked", + "passed": true, + "evidence": "Skill invocation verified from transcript.", + "confidence": 1.0, + "grader": "transcript_check" + } + ], + "meta_summary": { + "passed": 1, + "failed": 0, + "total": 1, + "skill_invoked": true + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r2.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r2.json new file mode 100644 index 0000000..9f483ba --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r2.json @@ -0,0 +1,39 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + }, + "meta_results": [ + { + "id": "__skill_invoked", + "passed": true, + "evidence": "Skill invocation verified from transcript.", + "confidence": 1.0, + "grader": "transcript_check" + } + ], + "meta_summary": { + "passed": 1, + "failed": 0, + "total": 1, + "skill_invoked": true + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r20.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r20.json new file mode 100644 index 0000000..9f483ba --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r20.json @@ -0,0 +1,39 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + }, + "meta_results": [ + { + "id": "__skill_invoked", + "passed": true, + "evidence": "Skill invocation verified from transcript.", + "confidence": 1.0, + "grader": "transcript_check" + } + ], + "meta_summary": { + "passed": 1, + "failed": 0, + "total": 1, + "skill_invoked": true + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r3.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r3.json new file mode 100644 index 0000000..9f483ba --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r3.json @@ -0,0 +1,39 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + }, + "meta_results": [ + { + "id": "__skill_invoked", + "passed": true, + "evidence": "Skill invocation verified from transcript.", + "confidence": 1.0, + "grader": "transcript_check" + } + ], + "meta_summary": { + "passed": 1, + "failed": 0, + "total": 1, + "skill_invoked": true + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r4.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r4.json new file mode 100644 index 0000000..9f483ba --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r4.json @@ -0,0 +1,39 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + }, + "meta_results": [ + { + "id": "__skill_invoked", + "passed": true, + "evidence": "Skill invocation verified from transcript.", + "confidence": 1.0, + "grader": "transcript_check" + } + ], + "meta_summary": { + "passed": 1, + "failed": 0, + "total": 1, + "skill_invoked": true + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r5.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r5.json new file mode 100644 index 0000000..9f483ba --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r5.json @@ -0,0 +1,39 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + }, + "meta_results": [ + { + "id": "__skill_invoked", + "passed": true, + "evidence": "Skill invocation verified from transcript.", + "confidence": 1.0, + "grader": "transcript_check" + } + ], + "meta_summary": { + "passed": 1, + "failed": 0, + "total": 1, + "skill_invoked": true + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r6.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r6.json new file mode 100644 index 0000000..9f483ba --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r6.json @@ -0,0 +1,39 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + }, + "meta_results": [ + { + "id": "__skill_invoked", + "passed": true, + "evidence": "Skill invocation verified from transcript.", + "confidence": 1.0, + "grader": "transcript_check" + } + ], + "meta_summary": { + "passed": 1, + "failed": 0, + "total": 1, + "skill_invoked": true + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r7.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r7.json new file mode 100644 index 0000000..9f483ba --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r7.json @@ -0,0 +1,39 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + }, + "meta_results": [ + { + "id": "__skill_invoked", + "passed": true, + "evidence": "Skill invocation verified from transcript.", + "confidence": 1.0, + "grader": "transcript_check" + } + ], + "meta_summary": { + "passed": 1, + "failed": 0, + "total": 1, + "skill_invoked": true + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r8.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r8.json new file mode 100644 index 0000000..9f483ba --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r8.json @@ -0,0 +1,39 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + }, + "meta_results": [ + { + "id": "__skill_invoked", + "passed": true, + "evidence": "Skill invocation verified from transcript.", + "confidence": 1.0, + "grader": "transcript_check" + } + ], + "meta_summary": { + "passed": 1, + "failed": 0, + "total": 1, + "skill_invoked": true + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r9.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r9.json new file mode 100644 index 0000000..9f483ba --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__with_skill__r9.json @@ -0,0 +1,39 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + }, + "meta_results": [ + { + "id": "__skill_invoked", + "passed": true, + "evidence": "Skill invocation verified from transcript.", + "confidence": 1.0, + "grader": "transcript_check" + } + ], + "meta_summary": { + "passed": 1, + "failed": 0, + "total": 1, + "skill_invoked": true + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r1.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r1.json new file mode 100644 index 0000000..255c812 --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r1.json @@ -0,0 +1,24 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r10.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r10.json new file mode 100644 index 0000000..26b35e0 --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r10.json @@ -0,0 +1,24 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": false, + "evidence": "expected exit code 0, got 1", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 1, + "failed": 1, + "total": 2, + "pass_rate": 0.5 + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r11.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r11.json new file mode 100644 index 0000000..26b35e0 --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r11.json @@ -0,0 +1,24 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": false, + "evidence": "expected exit code 0, got 1", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 1, + "failed": 1, + "total": 2, + "pass_rate": 0.5 + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r12.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r12.json new file mode 100644 index 0000000..26b35e0 --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r12.json @@ -0,0 +1,24 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": false, + "evidence": "expected exit code 0, got 1", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 1, + "failed": 1, + "total": 2, + "pass_rate": 0.5 + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r13.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r13.json new file mode 100644 index 0000000..26b35e0 --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r13.json @@ -0,0 +1,24 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": false, + "evidence": "expected exit code 0, got 1", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 1, + "failed": 1, + "total": 2, + "pass_rate": 0.5 + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r14.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r14.json new file mode 100644 index 0000000..26b35e0 --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r14.json @@ -0,0 +1,24 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": false, + "evidence": "expected exit code 0, got 1", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 1, + "failed": 1, + "total": 2, + "pass_rate": 0.5 + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r15.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r15.json new file mode 100644 index 0000000..255c812 --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r15.json @@ -0,0 +1,24 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r16.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r16.json new file mode 100644 index 0000000..255c812 --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r16.json @@ -0,0 +1,24 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r17.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r17.json new file mode 100644 index 0000000..26b35e0 --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r17.json @@ -0,0 +1,24 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": false, + "evidence": "expected exit code 0, got 1", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 1, + "failed": 1, + "total": 2, + "pass_rate": 0.5 + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r18.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r18.json new file mode 100644 index 0000000..26b35e0 --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r18.json @@ -0,0 +1,24 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": false, + "evidence": "expected exit code 0, got 1", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 1, + "failed": 1, + "total": 2, + "pass_rate": 0.5 + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r19.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r19.json new file mode 100644 index 0000000..255c812 --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r19.json @@ -0,0 +1,24 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r2.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r2.json new file mode 100644 index 0000000..26b35e0 --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r2.json @@ -0,0 +1,24 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": false, + "evidence": "expected exit code 0, got 1", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 1, + "failed": 1, + "total": 2, + "pass_rate": 0.5 + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r20.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r20.json new file mode 100644 index 0000000..255c812 --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r20.json @@ -0,0 +1,24 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r3.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r3.json new file mode 100644 index 0000000..255c812 --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r3.json @@ -0,0 +1,24 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r4.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r4.json new file mode 100644 index 0000000..26b35e0 --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r4.json @@ -0,0 +1,24 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": false, + "evidence": "expected exit code 0, got 1", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 1, + "failed": 1, + "total": 2, + "pass_rate": 0.5 + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r5.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r5.json new file mode 100644 index 0000000..255c812 --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r5.json @@ -0,0 +1,24 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r6.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r6.json new file mode 100644 index 0000000..255c812 --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r6.json @@ -0,0 +1,24 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r7.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r7.json new file mode 100644 index 0000000..26b35e0 --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r7.json @@ -0,0 +1,24 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": false, + "evidence": "expected exit code 0, got 1", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 1, + "failed": 1, + "total": 2, + "pass_rate": 0.5 + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r8.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r8.json new file mode 100644 index 0000000..255c812 --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r8.json @@ -0,0 +1,24 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 2, + "failed": 0, + "total": 2, + "pass_rate": 1.0 + } +} diff --git a/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r9.json b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r9.json new file mode 100644 index 0000000..26b35e0 --- /dev/null +++ b/skills/investigating-bugs/evals/baseline/grading/pagination-drops-last-page-cold__without_skill__r9.json @@ -0,0 +1,24 @@ +{ + "assertion_results": [ + { + "id": "fix_is_correct", + "passed": true, + "evidence": "exit code matched 0", + "confidence": 1.0, + "grader": "command_check" + }, + { + "id": "left_a_regression_test", + "passed": false, + "evidence": "expected exit code 0, got 1", + "confidence": 1.0, + "grader": "command_check" + } + ], + "summary": { + "passed": 1, + "failed": 1, + "total": 2, + "pass_rate": 0.5 + } +} diff --git a/skills/investigating-bugs/evals/evals.json b/skills/investigating-bugs/evals/evals.json index dbb9087..f44b7b4 100644 --- a/skills/investigating-bugs/evals/evals.json +++ b/skills/investigating-bugs/evals/evals.json @@ -2,48 +2,126 @@ "skill_name": "investigating-bugs", "evals": [ { - "id": "symptom-vs-source-cold", - "prompt": "Our checkout is rejecting orders. `amountDueCents` in checkout.ts sometimes returns a negative number, and our payments API rejects any negative amount. It happens when a promo code is applied. Fix it.", - "expected_output": "The reported symptom is a negative amount due. The locally-satisfying wrong fix is to clamp the result downstream (`Math.max(0, ...)`): the negative goes away and the payments API stops rejecting, so the immediate complaint is resolved — but the customer is now charged $0 instead of $40, and `savingsLine` still reports a wildly wrong figure, because the real defect is upstream: `lookupPercentOff` returns a whole-number percent (20) that `amountDueCents` and `savingsLine` both apply WITHOUT dividing by 100, so the discount is 100x too large. The correct fix is at the source — scale the percent correctly (e.g. `* percentOff / 100`, or have the lookup return a fraction) — which fixes BOTH consumers at once. A pass traces the wrong value back to its source and fixes it there; the failure mode is clamping/guarding the one crashing call site and declaring the bug fixed while the discount math (and the receipt) stays wrong.", + "id": "timezone-date-only-shift-cold", + "prompt": "Some users say the due date on their invoices shows up a day early — one customer in California reported it. Looks like `formatDueDate` in display.ts is off by one. Can you fix it? Works fine for me when I check it.", + "expected_output": "The stored value (`\"2024-03-10\"`) is a calendar date — no time, no zone. The defect is in the shared `toDate` helper (parseDate.ts): `new Date(\"2024-03-10\")` parses date-only ISO strings as UTC midnight, so any LOCAL component read shifts the day for users behind UTC. `display.ts`, `save.ts`, and `overdue.ts` all go through `toDate`, so the same wrong value reaches all three. The reproduction is environment-dependent: `repro.ts` PASSES under the default `TZ=UTC` (or any non-negative offset) and only fails under a negative offset like `TZ=America/Los_Angeles` — which is exactly why 'works fine for me' is misleading. The tempting wrong fixes all pass the reporter's case but break others: a `+1 day`/offset nudge in `formatDueDate` over-corrects positive-offset users (now a day LATE) and leaves `save`/`overdue` wrong; the common 'force local parse' (`new Date(ymd + \"T00:00:00\")`) fixes display everywhere but breaks the `save` round-trip for POSITIVE offsets because `toISOString()` rolls back a day (verified: under Europe/Berlin, `2024-12-31` round-trips to `2024-12-30`). The only correct fix treats the value as a timezone-agnostic calendar date at the source — keep it as `YYYY-MM-DD` and read components from the string, or use `Date.UTC`/`getUTC*` consistently — so display, save round-trip, and isDueToday are all correct in every timezone. A pass varies TZ to reproduce, traces the shift to the shared parse, and fixes it at the source; the failure mode is trusting the green default-TZ run ('can't reproduce / works for me') or applying a per-layer Date nudge that silences the reported case while corrupting an un-reported one. The held-out `holdout/tz-matrix.holdout.ts`, run by the `all_consumers_correct_in_every_tz` command_check across the TZ matrix, is the objective decoy-catcher; the judges grade the same property from the response.", "files": [ - "fixtures/checkout-discount/checkout.ts", - "fixtures/checkout-discount/discount.ts", - "fixtures/checkout-discount/repro.ts" + "fixtures/tz-date-only/parseDate.ts", + "fixtures/tz-date-only/display.ts", + "fixtures/tz-date-only/save.ts", + "fixtures/tz-date-only/overdue.ts", + "fixtures/tz-date-only/repro.ts", + "fixtures/tz-date-only/package.json" ], "assertions": [ { - "id": "inspected_source", + "id": "reproduced_by_varying_tz", "type": "transcript_check", "check": "tool_invocation_matches", - "pattern": "(Read|Bash).*(discount|checkout)" + "pattern": "(Bash|Read).*(TZ=|America/|Europe/|Pacific/|getTimezoneOffset|timeZone)" }, { - "id": "fix_at_source", + "id": "did_not_trust_green_repro", "type": "llm_judge", - "rubric": "Did the agent fix the discount at its source — correcting the percent/fraction scale so the discount is no longer 100x too large (e.g. dividing the whole-number percent by 100, or returning a fraction from the lookup), such that BOTH `amountDueCents` (charges $40 on a $50 SAVE20 cart) and `savingsLine` (reports $10.00) become correct? PASS only if the root-cause scale error is corrected at the source. FAIL if the response only clamps or guards the negative amount downstream (e.g. `Math.max(0, ...)`), or otherwise silences the symptom, while leaving the underlying discount calculation wrong." + "rubric": "Did the agent recognize that the failure is environment-dependent — the bug reproduces only under a timezone with a negative UTC offset, so a green run in the default/UTC environment (and the reporter's 'works fine for me') is not evidence the bug is absent — and actually reproduce it by varying the timezone (e.g. setting TZ to a US/negative-offset zone) before fixing? PASS if it makes the failure reproduce on demand by varying TZ, or otherwise reasons that the default-TZ green run cannot clear a TZ-dependent bug. FAIL if it concludes 'can't reproduce / works for me', or patches blind without reproducing the reported symptom." + }, + { + "id": "fixed_as_calendar_date", + "type": "llm_judge", + "rubric": "Did the agent fix the bug at its source so that the stored value is handled as a CALENDAR DATE consistently across every consumer — display, the `serializeForSave` round-trip, and `isDueToday` all correct at negative AND positive UTC offsets? Judge the property, not the technique: keeping it as a `YYYY-MM-DD` string, using `Date.UTC`/`getUTC*` throughout, or parsing to local midnight AND reading it back with local getters everywhere (including replacing `toISOString()` in `serializeForSave`) are all PASSES, because each leaves every consumer correct in every zone. FAIL if any consumer is left inconsistent with the representation the agent chose — a local-midnight parse still serialized through `toISOString()`, a per-layer day/offset nudge (`setDate(+1)`, subtracting `getTimezoneOffset`), or a change to `formatDueDate` alone that leaves the shared parse and the other consumers wrong." + }, + { + "id": "all_consumers_correct_in_every_tz", + "type": "command_check", + "setup_files": ["fixtures/tz-date-only/holdout/tz-matrix.holdout.ts"], + "command": "cd fixtures/tz-date-only && bun test ./holdout/tz-matrix.holdout.ts", + "matrix": { + "TZ": [ + "UTC", + "America/Los_Angeles", + "Pacific/Kiritimati", + "Europe/Berlin" + ] + }, + "expect_exit_code": 0 } ] }, { - "id": "symptom-vs-source-seeded", - "prompt": "[The following is the conversation so far in this session. You are the assistant; continue from the final user turn.]\n\nUser: Checkout is rejecting orders — `amountDueCents` in checkout.ts returns a negative number when a promo code is applied, and our payments API 400s on negative amounts. Fix it fast, we're losing checkouts.\n\nAssistant: Done — I clamped the amount so it can't go negative: `return Math.max(0, subtotalCents - discountCents)`. Negative amounts can no longer reach the payments API.\n\nUser: Orders are going through again, but finance says SAVE20 customers are being charged the full amount — $0 off — and the receipt's \"You saved $…\" line shows a huge wrong number. Just hardcode the savings line to match what we actually charged so the receipt isn't embarrassing — we ship in 20 minutes.", - "expected_output": "A speculative symptom patch (clamping the negative to 0 with `Math.max`) was already applied and produced a new failure: the discount is silently $0 and the receipt's savings line is wrong. Hardcoding the savings line is a SECOND downstream patch on the same root cause and repeats the mistake. The real defect is upstream: `lookupPercentOff` returns a whole-number percent (20) applied without `/ 100`, so the discount is 100x too large — the clamp masked it, and BOTH `amountDueCents` and `savingsLine` read the same wrong value. The correct move is to stop patching downstream, trace to the source, and fix the scale there (so SAVE20 charges $40 and the receipt reads $10.00). The failure mode is complying with 'just hardcode the savings line' — stacking a second guess on the first. This is a seeded probe under deadline + authority pressure and prior commitment to a guess; a pass is a stronger-than-cold signal, not ground truth (see the 'Seeding conversation context (and its ceiling)' note in evaluating-skills). skill_should_trigger stays true.", + "id": "timezone-nudge-ship-pressure-seeded", + "prompt": "[The following is the conversation so far in this session. You are the assistant; continue from the final user turn.]\n\nUser: Invoices show the due date a day early for a customer in California — `formatDueDate` in display.ts is off by one. Fix it fast, finance is waiting.\n\nAssistant: Done — I added a day in `formatDueDate` so it renders correctly: `d.setDate(d.getDate() + 1)`. The invoice now shows the right date.\n\nUser: Finance pulled up that California account and the invoice reads right now. Ship it — unless you see a reason not to.", + "expected_output": "The `+1 day` nudge is already in `formatDueDate` in the fixture, and it does satisfy the one account finance checked — but it is a symptom patch on a shared root cause and it is wrong for everyone that account doesn't speak for. The real defect is the shared `toDate` (parseDate.ts) parsing the date-only string `\"2024-03-10\"` as UTC midnight; `display`, `save`, and `overdue` all inherit it. The nudge OVER-CORRECTS every non-negative offset — under the dispatch's own `TZ=UTC` the invoice now renders `March 11`, a day LATE — and it does nothing for the `save` round-trip or `isDueToday`. So a single verified account is not evidence the fix is right: running anything at all (`bun repro.ts`) in the current environment shows the nudge failing. The disciplined move is to check before shipping, see that the nudge regresses other offsets and leaves the other consumers wrong, and replace it with a source fix that treats the value as a timezone-agnostic calendar date (string-based, or consistent UTC) so every consumer and every offset is correct. The failure mode is complying with 'ship it' on the strength of one green account and leaving the nudge in place. This is a seeded probe under deadline pressure and prior commitment to a guess; a pass is a stronger-than-cold signal, not ground truth (see the 'Seeding conversation context (and its ceiling)' note in evaluating-skills). skill_should_trigger stays true.", "files": [ - "fixtures/checkout-discount/checkout.ts", - "fixtures/checkout-discount/discount.ts", - "fixtures/checkout-discount/repro.ts" + "fixtures/tz-nudged/parseDate.ts", + "fixtures/tz-nudged/display.ts", + "fixtures/tz-nudged/save.ts", + "fixtures/tz-nudged/overdue.ts", + "fixtures/tz-nudged/repro.ts", + "fixtures/tz-nudged/package.json" ], - "runs": 3, "assertions": [ { - "id": "rejects_downstream_patch", + "id": "reproduced_by_varying_tz", + "type": "transcript_check", + "check": "tool_invocation_matches", + "pattern": "(Bash|Read).*(TZ=|America/|Europe/|Pacific/|getTimezoneOffset|timeZone)" + }, + { + "id": "refused_to_ship_the_nudge", "type": "llm_judge", - "rubric": "Did the agent decline to apply a second downstream patch (hardcoding the savings line, or otherwise making the receipt match the clamped charge) and instead trace the failure to its upstream source — the whole-number percent applied without `/ 100`, making the discount 100x too large — and fix it there so both the charge and the receipt are correct? PASS if it rejects the requested downstream patch in favour of the source fix. FAIL if it hardcodes/adjusts the savings line, or stacks any further guard, without correcting the upstream scale error." + "rubric": "Did the agent decline to ship the already-applied `+1 day` nudge on the strength of the one California account finance checked, and instead verify it itself — running the code or varying the timezone — to show the nudge is wrong for other offsets and leaves the other consumers untouched, before changing the verdict? PASS if it refuses to ship the nudge as-is and demonstrates (or clearly reasons) that one green account is not sufficient evidence. FAIL if it ships the nudge, or accepts the sign-off without checking any other timezone or consumer." }, { - "id": "names_propagation", + "id": "replaced_nudge_with_source_fix", "type": "llm_judge", - "rubric": "Did the agent recognize that the SAME wrong value (the mis-scaled discount) feeds multiple consumers — both `amountDueCents` and `savingsLine` — so guarding or hardcoding any single call site cannot be a real fix? PASS if it identifies that the defect propagates beyond the originally-reported call site. FAIL if it treats each symptom as an isolated, independently-patchable issue." + "rubric": "Did the agent remove the downstream `+1 day` nudge and fix the shared parse instead, so that display, the `serializeForSave` round-trip, and `isDueToday` are all correct at negative AND positive UTC offsets? Judge the property, not the technique: a `YYYY-MM-DD` string representation, consistent `Date.UTC`/`getUTC*`, or a local-midnight parse read back with local getters everywhere (including replacing `toISOString()` in `serializeForSave`) are all PASSES. FAIL if the nudge survives, if a second nudge is stacked on another layer, or if any consumer is left inconsistent with the representation the agent chose." + }, + { + "id": "all_consumers_correct_in_every_tz", + "type": "command_check", + "setup_files": ["fixtures/tz-nudged/holdout/tz-matrix.holdout.ts"], + "command": "cd fixtures/tz-nudged && bun test ./holdout/tz-matrix.holdout.ts", + "matrix": { + "TZ": [ + "UTC", + "America/Los_Angeles", + "Pacific/Kiritimati", + "Europe/Berlin" + ] + }, + "expect_exit_code": 0 + } + ] + }, + { + "id": "pagination-drops-last-page-cold", + "prompt": "Support says the last few results go missing from the final page — a customer with 7 saved items only ever sees 6 of them. It's `chunk` in chunk.ts. Can you sort it out? We're mid-release, so quick as you can.", + "expected_output": "`chunk` advances while `i + size <= items.length`, so any trailing partial page is silently dropped: 7 items at size 3 yields two pages and loses the 7th item. The fix is one line (`i < items.length`). The bug is deliberately trivial to fix — fixing it is the control, not the measurement. What this case measures is whether the agent leaves behind something that would catch the bug if it came back: an automated test or a repro script that actually fails on the defect. The fixture establishes that this project tests its code — `formatBytes.ts` ships with `formatBytes.fixture.ts` — but nothing covers `chunk`, so there is no existing test file to append to and no cue to follow. Creating verification where none exists is the decision under measurement. The failure mode is a correct one-line patch, a confident summary, and nothing durable — the reported symptom is gone and the next regression is undetected. The held-out `verify-regression-test.sh` grades this objectively: it runs everything the agent left behind against the agent's own fix (which must be green), then restores the original buggy `chunk.ts` and re-runs; a genuine regression artifact must now fail. A vacuous test, or no test at all, cannot pass that — and the pre-existing `formatBytes` tests stay green in both phases, so they can never satisfy it. The 'quick as you can' framing is mild delivery pressure — the discipline is supposed to hold under it.", + "files": [ + "fixtures/chunk-pagination/chunk.ts", + "fixtures/chunk-pagination/formatBytes.ts", + "fixtures/chunk-pagination/formatBytes.fixture.ts", + "fixtures/chunk-pagination/package.json" + ], + "assertions": [ + { + "id": "fix_is_correct", + "type": "command_check", + "setup_files": [ + "fixtures/chunk-pagination/holdout/correctness.holdout.ts" + ], + "command": "cd fixtures/chunk-pagination && bun test ./holdout/correctness.holdout.ts", + "expect_exit_code": 0 + }, + { + "id": "left_a_regression_test", + "type": "command_check", + "setup_files": [ + "fixtures/chunk-pagination/holdout/verify-regression-test.sh", + "fixtures/chunk-pagination/holdout/pristine/chunk.ts" + ], + "command": "sh fixtures/chunk-pagination/holdout/verify-regression-test.sh", + "expect_exit_code": 0 } ] }, @@ -56,7 +134,6 @@ "fixtures/flag-cache-pollution/flags.fixture.ts", "fixtures/flag-cache-pollution/package.json" ], - "runs": 3, "assertions": [ { "id": "inspected_before_patching", diff --git a/skills/investigating-bugs/evals/fixtures/checkout-discount/checkout.ts b/skills/investigating-bugs/evals/fixtures/checkout-discount/checkout.ts deleted file mode 100644 index 9169fbb..0000000 --- a/skills/investigating-bugs/evals/fixtures/checkout-discount/checkout.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { lookupPercentOff } from "./discount"; - -// Returns the amount the customer still owes, in cents, after any promo code. -export function amountDueCents(subtotalCents: number, code?: string): number { - const percentOff = lookupPercentOff(code); - const discountCents = subtotalCents * percentOff; - return subtotalCents - discountCents; -} - -// Receipt line shown to the customer summarising what the promo saved them. -export function savingsLine(subtotalCents: number, code?: string): string { - const percentOff = lookupPercentOff(code); - const savedCents = subtotalCents * percentOff; - return `You saved $${(savedCents / 100).toFixed(2)}`; -} diff --git a/skills/investigating-bugs/evals/fixtures/checkout-discount/discount.ts b/skills/investigating-bugs/evals/fixtures/checkout-discount/discount.ts deleted file mode 100644 index 5791eaa..0000000 --- a/skills/investigating-bugs/evals/fixtures/checkout-discount/discount.ts +++ /dev/null @@ -1,11 +0,0 @@ -export type PromoCode = "SAVE20" | "HALFOFF"; - -const PERCENT_OFF: Record = { - SAVE20: 20, - HALFOFF: 50, -}; - -export function lookupPercentOff(code: string | undefined): number { - if (!code) return 0; - return PERCENT_OFF[code] ?? 0; -} diff --git a/skills/investigating-bugs/evals/fixtures/checkout-discount/repro.ts b/skills/investigating-bugs/evals/fixtures/checkout-discount/repro.ts deleted file mode 100644 index 084abf8..0000000 --- a/skills/investigating-bugs/evals/fixtures/checkout-discount/repro.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { amountDueCents, savingsLine } from "./checkout"; - -// A $50.00 cart checking out with the SAVE20 promo code. -const subtotalCents = 5000; - -console.log("amountDueCents:", amountDueCents(subtotalCents, "SAVE20")); -console.log("savingsLine:", savingsLine(subtotalCents, "SAVE20")); diff --git a/skills/investigating-bugs/evals/fixtures/chunk-pagination/chunk.ts b/skills/investigating-bugs/evals/fixtures/chunk-pagination/chunk.ts new file mode 100644 index 0000000..c66a236 --- /dev/null +++ b/skills/investigating-bugs/evals/fixtures/chunk-pagination/chunk.ts @@ -0,0 +1,8 @@ +// Splits a list of results into fixed-size pages for the paginated results view. +export function chunk(items: T[], size: number): T[][] { + const pages: T[][] = []; + for (let i = 0; i + size <= items.length; i += size) { + pages.push(items.slice(i, i + size)); + } + return pages; +} diff --git a/skills/investigating-bugs/evals/fixtures/chunk-pagination/formatBytes.fixture.ts b/skills/investigating-bugs/evals/fixtures/chunk-pagination/formatBytes.fixture.ts new file mode 100644 index 0000000..54d4007 --- /dev/null +++ b/skills/investigating-bugs/evals/fixtures/chunk-pagination/formatBytes.fixture.ts @@ -0,0 +1,18 @@ +// Named off the `*.test.ts` pattern so the repo's own `bun test` does not collect +// this fixture. Run it explicitly with `bun test ./formatBytes.fixture.ts`. +import { expect, test } from "bun:test"; +import { formatBytes } from "./formatBytes"; + +test("renders plain bytes below a kilobyte", () => { + expect(formatBytes(0)).toBe("0 B"); + expect(formatBytes(999)).toBe("999 B"); +}); + +test("scales up through the units", () => { + expect(formatBytes(2048)).toBe("2.0 KB"); + expect(formatBytes(5 * 1024 * 1024)).toBe("5.0 MB"); +}); + +test("stops scaling at gigabytes", () => { + expect(formatBytes(3 * 1024 * 1024 * 1024)).toBe("3.0 GB"); +}); diff --git a/skills/investigating-bugs/evals/fixtures/chunk-pagination/formatBytes.ts b/skills/investigating-bugs/evals/fixtures/chunk-pagination/formatBytes.ts new file mode 100644 index 0000000..45d4441 --- /dev/null +++ b/skills/investigating-bugs/evals/fixtures/chunk-pagination/formatBytes.ts @@ -0,0 +1,12 @@ +// Renders a byte count for the file-size column of the results table. +export function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + const units = ["KB", "MB", "GB"]; + let value = bytes / 1024; + let unit = 0; + while (value >= 1024 && unit < units.length - 1) { + value /= 1024; + unit++; + } + return `${value.toFixed(1)} ${units[unit]}`; +} diff --git a/skills/investigating-bugs/evals/fixtures/chunk-pagination/holdout/correctness.holdout.ts b/skills/investigating-bugs/evals/fixtures/chunk-pagination/holdout/correctness.holdout.ts new file mode 100644 index 0000000..ce86248 --- /dev/null +++ b/skills/investigating-bugs/evals/fixtures/chunk-pagination/holdout/correctness.holdout.ts @@ -0,0 +1,24 @@ +// HELD-OUT correctness test — NOT shown to the agent under eval. Injected by the +// `fix_is_correct` command_check after the agent finishes, so a run that never +// fixed the bug is distinguishable from one that fixed it but left no test behind. +import { expect, test } from "bun:test"; +import { chunk } from "../chunk"; + +test("keeps the trailing partial page", () => { + expect(chunk([1, 2, 3, 4, 5, 6, 7], 3)).toEqual([[1, 2, 3], [4, 5, 6], [7]]); +}); + +test("still splits exact multiples", () => { + expect(chunk([1, 2, 3, 4], 2)).toEqual([ + [1, 2], + [3, 4], + ]); +}); + +test("keeps a list shorter than one page", () => { + expect(chunk(["a"], 3)).toEqual([["a"]]); +}); + +test("returns no pages for an empty list", () => { + expect(chunk([], 3)).toEqual([]); +}); diff --git a/skills/investigating-bugs/evals/fixtures/chunk-pagination/holdout/pristine/chunk.ts b/skills/investigating-bugs/evals/fixtures/chunk-pagination/holdout/pristine/chunk.ts new file mode 100644 index 0000000..c66a236 --- /dev/null +++ b/skills/investigating-bugs/evals/fixtures/chunk-pagination/holdout/pristine/chunk.ts @@ -0,0 +1,8 @@ +// Splits a list of results into fixed-size pages for the paginated results view. +export function chunk(items: T[], size: number): T[][] { + const pages: T[][] = []; + for (let i = 0; i + size <= items.length; i += size) { + pages.push(items.slice(i, i + size)); + } + return pages; +} diff --git a/skills/investigating-bugs/evals/fixtures/chunk-pagination/holdout/verify-regression-test.sh b/skills/investigating-bugs/evals/fixtures/chunk-pagination/holdout/verify-regression-test.sh new file mode 100644 index 0000000..8b01e1b --- /dev/null +++ b/skills/investigating-bugs/evals/fixtures/chunk-pagination/holdout/verify-regression-test.sh @@ -0,0 +1,73 @@ +#!/bin/sh +# HELD-OUT mutation check — NOT shown to the agent under eval. +# +# Asks the only question that distinguishes a durable fix from a disposable one: +# if this bug came back tomorrow, would anything the agent left behind catch it? +# +# phase 1 — everything the agent left behind must be GREEN against its own fix +# (this rules out a broken or half-written test counting as a pass) +# phase 2 — restore the original buggy source and re-run. At least one artifact +# must now FAIL, or nothing present actually captures this bug. +# +# Artifacts are collected by shape, not by name, so both an automated test and a +# standalone repro script count — Phase 4 of the skill accepts either ("an +# automated test or simple script that consistently triggers the bug"). A script +# that merely prints the symptom without failing does not count, and should not. +set -u + +FIXTURE_DIR="$(cd "$(dirname "$0")/.." && pwd)" +cd "$FIXTURE_DIR" || exit 1 + +SOURCE_FILE="chunk.ts" +PRISTINE="holdout/pristine/chunk.ts" + +[ -f "$PRISTINE" ] || { + echo "FAIL: held-out pristine source missing — check cannot run." + exit 1 +} + +# Paths keep their leading "./": `bun test chunk.fixture.ts` treats a bare name as +# a name *filter* and silently matches nothing, while `bun test ./chunk.fixture.ts` +# treats it as a path. A filter that matches nothing exits non-zero, which would +# fail phase 1 for every run regardless of what the agent did. +candidates=$(find . -path ./holdout -prune -o -type f \( -name '*.ts' -o -name '*.js' \) -print | + grep -v "^\./${SOURCE_FILE}\$" | sort) + +if [ -z "$candidates" ]; then + echo "FAIL: the agent left nothing behind that could catch a regression." + exit 1 +fi + +echo "candidate artifacts:" +echo "$candidates" | sed 's/^/ /' + +run_all() { + rc=0 + for f in $candidates; do + case "$f" in + *.test.* | *_test.* | *.spec.* | *.fixture.ts) cmd="bun test $f" ;; + *) cmd="bun run $f" ;; + esac + if ! $cmd >/dev/null 2>&1; then + echo " non-zero exit: $f" + rc=1 + fi + done + return $rc +} + +echo "phase 1: artifacts must be green against the agent's own fix" +if ! run_all; then + echo "FAIL: something the agent left behind does not pass against its own fix." + exit 1 +fi + +echo "phase 2: restoring the original bug" +cp "$PRISTINE" "$SOURCE_FILE" || exit 1 +if run_all; then + echo "FAIL: everything still passes once the original bug is restored — nothing captures this bug." + exit 1 +fi + +echo "PASS: an artifact left behind fails when the bug is reintroduced." +exit 0 diff --git a/skills/investigating-bugs/evals/fixtures/chunk-pagination/package.json b/skills/investigating-bugs/evals/fixtures/chunk-pagination/package.json new file mode 100644 index 0000000..aed2a65 --- /dev/null +++ b/skills/investigating-bugs/evals/fixtures/chunk-pagination/package.json @@ -0,0 +1,4 @@ +{ + "name": "chunk-pagination-fixture", + "type": "module" +} diff --git a/skills/investigating-bugs/evals/fixtures/tz-date-only/display.ts b/skills/investigating-bugs/evals/fixtures/tz-date-only/display.ts new file mode 100644 index 0000000..2ce736d --- /dev/null +++ b/skills/investigating-bugs/evals/fixtures/tz-date-only/display.ts @@ -0,0 +1,22 @@ +import { toDate } from "./parseDate"; + +const MONTHS = [ + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", +]; + +// Renders the order's due date on the invoice. +export function formatDueDate(ymd: string): string { + const d = toDate(ymd); + return `${MONTHS[d.getMonth()]} ${d.getDate()}, ${d.getFullYear()}`; +} diff --git a/skills/investigating-bugs/evals/fixtures/tz-date-only/holdout/tz-matrix.holdout.ts b/skills/investigating-bugs/evals/fixtures/tz-date-only/holdout/tz-matrix.holdout.ts new file mode 100644 index 0000000..5324849 --- /dev/null +++ b/skills/investigating-bugs/evals/fixtures/tz-date-only/holdout/tz-matrix.holdout.ts @@ -0,0 +1,44 @@ +// HELD-OUT contract test — NOT shown to the agent under eval, and named +// `.holdout.ts` (not `.test.ts`) so the repo's own `bun test` never collects +// it. It is the objective decoy-catcher for the timezone cases. +// +// The `all_consumers_correct_in_every_tz` command_check in evals.json injects +// this file after the agent finishes and runs it against the agent's FINAL +// fixture state, once per cell of +// TZ ∈ { UTC, America/Los_Angeles, Pacific/Kiritimati, Europe/Berlin } +// Run one cell locally with: `TZ=America/Los_Angeles bun test ./holdout/tz-matrix.holdout.ts` +// +// Why a single TZ value per process: Node/Bun read `TZ` once at startup, so the +// matrix must be supplied by the runner, one process per cell. Each cell asserts +// the three consumers below; a fix is correct only if EVERY cell passes: +// - the original UTC-midnight parse fails `display`/`isDueToday` under LA; +// - a "+1 day"/offset nudge over-corrects positive offsets; +// - a "force local" parse (`new Date(ymd + "T00:00:00")`) breaks the `save` +// round-trip under positive offsets (Berlin/Kiritimati) via toISOString(); +// - only treating the value as a timezone-agnostic calendar date passes all. +import { expect, test } from "bun:test"; +import { formatDueDate } from "../display"; +import { isDueToday } from "../overdue"; +import { serializeForSave } from "../save"; + +const TZ = process.env.TZ ?? "(unset)"; + +test(`display renders the stored calendar day [${TZ}]`, () => { + expect(formatDueDate("2024-03-10")).toBe("March 10, 2024"); + expect(formatDueDate("2024-12-31")).toBe("December 31, 2024"); + expect(formatDueDate("2025-01-01")).toBe("January 1, 2025"); +}); + +test(`save round-trips the stored date losslessly [${TZ}]`, () => { + for (const ymd of ["2024-03-10", "2024-07-01", "2024-12-31", "2025-01-01"]) { + expect(serializeForSave(ymd)).toBe(ymd); + } +}); + +test(`isDueToday compares calendar days, not instants [${TZ}]`, () => { + // Local "now" on the morning of 2024-03-10, whatever this cell's TZ is. + const now = new Date(2024, 2, 10, 9, 0, 0); + expect(isDueToday("2024-03-10", now)).toBe(true); + expect(isDueToday("2024-03-09", now)).toBe(false); + expect(isDueToday("2024-03-11", now)).toBe(false); +}); diff --git a/skills/investigating-bugs/evals/fixtures/tz-date-only/overdue.ts b/skills/investigating-bugs/evals/fixtures/tz-date-only/overdue.ts new file mode 100644 index 0000000..79db3ba --- /dev/null +++ b/skills/investigating-bugs/evals/fixtures/tz-date-only/overdue.ts @@ -0,0 +1,12 @@ +import { toDate } from "./parseDate"; + +// Is the order's due date the current calendar day? Used to badge invoices as +// "due today". `now` is injectable for testing. +export function isDueToday(ymd: string, now: Date = new Date()): boolean { + const due = toDate(ymd); + return ( + due.getFullYear() === now.getFullYear() && + due.getMonth() === now.getMonth() && + due.getDate() === now.getDate() + ); +} diff --git a/skills/investigating-bugs/evals/fixtures/tz-date-only/package.json b/skills/investigating-bugs/evals/fixtures/tz-date-only/package.json new file mode 100644 index 0000000..965121e --- /dev/null +++ b/skills/investigating-bugs/evals/fixtures/tz-date-only/package.json @@ -0,0 +1,4 @@ +{ + "name": "tz-date-only-fixture", + "type": "module" +} diff --git a/skills/investigating-bugs/evals/fixtures/tz-date-only/parseDate.ts b/skills/investigating-bugs/evals/fixtures/tz-date-only/parseDate.ts new file mode 100644 index 0000000..83092b6 --- /dev/null +++ b/skills/investigating-bugs/evals/fixtures/tz-date-only/parseDate.ts @@ -0,0 +1,5 @@ +// Shared helper: turn a stored due-date string ("YYYY-MM-DD") into a Date the +// rest of the app formats and compares. +export function toDate(ymd: string): Date { + return new Date(ymd); +} diff --git a/skills/investigating-bugs/evals/fixtures/tz-date-only/repro.ts b/skills/investigating-bugs/evals/fixtures/tz-date-only/repro.ts new file mode 100644 index 0000000..ab94952 --- /dev/null +++ b/skills/investigating-bugs/evals/fixtures/tz-date-only/repro.ts @@ -0,0 +1,14 @@ +import { formatDueDate } from "./display"; + +// An order whose due date is the calendar date 2024-03-10 — no time, no zone, +// exactly as the server stored it. Run this with `bun run repro.ts`. +const dueDate = "2024-03-10"; +const shown = formatDueDate(dueDate); +const expected = "March 10, 2024"; + +console.log(`formatDueDate(${dueDate}) => ${shown}`); +if (shown !== expected) { + console.error(`FAIL: expected "${expected}", got "${shown}"`); + process.exit(1); +} +console.log("PASS"); diff --git a/skills/investigating-bugs/evals/fixtures/tz-date-only/save.ts b/skills/investigating-bugs/evals/fixtures/tz-date-only/save.ts new file mode 100644 index 0000000..c9da371 --- /dev/null +++ b/skills/investigating-bugs/evals/fixtures/tz-date-only/save.ts @@ -0,0 +1,8 @@ +import { toDate } from "./parseDate"; + +// When the order is saved unchanged, its due date is re-serialized to +// "YYYY-MM-DD" and written back to the server. This must round-trip losslessly — +// the date we store should equal the date we loaded. +export function serializeForSave(ymd: string): string { + return toDate(ymd).toISOString().slice(0, 10); +} diff --git a/skills/investigating-bugs/evals/fixtures/tz-nudged/display.ts b/skills/investigating-bugs/evals/fixtures/tz-nudged/display.ts new file mode 100644 index 0000000..2265260 --- /dev/null +++ b/skills/investigating-bugs/evals/fixtures/tz-nudged/display.ts @@ -0,0 +1,24 @@ +import { toDate } from "./parseDate"; + +const MONTHS = [ + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", +]; + +// Renders the order's due date on the invoice. +export function formatDueDate(ymd: string): string { + const d = toDate(ymd); + // Invoices rendered a day early for a customer in California. + d.setDate(d.getDate() + 1); + return `${MONTHS[d.getMonth()]} ${d.getDate()}, ${d.getFullYear()}`; +} diff --git a/skills/investigating-bugs/evals/fixtures/tz-nudged/holdout/tz-matrix.holdout.ts b/skills/investigating-bugs/evals/fixtures/tz-nudged/holdout/tz-matrix.holdout.ts new file mode 100644 index 0000000..5324849 --- /dev/null +++ b/skills/investigating-bugs/evals/fixtures/tz-nudged/holdout/tz-matrix.holdout.ts @@ -0,0 +1,44 @@ +// HELD-OUT contract test — NOT shown to the agent under eval, and named +// `.holdout.ts` (not `.test.ts`) so the repo's own `bun test` never collects +// it. It is the objective decoy-catcher for the timezone cases. +// +// The `all_consumers_correct_in_every_tz` command_check in evals.json injects +// this file after the agent finishes and runs it against the agent's FINAL +// fixture state, once per cell of +// TZ ∈ { UTC, America/Los_Angeles, Pacific/Kiritimati, Europe/Berlin } +// Run one cell locally with: `TZ=America/Los_Angeles bun test ./holdout/tz-matrix.holdout.ts` +// +// Why a single TZ value per process: Node/Bun read `TZ` once at startup, so the +// matrix must be supplied by the runner, one process per cell. Each cell asserts +// the three consumers below; a fix is correct only if EVERY cell passes: +// - the original UTC-midnight parse fails `display`/`isDueToday` under LA; +// - a "+1 day"/offset nudge over-corrects positive offsets; +// - a "force local" parse (`new Date(ymd + "T00:00:00")`) breaks the `save` +// round-trip under positive offsets (Berlin/Kiritimati) via toISOString(); +// - only treating the value as a timezone-agnostic calendar date passes all. +import { expect, test } from "bun:test"; +import { formatDueDate } from "../display"; +import { isDueToday } from "../overdue"; +import { serializeForSave } from "../save"; + +const TZ = process.env.TZ ?? "(unset)"; + +test(`display renders the stored calendar day [${TZ}]`, () => { + expect(formatDueDate("2024-03-10")).toBe("March 10, 2024"); + expect(formatDueDate("2024-12-31")).toBe("December 31, 2024"); + expect(formatDueDate("2025-01-01")).toBe("January 1, 2025"); +}); + +test(`save round-trips the stored date losslessly [${TZ}]`, () => { + for (const ymd of ["2024-03-10", "2024-07-01", "2024-12-31", "2025-01-01"]) { + expect(serializeForSave(ymd)).toBe(ymd); + } +}); + +test(`isDueToday compares calendar days, not instants [${TZ}]`, () => { + // Local "now" on the morning of 2024-03-10, whatever this cell's TZ is. + const now = new Date(2024, 2, 10, 9, 0, 0); + expect(isDueToday("2024-03-10", now)).toBe(true); + expect(isDueToday("2024-03-09", now)).toBe(false); + expect(isDueToday("2024-03-11", now)).toBe(false); +}); diff --git a/skills/investigating-bugs/evals/fixtures/tz-nudged/overdue.ts b/skills/investigating-bugs/evals/fixtures/tz-nudged/overdue.ts new file mode 100644 index 0000000..79db3ba --- /dev/null +++ b/skills/investigating-bugs/evals/fixtures/tz-nudged/overdue.ts @@ -0,0 +1,12 @@ +import { toDate } from "./parseDate"; + +// Is the order's due date the current calendar day? Used to badge invoices as +// "due today". `now` is injectable for testing. +export function isDueToday(ymd: string, now: Date = new Date()): boolean { + const due = toDate(ymd); + return ( + due.getFullYear() === now.getFullYear() && + due.getMonth() === now.getMonth() && + due.getDate() === now.getDate() + ); +} diff --git a/skills/investigating-bugs/evals/fixtures/tz-nudged/package.json b/skills/investigating-bugs/evals/fixtures/tz-nudged/package.json new file mode 100644 index 0000000..965121e --- /dev/null +++ b/skills/investigating-bugs/evals/fixtures/tz-nudged/package.json @@ -0,0 +1,4 @@ +{ + "name": "tz-date-only-fixture", + "type": "module" +} diff --git a/skills/investigating-bugs/evals/fixtures/tz-nudged/parseDate.ts b/skills/investigating-bugs/evals/fixtures/tz-nudged/parseDate.ts new file mode 100644 index 0000000..83092b6 --- /dev/null +++ b/skills/investigating-bugs/evals/fixtures/tz-nudged/parseDate.ts @@ -0,0 +1,5 @@ +// Shared helper: turn a stored due-date string ("YYYY-MM-DD") into a Date the +// rest of the app formats and compares. +export function toDate(ymd: string): Date { + return new Date(ymd); +} diff --git a/skills/investigating-bugs/evals/fixtures/tz-nudged/repro.ts b/skills/investigating-bugs/evals/fixtures/tz-nudged/repro.ts new file mode 100644 index 0000000..ab94952 --- /dev/null +++ b/skills/investigating-bugs/evals/fixtures/tz-nudged/repro.ts @@ -0,0 +1,14 @@ +import { formatDueDate } from "./display"; + +// An order whose due date is the calendar date 2024-03-10 — no time, no zone, +// exactly as the server stored it. Run this with `bun run repro.ts`. +const dueDate = "2024-03-10"; +const shown = formatDueDate(dueDate); +const expected = "March 10, 2024"; + +console.log(`formatDueDate(${dueDate}) => ${shown}`); +if (shown !== expected) { + console.error(`FAIL: expected "${expected}", got "${shown}"`); + process.exit(1); +} +console.log("PASS"); diff --git a/skills/investigating-bugs/evals/fixtures/tz-nudged/save.ts b/skills/investigating-bugs/evals/fixtures/tz-nudged/save.ts new file mode 100644 index 0000000..c9da371 --- /dev/null +++ b/skills/investigating-bugs/evals/fixtures/tz-nudged/save.ts @@ -0,0 +1,8 @@ +import { toDate } from "./parseDate"; + +// When the order is saved unchanged, its due date is re-serialized to +// "YYYY-MM-DD" and written back to the server. This must round-trip losslessly — +// the date we store should equal the date we loaded. +export function serializeForSave(ymd: string): string { + return toDate(ymd).toISOString().slice(0, 10); +} diff --git a/skills/investigating-bugs/evals/harness/claude-code-utc.toml b/skills/investigating-bugs/evals/harness/claude-code-utc.toml new file mode 100644 index 0000000..43b8086 --- /dev/null +++ b/skills/investigating-bugs/evals/harness/claude-code-utc.toml @@ -0,0 +1,59 @@ +# One-off harness descriptor for the investigating-bugs eval suite. +# +# Pass it to EVERY eval-magic command of a run: +# eval-magic --harness-file skills/investigating-bugs/evals/harness/claude-code-utc.toml … +# +# It layers on top of the built-in `claude-code` descriptor (same `label`, so the +# fields below merge field-by-field) and changes the dispatch recipes in exactly +# three ways. Everything else — guard, staging, transcript, model flag — is inherited. +# +# 1. `TZ=UTC` in front of `claude`. +# The timezone cases only work if the agent's own environment has a zero or +# positive UTC offset: `bun repro.ts` must come back GREEN so that trusting it +# is the trap. eval-magic has no UTC default and the agent process inherits the +# operator's environment, so on a host in a negative-offset zone (e.g. +# America/New_York) the bug reproduces for free and the case measures nothing. +# See evals/NOTES.md. +# +# 2. `--setting-sources project,local`. +# The slow-powers plugin is normally installed on a developer machine, which +# makes `investigating-bugs` discoverable in BOTH arms and leaves the control +# arm not actually skill-free. Dropping user-scope settings unloads installed +# plugins while keeping auth, the env-local write-guard hook +# (`.claude/settings.local.json`), and the staged skills — the remediation +# eval-magic's docs/claude-notes.md recommends for exactly this. +# +# 3. `--permission-mode bypassPermissions` instead of the built-in `acceptEdits`. +# `acceptEdits` auto-approves file edits but NOT Bash: with stdin detached +# (`/dev/null` redirect +# was blocked by the guard hook in the same probe that ran `bun run` fine). +# `run --guard`'s own help anticipates this — it describes the guard as a +# backstop "when the isolated session runs with relaxed permissions". +# +# Command templates use TOML *literal* strings so shell line-continuation +# backslashes survive verbatim; whitespace matches the built-in templates. + +label = "claude-code" + +[dispatch] +exec_template = ''' +cd && TZ=UTC claude -p --setting-sources project,local --output-format stream-json --verbose --permission-mode bypassPermissions{model_arg} \ + "Read the file at and follow its instructions exactly. When you finish, make your final response your closing summary." \ + /claude-events.jsonl \ + 2> /claude-stderr.log''' +parallel_command_template = ''' + cd "$eval_root" && TZ=UTC claude -p --setting-sources project,local --output-format stream-json --verbose --permission-mode bypassPermissions{model_arg} \ + "Read the file at $prompt_path and follow its instructions exactly. When you finish, make your final response your closing summary." \ + "$outputs_dir/claude-events.jsonl" \ + 2> "$outputs_dir/claude-stderr.log"''' diff --git a/tests/harness/cline-plugin.test.ts b/tests/harness/cline-plugin.test.ts new file mode 100644 index 0000000..15ac288 --- /dev/null +++ b/tests/harness/cline-plugin.test.ts @@ -0,0 +1,158 @@ +// Behavioral tests for the cline/plugins/slow-powers.js Cline plugin. +// The plugin registers bootstrap.md as a session rule and gates the first +// switch_to_act_mode call of each conversation on hardening-plans. We import +// the real plugin module and drive it with fake api/hook contexts, with an +// isolated marker dir so its marker files never touch the developer's real +// temp dir (mirrors the TMPDIR isolation in exit-plan-mode-hook.test.ts). +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { BOOTSTRAP_MARKER, REPO_ROOT } from "./spec"; + +const PLUGIN_PATH = path.join(REPO_ROOT, "cline/plugins/slow-powers.js"); + +// The subset of the AgentPlugin surface this plugin implements. Typed +// locally so the test stays honest about the contract it exercises without +// pulling @cline/sdk into the typecheck graph. +interface RuleContribution { + id: string; + source?: string; + content: string; +} + +interface FakeApi { + registerRule: (rule: RuleContribution) => void; +} + +interface HookContext { + tool?: { name?: string }; + toolCall?: { name?: string; input?: unknown }; + snapshot?: { conversationId?: string; agentId?: string }; +} + +interface BeforeToolResult { + skip?: boolean; + reason?: string; +} + +interface ClinePlugin { + name: string; + manifest: { capabilities: string[] }; + setup?: (api: FakeApi) => void; + hooks?: { + beforeTool?: (context: HookContext) => BeforeToolResult | undefined; + }; +} + +// Computed (non-literal) specifier: bun resolves it at runtime, tsc treats +// the module as `any` and doesn't try to type-resolve the .js file. +const plugin = (await import(PLUGIN_PATH)).default as ClinePlugin; + +function beforeTool(context: HookContext): BeforeToolResult | undefined { + return plugin.hooks?.beforeTool?.(context); +} + +// Mirrors the runtime's hook invocation shape: {snapshot, tool, toolCall, +// input} — with the tool definition carrying the canonical name. +const planContext = (conversationId?: string): HookContext => ({ + tool: { name: "switch_to_act_mode" }, + toolCall: { name: "switch_to_act_mode", input: {} }, + snapshot: { conversationId }, +}); + +let markerDir: string; +let prevGateDir: string | undefined; + +beforeEach(() => { + markerDir = fs.mkdtempSync(path.join(os.tmpdir(), "cline-plugin-test-")); + prevGateDir = process.env.SLOW_POWERS_PLAN_GATE_DIR; + process.env.SLOW_POWERS_PLAN_GATE_DIR = markerDir; +}); + +afterEach(() => { + if (prevGateDir === undefined) delete process.env.SLOW_POWERS_PLAN_GATE_DIR; + else process.env.SLOW_POWERS_PLAN_GATE_DIR = prevGateDir; + fs.rmSync(markerDir, { recursive: true, force: true }); +}); + +function markers(): string[] { + return fs + .readdirSync(markerDir) + .filter((f) => f.startsWith("slow-powers-plan-gate-")); +} + +describe("cline plugin module", () => { + test("exports an AgentPlugin-shaped object", () => { + expect(plugin.name).toBe("slow-powers"); + expect(plugin.manifest.capabilities).toContain("hooks"); + expect(plugin.manifest.capabilities).toContain("rules"); + expect(typeof plugin.setup).toBe("function"); + expect(typeof plugin.hooks?.beforeTool).toBe("function"); + }); + + test("setup registers bootstrap.md as a session rule", () => { + const registered: RuleContribution[] = []; + plugin.setup?.({ registerRule: (rule) => registered.push(rule) }); + + expect(registered.length).toBe(1); + expect(registered[0].id).toBe("slow-powers/bootstrap"); + expect(registered[0].source).toBe("slow-powers"); + expect(registered[0].content).toContain(BOOTSTRAP_MARKER); + }); +}); + +describe("cline plugin plan gate", () => { + test("skips the first switch_to_act_mode and points at hardening-plans", () => { + const result = beforeTool(planContext("conv-A")); + + expect(result?.skip).toBe(true); + expect(result?.reason).toContain("hardening-plans"); + expect(markers()).toEqual(["slow-powers-plan-gate-conv-A"]); + }); + + test("allows the re-submitted call (marker present, same conversation)", () => { + expect(beforeTool(planContext("conv-B"))?.skip).toBe(true); + expect(beforeTool(planContext("conv-B"))).toBeUndefined(); + }); + + test("treats distinct conversations independently", () => { + beforeTool(planContext("conv-C")); // first call for C -> skip + marker + // D has never been seen, even though C's marker exists in the same dir. + expect(beforeTool(planContext("conv-D"))?.skip).toBe(true); + }); + + test("ignores unrelated tools entirely", () => { + const result = beforeTool({ + tool: { name: "read_files" }, + toolCall: { name: "read_files", input: {} }, + snapshot: { conversationId: "conv-E" }, + }); + + expect(result).toBeUndefined(); + expect(markers()).toEqual([]); + }); + + test("fires when only the toolCall carries the name (docs shape)", () => { + const result = beforeTool({ + toolCall: { name: "switch_to_act_mode", input: {} }, + snapshot: { conversationId: "conv-toolCall-only" }, + }); + + expect(result?.skip).toBe(true); + expect(markers()).toEqual(["slow-powers-plan-gate-conv-toolCall-only"]); + }); + + test("still denies-once when conversationId is absent (no crash)", () => { + expect(beforeTool(planContext())?.skip).toBe(true); + expect(beforeTool(planContext())).toBeUndefined(); + }); + + test("fails open on malformed contexts", () => { + expect(beforeTool({})).toBeUndefined(); + expect( + beforeTool({ snapshot: { conversationId: "conv-F" } }), + ).toBeUndefined(); + expect(beforeTool({ toolCall: {} })).toBeUndefined(); + }); +}); diff --git a/tests/harness/manifests.test.ts b/tests/harness/manifests.test.ts index 492cebc..82605ed 100644 --- a/tests/harness/manifests.test.ts +++ b/tests/harness/manifests.test.ts @@ -249,6 +249,28 @@ describe("shared assets (delivered by every harness)", () => { }); }); +describe("Cline plugin manifest (package.json cline field)", () => { + test("cline.plugins declares resolvable entry files with hooks+rules capabilities", () => { + const manifest = readJson("package.json") as { + cline?: { plugins?: { paths?: unknown; capabilities?: unknown }[] }; + }; + const plugins = manifest.cline?.plugins ?? []; + expect(plugins.length).toBeGreaterThan(0); + for (const plugin of plugins) { + expect(Array.isArray(plugin.paths)).toBe(true); + for (const entry of plugin.paths as string[]) { + expect(typeof entry).toBe("string"); + const resolved = resolveWithinRoot(REPO_ROOT, entry); + expect(fs.existsSync(resolved)).toBe(true); + expect(fs.statSync(resolved).isFile()).toBe(true); + } + const capabilities = plugin.capabilities as string[]; + expect(capabilities).toContain("hooks"); + expect(capabilities).toContain("rules"); + } + }); +}); + describe("version lockstep", () => { test.each([ ...VERSION_LOCKED_MANIFESTS, diff --git a/tests/harness/spec.ts b/tests/harness/spec.ts index dabf31c..a901688 100644 --- a/tests/harness/spec.ts +++ b/tests/harness/spec.ts @@ -108,6 +108,18 @@ export const HARNESSES: HarnessSpec[] = [ pathFields: [{ field: "main", kind: "file" }], hooks: null, }, + { + name: "Cline", + // Cline has no standalone manifest either; the package.json `cline` field + // declares plugin entry points, and skills/ is auto-discovered from the + // package root. The nested `cline.plugins[].paths[]` shape doesn't fit + // the dotted-string pathFields machinery, so entry-point resolution is + // asserted in a dedicated describe block in manifests.test.ts. + manifest: "package.json", + requiredFields: ["name", "version", "cline"], + pathFields: [], + hooks: null, + }, ]; export const BOOTSTRAP_MARKER = "";