From 1cdc4084af1ba91f602ad1c9854ebc7f5e7dbb7c Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Fri, 19 Jun 2026 06:18:05 +0200
Subject: [PATCH 1/3] chore(evals): split self suites by plugin

---
 .../docs/getting-started/installation.mdx     |    5 +-
 evals/README.md                               |   61 +-
 evals/{self => agentv-dev}/skills/README.md   |   59 +-
 .../skills/fixtures/agentv-bench-full.txt     | 3658 ++++++++++++-----
 .../skills/fixtures/agentv-bench.txt          |   31 +
 .../fixtures/agentv-eval-review-full.txt      |  294 ++
 .../skills/fixtures/agentv-eval-review.txt    |   53 +
 .../fixtures/agentv-eval-writer-full.txt      |   86 +-
 .../skills/fixtures/agentv-eval-writer.txt    |    0
 .../fixtures/agentv-governance-full.txt       |  342 +-
 .../skills/fixtures/agentv-governance.txt     |   16 +
 .../fixtures/agentv-trace-analyst-full.txt    |   16 +
 .../skills/fixtures/agentv-trace-analyst.txt  |   16 +
 .../fixtures/skills-get-nonexistent.txt       |    2 +-
 .../skills/fixtures/skills-list-all.txt       |    1 +
 .../skills/output-correctness.eval.yaml       |    8 +-
 .../skills/skill-invocation.eval.yaml         |    2 +
 .../skills/skill-selection.eval.yaml          |   30 +-
 evals/agentv-self/agentv-self.eval.yaml       |   70 +
 .../azure-smoke.eval.yaml                     |    3 +-
 evals/agentv-self/scripts/setup.mjs           |   41 +
 evals/self/eval.yaml                          |   87 -
 .../fixtures/agentv-eval-review-full.txt      |   52 -
 .../skills/fixtures/agentv-eval-review.txt    |   52 -
 .../fixtures/agentv-onboarding-full.txt       |   63 -
 .../skills/fixtures/agentv-onboarding.txt     |   63 -
 .../self/skills/fixtures/skills-list-all.txt  |    1 -
 .../copilot-log-eval/workspace/AGENTS.md      |    5 +-
 28 files changed, 3400 insertions(+), 1717 deletions(-)
 rename evals/{self => agentv-dev}/skills/README.md (64%)
 rename evals/{self => agentv-dev}/skills/fixtures/agentv-bench-full.txt (55%)
 rename evals/{self => agentv-dev}/skills/fixtures/agentv-bench.txt (96%)
 create mode 100644 evals/agentv-dev/skills/fixtures/agentv-eval-review-full.txt
 create mode 100644 evals/agentv-dev/skills/fixtures/agentv-eval-review.txt
 rename evals/{self => agentv-dev}/skills/fixtures/agentv-eval-writer-full.txt (99%)
 rename evals/{self => agentv-dev}/skills/fixtures/agentv-eval-writer.txt (100%)
 rename evals/{self => agentv-dev}/skills/fixtures/agentv-governance-full.txt (98%)
 rename evals/{self => agentv-dev}/skills/fixtures/agentv-governance.txt (88%)
 rename evals/{self => agentv-dev}/skills/fixtures/agentv-trace-analyst-full.txt (93%)
 rename evals/{self => agentv-dev}/skills/fixtures/agentv-trace-analyst.txt (93%)
 rename evals/{self => agentv-dev}/skills/fixtures/skills-get-nonexistent.txt (60%)
 create mode 100644 evals/agentv-dev/skills/fixtures/skills-list-all.txt
 rename evals/{self => agentv-dev}/skills/output-correctness.eval.yaml (97%)
 rename evals/{self => agentv-dev}/skills/skill-invocation.eval.yaml (99%)
 rename evals/{self => agentv-dev}/skills/skill-selection.eval.yaml (83%)
 create mode 100644 evals/agentv-self/agentv-self.eval.yaml
 rename evals/{self => agentv-self}/azure-smoke.eval.yaml (74%)
 create mode 100644 evals/agentv-self/scripts/setup.mjs
 delete mode 100644 evals/self/eval.yaml
 delete mode 100644 evals/self/skills/fixtures/agentv-eval-review-full.txt
 delete mode 100644 evals/self/skills/fixtures/agentv-eval-review.txt
 delete mode 100644 evals/self/skills/fixtures/agentv-onboarding-full.txt
 delete mode 100644 evals/self/skills/fixtures/agentv-onboarding.txt
 delete mode 100644 evals/self/skills/fixtures/skills-list-all.txt

diff --git a/apps/web/src/content/docs/docs/getting-started/installation.mdx b/apps/web/src/content/docs/docs/getting-started/installation.mdx
index dca9211f9..5d3eec5f8 100644
--- a/apps/web/src/content/docs/docs/getting-started/installation.mdx
+++ b/apps/web/src/content/docs/docs/getting-started/installation.mdx
@@ -17,10 +17,11 @@ Install the AgentV CLI:
 npm install -g agentv
 ```
 
-Then load the onboarding skill and follow the instructions:
+Then load a bundled skill and follow its instructions. For eval authoring,
+`agentv-eval-writer` is the best starting point:
 
 ```bash
-agentv skills get agentv-onboarding
+agentv skills get agentv-eval-writer
 ```
 
 Paste the output to your AI agent and ask it to set up AgentV in your repository.
diff --git a/evals/README.md b/evals/README.md
index 0c0c2bc27..efec5c6b3 100644
--- a/evals/README.md
+++ b/evals/README.md
@@ -1,36 +1,49 @@
-# AgentV Self-Evaluation
+# AgentV Plugin-Aligned Self Evals
 
-This folder contains evaluations to ensure AgentV's `AGENTS.md` design principles don't regress.
+This directory holds the repo-owned self-evaluation suites, split to match the
+current plugin boundary:
+
+- `agentv-self` covers AgentV's own repo guidance and self-eval workspace
+  behavior.
+- `agentv-dev` covers the bundled `agentv skills` CLI surface and skill content
+  shipped with the developer plugin.
 
 ## Structure
 
-```
+```text
 evals/
-├── targets.yaml              # Target definitions
-├── design-principles.yaml    # Tests
-└── design-principles-grader.md # LLM grader prompt
+├── agentv-self/
+│   ├── agentv-self.eval.yaml
+│   ├── azure-smoke.eval.yaml
+│   └── scripts/setup.mjs
+├── agentv-dev/
+│   └── skills/
+│       ├── *.eval.yaml
+│       ├── fixtures/
+│       └── README.md
+└── agentic-engineering/
 ```
 
-## Known Issue: pi-coding-agent on Windows
-
-The pi-coding-agent provider has issues passing multi-line prompts on Windows. When running pi directly from CLI, it works correctly, but through the provider the prompt content is not received by pi.
-
-**Workaround**: Use mock provider for testing eval structure, or run on macOS/Linux.
+## Running
 
-**To fix**: Investigate `packages/core/src/evaluation/providers/pi-coding-agent.ts` - specifically how `spawn()` handles arguments with newlines on Windows.
-
-## Running the Evals
+Use the local CLI source from the repo root:
 
 ```bash
-# Dry run to verify structure
-bun agentv eval evals/design-principles.yaml --targets evals/targets.yaml --dry-run
-
-# Full run (requires working pi-coding-agent or alternative target)
-bun agentv eval evals/design-principles.yaml --targets evals/targets.yaml
+# Validate the renamed suites
+bun apps/cli/src/cli.ts validate evals/agentv-self/agentv-self.eval.yaml
+bun apps/cli/src/cli.ts validate evals/agentv-self/azure-smoke.eval.yaml
+bun apps/cli/src/cli.ts validate evals/agentv-dev/skills/*.eval.yaml
+
+# Prepare one agentv-self case and inspect the materialized workspace
+bun apps/cli/src/cli.ts prepare \
+  evals/agentv-self/agentv-self.eval.yaml \
+  --test-id guidance-split-paths \
+  --target codex
+
+# Run the agentv-dev skills suite against a target
+bun apps/cli/src/cli.ts eval run evals/agentv-dev/skills/*.eval.yaml --target <target>
 ```
 
-## Test Cases
-
-1. **violates-lightweight-core** - Proposes adding complex built-in (should fail)
-2. **violates-ai-first-design** - Proposes rigid CLI instructions (should fail)
-3. **follows-principles** - Proposes plugin/example approach (should pass)
+`agentv-self/agentv-self.eval.yaml` uses a `before_all` hook to copy the current repo
+checkout into the eval workspace. That keeps `/AGENTS.md` and `/.agents/`
+current without declaring extra repos in workspace config.
diff --git a/evals/self/skills/README.md b/evals/agentv-dev/skills/README.md
similarity index 64%
rename from evals/self/skills/README.md
rename to evals/agentv-dev/skills/README.md
index e476eeb9e..949334c2c 100644
--- a/evals/self/skills/README.md
+++ b/evals/agentv-dev/skills/README.md
@@ -1,9 +1,9 @@
-# `agentv skills` evals
+# `agentv-dev` skill evals
 
-Three-category eval suite covering the `agentv skills` CLI subcommand
-(`list`, `get`, `path`) shipped in #1224. Each category isolates one
-aspect of the skill UX so a regression in any single dimension is easy
-to read off the run report.
+Three-category eval suite covering the bundled `agentv skills` CLI surface
+owned by the `agentv-dev` plugin (`list`, `get`, `path`). Each category
+isolates one aspect of the skill UX so a regression in any single
+dimension is easy to read off the run report.
 
 ## Categories
 
@@ -19,16 +19,16 @@ skill evals: invocation (does the agent reach for the tool), selection
 
 ## Fixtures
 
-`fixtures/` holds frozen snapshots of CLI output. They are checked in
-so eval runs are deterministic and don't require network or build
-state. Two flavours per skill:
+`fixtures/` holds frozen snapshots of CLI output. They are checked in so
+eval runs are deterministic and don't require network or build state.
+Two flavours per skill:
 
 - `agentv-<name>.txt` — bare `SKILL.md` content (`agentv skills get <name>`).
   Used in most tests; small (1.5–25 KB).
-- `agentv-<name>-full.txt` — same plus every file under `references/`
-  and `templates/` (`agentv skills get <name> --full`). Only used in
-  tests that specifically validate `--full` behaviour, since these can
-  be hundreds of KB.
+- `agentv-<name>-full.txt` — same plus any bundled support directories
+  (`agentv skills get <name> --full`). Only used in tests that
+  specifically validate `--full` behaviour, since these can be hundreds
+  of KB.
 
 Plus two single-purpose fixtures:
 
@@ -41,52 +41,49 @@ After any change to bundled skill content or the `agentv skills` CLI,
 regenerate fixtures from the worktree root:
 
 ```bash
-cd evals/self/skills
+cd evals/agentv-dev/skills
 
 # Bare SKILL.md per skill
 for skill in agentv-bench agentv-eval-review agentv-eval-writer \
-             agentv-governance agentv-onboarding agentv-trace-analyst; do
-  node ../../../apps/cli/dist/cli.js skills get "$skill" \
+             agentv-governance agentv-trace-analyst; do
+  bun ../../../apps/cli/src/cli.ts skills get "$skill" \
     > "fixtures/${skill}.txt" 2>&1
 done
 
-# --full variants (with references/ + templates/)
+# --full variants (with references/, scripts/, agents/, and similar)
 for skill in agentv-bench agentv-eval-review agentv-eval-writer \
-             agentv-governance agentv-onboarding agentv-trace-analyst; do
-  node ../../../apps/cli/dist/cli.js skills get "$skill" --full \
+             agentv-governance agentv-trace-analyst; do
+  bun ../../../apps/cli/src/cli.ts skills get "$skill" --full \
     > "fixtures/${skill}-full.txt" 2>&1
 done
 
 # Listing + error fixtures
-node ../../../apps/cli/dist/cli.js skills list --json \
+bun ../../../apps/cli/src/cli.ts skills list --json \
   > fixtures/skills-list-all.txt 2>&1
-node ../../../apps/cli/dist/cli.js skills get does-not-exist \
+bun ../../../apps/cli/src/cli.ts skills get does-not-exist \
   > fixtures/skills-get-nonexistent.txt 2>&1
 ```
 
-`bun apps/cli/src/cli.ts skills …` works equivalently when running
-against TypeScript sources.
-
 ## Running
 
 From the worktree root:
 
 ```bash
 # All three categories against one target
-node apps/cli/dist/cli.js eval run evals/self/skills/*.eval.yaml --target <target>
+bun apps/cli/src/cli.ts eval run evals/agentv-dev/skills/*.eval.yaml --target <target>
 
 # A single category
-node apps/cli/dist/cli.js eval run \
-  evals/self/skills/skill-selection.eval.yaml --target azure
+bun apps/cli/src/cli.ts eval run \
+  evals/agentv-dev/skills/skill-selection.eval.yaml --target azure
 
 # A single test
-node apps/cli/dist/cli.js eval run \
-  evals/self/skills/skill-invocation.eval.yaml \
+bun apps/cli/src/cli.ts eval run \
+  evals/agentv-dev/skills/skill-invocation.eval.yaml \
   --test-id invoke-get-full-flag --target azure
 ```
 
-`<target>` is any name resolvable from `targets.yaml` in the worktree
-(e.g. `azure`, `claude`, `mock`).
+`<target>` is any name resolvable from `.agentv/targets.yaml` in the worktree
+(for example `azure`, `claude`, or `codex`).
 
 ## Adding test cases
 
@@ -110,7 +107,7 @@ Pattern for fixture-driven tests:
     - role: user
       content:
         - type: file
-          value: fixtures/agentv-onboarding.txt
+          value: fixtures/agentv-eval-writer.txt
         - type: text
           value: |
             <prompt referring to the fixture>
diff --git a/evals/self/skills/fixtures/agentv-bench-full.txt b/evals/agentv-dev/skills/fixtures/agentv-bench-full.txt
similarity index 55%
rename from evals/self/skills/fixtures/agentv-bench-full.txt
rename to evals/agentv-dev/skills/fixtures/agentv-bench-full.txt
index 29ef810f0..36d962d71 100644
--- a/evals/self/skills/fixtures/agentv-bench-full.txt
+++ b/evals/agentv-dev/skills/fixtures/agentv-bench-full.txt
@@ -175,6 +175,21 @@ When each subagent task completes, you receive a notification containing `total_
 
 This is the only opportunity to capture this data — it comes through the task notification and isn't persisted elsewhere. Process each notification as it arrives.
 
+### After all executors complete
+
+Once all executor subagent notifications arrive, **read `response.md` directly from disk**.
+Do NOT use `read_agent` to fetch executor results — they are already written to the run
+directory. Verify all responses exist before proceeding:
+
+```bash
+for d in <run-dir>/<evalset>/*/; do
+  echo "$(basename $d): $(ls "$d"/response.md 2>/dev/null && echo OK || echo MISSING)"
+done
+```
+
+If any `response.md` is MISSING, re-run that specific executor subagent. Do not proceed to
+grading until all responses are present.
+
 ### Grading
 
 **In CLI mode**, `agentv eval` handles all grading end-to-end — no manual phases needed.
@@ -427,1512 +442,2881 @@ Repeating the core loop for emphasis:
 
 Take your time with improvements. Read the transcripts. Understand why failures happened. Make changes that generalize beyond the test set. This is important work.
 
---- references/autoresearch.md ---
-# Autoresearch Mode
-
-Autoresearch is an unattended eval-improve loop that runs multiple optimize cycles without human intervention. The user triggers it with natural language (e.g., "run autoresearch on this skill", "optimize this skill unattended"). No YAML schema changes or CLI flags are needed.
-
-## Automated Keep/Discard
+## Accessing reference files
 
-After each iteration, you can automatically decide whether to keep or discard the change using structured comparison output. This replaces manual judgment at steps 3–4 of the iteration loop (Step 5 in SKILL.md), except at human checkpoint iterations (3, 6, 9) where you must still present results to the user.
+To load a specific reference without pulling the entire skill into context:
 
-### 1. Run the comparison
+```bash
+agentv skills get agentv-bench --ref eval-yaml-spec
+```
 
-After re-running test cases, compare the new results against the previous iteration's baseline:
+Or resolve the skill directory and read files directly:
 
 ```bash
-agentv compare <baseline>.jsonl <candidate>.jsonl --json
+cat $(agentv skills path agentv-bench)/references/eval-yaml-spec.md
 ```
 
-Where `<baseline>.jsonl` is the `index.jsonl` from the previous best iteration and `<candidate>.jsonl` is the `index.jsonl` from the run you just completed.
+Use `--full` to retrieve every file in the skill at once.
 
-### 2. Parse the output
+--- agents/executor.md ---
+---
+name: executor
+description: >-
+  Execute an AgentV evaluation test case by performing the task described in the
+  input. Reads input.json from the test directory, carries out the task using
+  available tools, and writes response.md with the result. Dispatch one executor
+  subagent per test case, all in parallel.
+model: inherit
+color: cyan
+---
 
-The `--json` flag produces structured output:
+You are the executor for an AgentV evaluation test case. Your job is to **perform the task** described in the input and write your response.
+
+You are the target agent being evaluated. Do the task to the best of your ability — your output will be graded by a separate grader agent.
+
+**You will receive these parameters:**
+- `test-dir`: Path to the test case directory (e.g., `.agentv/results/runs/<timestamp>/<test-id>/`)
+
+## Process
+
+1. **Read `{test-dir}/input.json`**. It contains `input` (Message array), `input_files` (optional file paths), and `metadata` (optional context). If `input_files` are listed, read those files too.
+
+2. **Perform the task** described in the input.
+
+3. **Write `{test-dir}/response.md`** with everything a grader needs to evaluate your work — your answer, actions taken, code produced, and any errors encountered. If you modified files, summarize the changes so the grader can evaluate without reading every file.
+
+## Important
+
+- Do NOT read grading criteria, assertions, or expected outputs — those are for the grader, not for you.
+- Write `response.md` even if you couldn't complete the task — explain what happened and what you tried.
+
+--- agents/comparator.md ---
+---
+name: comparator
+description: >-
+  Perform bias-free blind comparison of evaluation outputs from multiple providers
+  or configurations. Randomizes labeling, generates task-specific rubrics, scores
+  N-way comparisons, then unblinds results and attributes improvements. Dispatch
+  this agent when comparing outputs across targets or iterations.
+model: inherit
+color: cyan
+tools: ["Read", "Bash", "Glob", "Grep", "Write"]
+---
+
+You are the Blind Comparator for AgentV's evaluation workflow. Your job is to compare outputs from multiple targets (providers, configurations, agent versions) without knowing which target produced which output, then score them on dynamically generated rubrics.
+
+## Core Principles
+
+1. **Blind evaluation**: You MUST NOT know which target produced which output during scoring. Outputs are labeled A, B, C, ... only.
+2. **Dynamic rubrics**: Generate scoring criteria specific to the task — do not use a fixed rubric for all comparisons.
+3. **Multi-dimensional scoring**: Score each output on content quality AND structural quality independently.
+4. **N-way support**: Handle 2 or more outputs, not just binary A/B.
+
+## Input Parameters
+
+You will receive:
+- `outputs`: Array of evaluation outputs to compare. Each contains:
+  - `target_id`: The provider/configuration identifier (DO NOT read this during scoring)
+  - `answer`: The candidate response text
+  - `evaluator_results`: Array of grader scores and details (code-grader, tool-trajectory, llm-grader, deterministic)
+  - `workspace_changes`: File changes made during workspace evaluation (if applicable)
+  - `tool_calls`: Tool invocations and results from multi-turn conversations (if applicable)
+  - `conversation`: Full multi-turn conversation history (if applicable)
+- `task_context`: Description of what the evaluation tests (task type, domain, expected behavior)
+- `results_file`: Path to write the comparison results
+
+## Process
+
+### Phase 1: Blind Labeling
+
+Assign random labels to outputs. Use the following procedure:
+
+1. Collect all outputs into an array
+2. Shuffle the array randomly (use Python if deterministic randomization is needed):
+   ```bash
+   python3 -c "
+   import json, random, sys
+   outputs = json.loads(sys.stdin.read())
+   random.shuffle(outputs)
+   labels = [chr(65 + i) for i in range(len(outputs))]  # A, B, C, ...
+   mapping = {labels[i]: outputs[i]['target_id'] for i in range(len(outputs))}
+   labeled = [{'label': labels[i], 'answer': outputs[i]['answer'],
+                'evaluator_results': outputs[i].get('evaluator_results', []),
+                'workspace_changes': outputs[i].get('workspace_changes', []),
+                'tool_calls': outputs[i].get('tool_calls', []),
+                'conversation': outputs[i].get('conversation', [])}
+               for i in range(len(outputs))]
+   print(json.dumps({'labeled': labeled, 'mapping': mapping}))
+   " <<< '<outputs_json>'
+   ```
+3. Store the label→target mapping but DO NOT reference it until Phase 4
+4. Proceed with scoring using only the labeled outputs
+
+### Phase 2: Dynamic Rubric Generation
+
+Generate task-specific rubrics based on `task_context` and the grader types present. The rubric has two dimensions:
+
+**Content Rubric** — adapts criteria to the task type:
+
+| Task Type | Content Criteria |
+|---|---|
+| Code generation | Correctness, completeness, edge case handling, idiomatic usage |
+| Code review | Issue identification accuracy, severity assessment, actionable suggestions |
+| Q&A / knowledge | Factual accuracy, completeness, source grounding |
+| Creative writing | Relevance, coherence, style adherence, originality |
+| Tool use / agent | Tool selection appropriateness, execution correctness, goal completion |
+| Multi-turn conversation | Context retention, coherent progression, task completion across turns |
+| Workspace evaluation | File change correctness, build/test pass rate, requirement coverage |
+
+For each content criterion, define:
+- Name and description
+- Weight (0.0–1.0, sum to 1.0 within content)
+- Scoring anchor: what 1, 5, and 10 look like
+
+**Structure Rubric** — consistent across task types:
+
+| Criterion | Weight | Description |
+|---|---|---|
+| Organization | 0.3 | Logical flow, section structure, progressive disclosure |
+| Clarity | 0.3 | Unambiguous language, concise expression, no unnecessary jargon |
+| Format compliance | 0.2 | Adherence to requested output format (JSON, markdown, code blocks) |
+| Completeness | 0.2 | All requested sections present, no truncation |
+
+**Grader-Specific Scoring** — when grader results are present:
+
+- **code-grader**: Factor in pass/fail results, test coverage, assertion hit rates
+- **tool-trajectory**: Factor in tool call accuracy, sequence correctness, unnecessary tool calls
+- **llm-grader**: Factor in existing LLM grader scores as a reference signal (not as ground truth)
+- **deterministic**: Factor in exact match / keyword hit rates
+
+### Phase 3: Scoring
+
+For each labeled output (A, B, C, ...):
+
+1. **Content score** (1–10): Apply the content rubric criteria with weights
+2. **Structure score** (1–10): Apply the structure rubric criteria with weights
+3. **Grader score** (1–10): Normalize grader results to a 1–10 scale. If no grader results, omit this dimension.
+4. **Overall score**: Weighted combination:
+   - If grader results present: `0.5 × content + 0.2 × structure + 0.3 × grader`
+   - If no grader results: `0.7 × content + 0.3 × structure`
+
+For N > 2 outputs, use **round-robin pairwise comparison** to establish ranking:
+- Compare every pair (A vs B, A vs C, B vs C, ...)
+- Track pairwise wins for each output
+- Final ranking uses: (1) overall score, (2) pairwise win count as tiebreaker
+
+For each output, record:
+- Per-criterion scores with brief justification
+- Top 3 strengths
+- Top 3 weaknesses
+- Key differentiators vs other outputs
+
+### Phase 4: Unblinding
+
+After ALL scoring is complete:
+1. Reveal the label→target mapping
+2. Associate scores with actual target identifiers
+3. Do NOT revise any scores after unblinding
+
+### Phase 5: Post-hoc Analysis
+
+After unblinding, analyze *why* the winner won. This phase absorbs the logic from the former comparison-analyzer agent.
+
+1. **Improvement attribution** — identify what specific changes between iterations or configurations drove improvements or regressions. Quote from the outputs.
+2. **Instruction-following analysis** — did each target follow the task instructions? Score 1-10 with specific issues noted.
+3. **Actionable suggestions** — produce concrete improvement suggestions for the losing output(s), prioritized by expected impact:
+   - `high`: Would likely change the outcome
+   - `medium`: Would improve quality but may not change ranking
+   - `low`: Nice to have, marginal improvement
+4. **Categorize suggestions**: instructions, tools, examples, error_handling, structure, references
+
+Include the analysis in the output JSON under `post_hoc_analysis`.
+
+## Output Format
+
+Write the comparison results to `results_file` as JSON:
 
 ```json
 {
-  "summary": {
-    "wins": 3,
-    "losses": 1,
-    "ties": 6,
-    "mean_delta": 0.05
+  "comparison_id": "<timestamp>-<random-suffix>",
+  "task_context": "<task description>",
+  "output_count": <N>,
+  "rubric": {
+    "content": {
+      "criteria": [
+        {"name": "<criterion>", "weight": <0.0-1.0>, "description": "<what this measures>"}
+      ]
+    },
+    "structure": {
+      "criteria": [
+        {"name": "<criterion>", "weight": <0.0-1.0>, "description": "<what this measures>"}
+      ]
+    },
+    "overall_weights": {
+      "content": <weight>,
+      "structure": <weight>,
+      "grader": <weight or null>
+    }
+  },
+  "results": [
+    {
+      "label": "A",
+      "target_id": "<revealed after unblinding>",
+      "scores": {
+        "content": <1-10>,
+        "structure": <1-10>,
+        "grader": <1-10 or null>,
+        "overall": <1-10>
+      },
+      "content_breakdown": [
+        {"criterion": "<name>", "score": <1-10>, "justification": "<brief>"}
+      ],
+      "structure_breakdown": [
+        {"criterion": "<name>", "score": <1-10>, "justification": "<brief>"}
+      ],
+      "evaluator_breakdown": [
+        {"evaluator_name": "<name>", "type": "<type>", "raw_score": <0.0-1.0>, "normalized": <1-10>}
+      ],
+      "strengths": ["<strength 1>", "<strength 2>", "<strength 3>"],
+      "weaknesses": ["<weakness 1>", "<weakness 2>", "<weakness 3>"]
+    }
+  ],
+  "pairwise": [
+    {"pair": ["A", "B"], "winner": "A", "margin": <score_diff>}
+  ],
+  "ranking": [
+    {"rank": 1, "label": "A", "target_id": "<id>", "overall_score": <score>, "pairwise_wins": <N>}
+  ],
+  "winner": {
+    "label": "<winning label>",
+    "target_id": "<winning target>",
+    "overall_score": <score>,
+    "margin_over_second": <score_diff>
   }
 }
 ```
 
-- **wins**: number of test cases where the candidate scored higher than the baseline
-- **losses**: number of test cases where the candidate scored lower
-- **ties**: number of test cases with no score change
-- **mean_delta**: average score difference across all test cases (positive = candidate is better)
-
-### 3. Apply decision rules
+Also produce a human-readable markdown summary:
 
-Use these rules in order:
+```markdown
+## Blind Comparison Results
 
-| Condition | Decision | Action |
-|-----------|----------|--------|
-| `wins > losses` | **KEEP** | Promote the candidate to the new baseline. Copy or note its `index.jsonl` path as the baseline for the next iteration. |
-| `wins <= losses` | **DISCARD** | Revert the prompt/skill/config change. The previous baseline remains. Try a different mutation on the next iteration. |
-| `mean_delta == 0` AND candidate prompt is shorter (fewer lines) | **KEEP** | Simpler prompts are preferred when performance is equal. Promote the candidate as the new baseline. |
+### Task
+<task_context>
 
-When `mean_delta == 0` and the candidate prompt is *not* shorter, treat it as a **DISCARD** — there's no reason to keep a change that adds complexity without improving results.
+### Rubric
+<generated rubric summary>
 
-### 4. Log the decision
+### Rankings
+| Rank | Label | Target | Overall | Content | Structure | Grader |
+|------|-------|--------|---------|---------|-----------|-----------|
+| 1    | A     | <id>   | 8.5     | 9.0     | 7.5       | 8.5       |
 
-Before proceeding to the next iteration, log the decision and rationale so the user can review later:
+### Winner: <label> (<target_id>)
+- **Margin**: +<diff> over second place
+- **Key differentiators**: <why this output won>
 
-```
-Iteration 2: KEEP
-  wins=3, losses=1, ties=6, meanDelta=+0.05
-  Rationale: candidate wins outweigh losses (3 > 1)
-  Baseline promoted: .agentv/results/runs/20250101-120000/index.jsonl
+### Per-Output Analysis
+#### Output A (<target_id>)
+- **Strengths**: ...
+- **Weaknesses**: ...
 ```
 
-```
-Iteration 3: DISCARD
-  wins=1, losses=2, ties=7, meanDelta=-0.03
-  Rationale: candidate losses outweigh wins (2 > 1)
-  Reverted to baseline: .agentv/results/runs/20250101-110000/index.jsonl
-  Next: try a different mutation
-```
+## Scoring Guidelines
 
-Include this log in your progress summary. At human checkpoints (iterations 3, 6, 9), present the full log of automated decisions since the last checkpoint alongside the current results.
+- **Be rigorous**: Do not inflate scores. A score of 7 means good but with notable gaps.
+- **Be consistent**: Apply the same rubric uniformly to all outputs.
+- **Be evidence-based**: Every score must cite specific evidence from the output.
+- **Evaluate substance over style**: Correct, complete answers with rough formatting score higher than polished but incorrect answers.
+- **Handle missing data gracefully**: If an output lacks workspace changes or tool calls but others have them, score what is present — do not penalize for data the target wasn't expected to produce.
+- **Respect grader signals**: When code-grader or tool-trajectory results exist, they represent objective ground truth. Weight these heavily.
 
-### 5. Integration with the iteration loop
+## Edge Cases
 
-The automated keep/discard replaces the manual compare-and-present cycle (steps 3–4) during non-checkpoint iterations. The full flow becomes:
+- **Identical outputs**: If two outputs are effectively identical, score them equally and note the duplication.
+- **Single output**: If only one output is provided, still generate the rubric and score it — this serves as a baseline for future comparisons.
+- **Missing grader results**: If some outputs have grader results and others don't, score grader dimension only for those that have it. Adjust overall weights accordingly.
+- **Very long outputs**: Focus scoring on substance and correctness. Length alone is neither a positive nor negative signal.
+- **Tie in overall scores**: Use pairwise comparison wins as tiebreaker. If still tied, declare a tie and explain the tradeoffs.
 
-1. Apply change to prompts/skills/config
-2. Re-run all test cases
-3. Run `agentv compare baseline.jsonl candidate.jsonl --json`
-4. Apply keep/discard rules → promote or revert
-5. Log the decision
-6. If this is iteration 3, 6, or 9 → present progress to the user (human checkpoint)
-7. Check stop conditions → continue or stop
+--- agents/mutator.md ---
+---
+name: mutator
+description: >-
+  Generate improved versions of the artifact under test (skill, prompt, config,
+  or directory of related files) based on failure analysis. Reads the current
+  best artifact from the working tree, applies targeted mutations to address
+  failing assertions, and writes changes in place. Supports single files and
+  multi-file directories. Dispatch this agent after analyzer identifies failure patterns.
+model: inherit
+color: green
+tools: ["Read", "Write", "Bash", "Glob", "Grep"]
+---
 
-Both modes coexist: if the user is actively reviewing results, present to them as before. If the user has asked you to iterate autonomously, use automated keep/discard and only pause at human checkpoints.
+You are the Mutator for AgentV's evaluation workflow. Your job is to rewrite the artifact under test so that failing assertions start passing, while preserving everything that already works. You produce **complete replacement files** — never diffs, patches, or suggestion lists.
 
----
+## Core Principles
 
-## Prerequisites
+1. **Hill-climbing ratchet**: Always read from the "best" version, never from a failed candidate. Each mutation builds on the highest-scoring artifact so far.
+2. **Evidence-driven only**: Every change you make must trace back to a specific failing assertion or failure description. Never add speculative features.
+3. **Preserve passing behavior**: Instructions that already pass consistently must survive unchanged in meaning. You may rephrase for clarity, but do not alter intent.
+4. **Simplicity criterion**: When two versions score equally, prefer the simpler one. Remove redundant or verbose instructions that don't contribute to passing assertions. Cleaner artifacts at equal performance are improvements.
 
-- An eval file (`EVAL.yaml` or `evals.json`) must exist for the artifact being optimized.
-- The artifact must be a file or directory (SKILL.md, prompt template, agent config, or a directory of related files like a skill with references/).
-- The user should have run at least one interactive eval cycle to build confidence in eval quality before going unattended.
+## Input Parameters
 
-## The loop
+You will receive:
+- `artifact-path`: Path to the file or directory to mutate (the artifact under test). **Write changes back to this same path.**
+- `artifact-mode`: `file` or `directory`. Determines how you read and write the artifact.
+- `initial-sha`: The git commit SHA before any autoresearch mutations began. Use `git show <initial-sha>:<path>` to reference the original version when needed.
+- `pass-rates`: Per-assertion pass rates as a JSON mapping, e.g. `{"IDENTIFIES_CLARITY_ISSUES": 0.6, "SUGGESTS_CONCRETE_FIX": 1.0, "OUTPUT_IS_STRUCTURED": 0.2}`
+- `run-dir`: Path to this cycle's eval run directory. **Read `grading.json` here** to understand why assertions failed (evidence, per-test scores). Read test transcripts/responses as needed.
+- `iterations-path`: Path to `_autoresearch/iterations.jsonl`. **Read this** to see mutation history and avoid repeating failed strategies.
+- `iteration`: Current iteration number (for context in the changelog)
+- `focus-files` (directory mode, optional): Files most likely contributing to failures — read these first.
 
-```
-1. RUN EVAL   — agentv eval with current artifact
-2. ANALYZE    — dispatch analyzer subagent on results
-3. DECIDE     — if score > best_score: KEEP, else DROP (automated keep/discard above)
-4. MUTATE     — dispatch mutator subagent with failure analysis (agents/mutator.md)
-5. GOTO 1     — until convergence or max_cycles
-```
+## Process
 
-## Experiment naming
+### Step 1: Read Inputs
 
-Derive the experiment name from the artifact: `autoresearch-<name>` (e.g., `autoresearch-pdf-skill`). The user can also provide a custom name.
+1. **Read the current best artifact** from the working tree at `artifact-path`. This is your mutation base (HEAD always contains the best-known version after KEEP commits or DROP reverts).
+2. **Reference the original** via `git show <initial-sha>:<path>` when you need to understand the author's original intent. Don't use this as the mutation base.
+3. **For directory mode**: Read `focus-files` first if provided, then selectively read others as needed. For large directories (>15 files), don't read everything — focus on the files most relevant to failing assertions.
+4. **Parse pass rates** to classify each assertion:
+   - **Passing** (≥ 80%): Preserve the instructions responsible for these.
+   - **Failing** (< 80%): These are your mutation targets.
+   - **Near-passing** (60–79%): May need only minor reinforcement.
+   - **Hard-failing** (< 40%): Need substantial new instructions.
+5. **Read failure evidence** from `<run-dir>/grading.json` to understand *why* assertions fail — look at per-test assertion evidence, not just which ones fail. For deeper analysis, read individual test responses in `<run-dir>/<test-id>/response.md`.
+6. **Read mutation history** from `iterations-path` to see what was tried before — avoid repeating strategies that led to DROPs.
 
-## Artifact mutation flow
+### Step 2: Analyze Failure Causes
 
-The mutator rewrites artifacts in the working tree in place. **Git is used for versioning** — HEAD always contains the best-known version:
+For each failing assertion, determine the root cause:
 
-1. Record the starting commit SHA before the first cycle: `initial_sha=$(git rev-parse HEAD)`.
-2. On each **KEEP**: `git add <artifact-path> && git commit -m "autoresearch cycle N: <mutation summary>"`.
-3. On each **DROP**: `git checkout -- <artifact-path>` (restores working tree to HEAD, the last KEEP commit).
-4. The eval always runs against the real file path — no temp files or indirection.
-5. The mutator can reference the original via `git show <initial_sha>:<path>`.
+| Pattern | Likely Cause | Mutation Strategy |
+|---------|-------------|-------------------|
+| Agent omits a required behavior | Missing instruction | Add an explicit, concrete instruction |
+| Agent does the opposite of what's expected | Ambiguous or contradictory instruction | Rewrite the instruction to be unambiguous |
+| Agent partially satisfies the criterion | Instruction is vague | Add specifics — examples, formats, constraints |
+| Agent satisfies it sometimes but not always | Instruction exists but is easy to overlook | Elevate priority — move to a prominent position, add emphasis |
+| Output format doesn't match expectations | Missing format specification | Add explicit format requirements with examples |
 
-## How the skill invokes eval
+### Step 3: Plan Mutations
 
-Shell out to `agentv eval <eval-path> --experiment autoresearch-<name>` via the Bash tool, same as the existing interactive bench workflow.
+Before writing, plan your changes:
 
-## Artifact layout
+1. **List each failing assertion** and the specific instruction change that addresses it.
+2. **Check for conflicts**: Will a new instruction contradict or undermine a passing one? If so, find a formulation that satisfies both.
+3. **Check for redundancy**: If two failing assertions share a root cause, one instruction change may fix both.
+4. **Apply simplicity criterion**: If the best artifact has verbose instructions for passing assertions, consider simplifying them — but only if you're confident the simplification won't cause regressions.
 
-Each cycle is a standard eval run. Autoresearch session metadata lives in `_autoresearch/` within the experiment directory:
+### Step 4: Write the Mutated Artifact
 
-```
-.agentv/results/runs/<experiment>/
-  _autoresearch/
-    iterations.jsonl               # one line per cycle — data for chart + mutator
-    trajectory.html                # live-updating score trajectory chart
-  2026-04-15T10-30-00/             # cycle 1 — standard run artifacts
-    index.jsonl
-    grading.json
-    timing.json
-    benchmark.json
-    report.html
-  2026-04-15T10-35-00/             # cycle 2 — standard run artifacts
-    ...
-```
+1. **Re-read the artifact** from the working tree to ensure you have the latest content.
+2. **Apply your planned mutations** to produce complete rewritten files.
+3. **Write the result** to `artifact-path` (in-place mutation).
 
-No `original.md` or `best.md` files — git history serves as the backup. The `_` prefix convention distinguishes workflow folders from timestamped run dirs.
+**File mode**: Write a single complete file. The output must be standalone — no diff markers, comments about what changed, or meta-content.
 
-## iterations.jsonl
+**Directory mode**: You can modify any file within the artifact scope, and you can create new files within it. Only write files you actually changed — don't rewrite unchanged files. Do not delete files (modifications and creations only).
 
-One JSON object per line, one line per cycle:
+### Step 5: Produce a Changelog
+
+After writing the artifact, output a structured changelog explaining what you changed and why. This will be logged in `iterations.jsonl` for audit.
 
-```jsonl
-{"cycle":1,"score":0.65,"decision":"keep","cost_usd":0.12,"assertions":{"IDENTIFIES_BUG":0.8,"SUGGESTS_FIX":0.4},"mutation":"added explicit null-check instruction","run_dir":"2026-04-15T10-30-00","timestamp":"2026-04-15T10:32:15Z"}
 ```
+## Mutation Report (Iteration {iteration})
 
-Fields: `cycle` (1-indexed), `score` (overall pass rate 0–1), `decision` ("keep" or "drop"), `cost_usd` (eval run cost), `assertions` (per-assertion pass rates), `mutation` (one-line description of what changed), `run_dir` (timestamped directory name), `timestamp` (ISO 8601).
+### Assertions Targeted
 
-## trajectory.html
+| Assertion | Pass Rate | Action Taken |
+|-----------|-----------|-------------|
+| IDENTIFIES_CLARITY_ISSUES | 3/5 (60%) | Added explicit instruction to check for ambiguous pronouns |
+| OUTPUT_IS_STRUCTURED | 1/5 (20%) | Added format specification with markdown header requirements |
+| SUGGESTS_CONCRETE_FIX | 5/5 (100%) | No change (passing) |
 
-A standalone HTML chart file with embedded Chart.js. Copy the template from `scripts/trajectory.html` into the `_autoresearch/` directory. It fetches `iterations.jsonl` from the same directory on each auto-refresh — no data injection needed. Shows:
+### Changes Made
 
-- Score over iterations (line chart) with KEEP (green) / DISCARD (red) markers
-- Per-assertion pass rates over iterations
-- Cumulative cost across iterations
-- Best vs original score summary
+1. **[Section/Location]**: [What changed] — addresses [ASSERTION_NAME] failing because [reason from failure descriptions]
+2. ...
 
-Auto-refreshes every 2 seconds during the loop. Becomes static after completion (remove the auto-refresh meta tag on final update).
+### Preserved
 
-## Convergence
+- [List of key instructions left unchanged because their assertions pass]
 
-Stop after **3** consecutive cycles with no improvement (no KEEP). Also stop at **max_cycles** (default 10). Either limit can be overridden by the user.
+### Simplifications
 
-## Human checkpoints
+- [Any instructions simplified or removed, with justification]
 
-Autoresearch mode **skips** human checkpoints at iterations 3/6/9. The user opted in to unattended operation by requesting autoresearch.
+### Risk Assessment
 
-## Context hygiene
+- [Any changes that might affect currently-passing assertions, and why you believe they're safe]
+```
 
-The orchestrator must run indefinitely without exhausting its context window. To do this:
+## Mutation Strategies
 
-- **Never read eval results, artifacts, or transcripts into your own context.** Use bash commands (jq, agentv CLI) that output small structured summaries.
-- **Delegate all heavy reading to subagents.** The mutator reads artifacts, grading results, and transcripts from disk — you pass it paths, not content.
-- **Use bash for all file I/O** in the loop body: appending to `iterations.jsonl`, git operations, score extraction. The only tool calls per cycle should be bash commands and one subagent dispatch (mutator).
-- **trajectory.html auto-loads `iterations.jsonl`** via fetch — no need to read or update the HTML file after initial copy.
+### For assertions below 80% pass rate: Add explicit instructions
 
-## Procedure
+**Bad** (vague):
+> Be thorough in your analysis.
 
-Follow this step-by-step procedure to execute autoresearch:
+**Good** (concrete and actionable):
+> For each input, check for: (1) ambiguous pronouns — flag any pronoun without a clear antecedent within the same sentence, (2) implicit assumptions — identify claims that assume context not provided in the input.
 
-### 1. Setup
+### For near-passing assertions (60–79%): Reinforce existing instructions
 
-1. Determine the **artifact path** (file or directory to optimize) and **eval path** (EVAL.yaml or evals.json).
-2. Detect **artifact mode**: `file` if the artifact path is a file, `directory` if it's a directory.
-3. Derive the **experiment name**: `autoresearch-<name>` from the artifact filename/dirname, or use a user-provided name.
-4. Set the experiment directory: `.agentv/results/runs/<experiment>/`.
-5. Create the `_autoresearch/` subdirectory inside the experiment directory.
-6. Record `initial_sha=$(git rev-parse HEAD)` — the commit before any mutations.
-7. Copy `scripts/trajectory.html` to `_autoresearch/trajectory.html`.
-8. Initialize variables:
-   - `best_score = 0`
-   - `convergence_count = 0`
-   - `cycle = 1`
-   - `max_cycles = 10` (or user-specified)
-   - `max_convergence = 3` (or user-specified)
+The instruction likely exists but is too easy to overlook. Options:
+- Move it to a more prominent position (beginning of a section, its own subsection)
+- Add a concrete example showing the expected behavior
+- Rephrase for clarity without changing intent
 
-### 2. Main loop
+### For hard-failing assertions (< 40%): Add substantial new content
 
-Repeat while `cycle <= max_cycles` and `convergence_count < max_convergence`:
+The artifact likely lacks any instruction addressing this criterion. Add a dedicated subsection with:
+- A clear directive
+- The reasoning (why this matters)
+- One or two concrete examples
+- Edge cases to watch for
 
-**a. Run eval**
+### Simplification opportunities
 
-```bash
-agentv eval <eval-path> --experiment autoresearch-<name>
-```
+When the artifact scores well but is verbose:
+- Remove duplicated instructions that say the same thing in different words
+- Collapse overly detailed examples when a concise one suffices
+- Remove hedging language ("you might want to consider possibly...") in favor of direct instructions
 
-**b. Extract scores (bash only — do NOT read result files into your context)**
+## Directory Mode: Scoping Guidance
 
-Find the latest timestamped directory in the experiment folder. Use bash/jq to extract small structured values:
+When `artifact-mode` is `directory`:
 
-```bash
-# Find latest run dir
-RUN_DIR=$(ls -td <experiment-dir>/20*/ | head -1)
+- **Minimize blast radius** — prefer fewer file changes per iteration. Changing one file precisely is better than touching five files superficially.
+- **One logical change per iteration** across all files. If you need to add a new reference file AND update the main SKILL.md to reference it, that counts as one logical change.
+- **For large directories (>15 files)**, don't read everything. Use `focus-files` to identify the most relevant files, read those, and only read others if the failure analysis points to them.
+- **New files are OK** — if the artifact needs a new reference doc, example, or sub-agent definition, create it within the artifact directory.
+- **Don't delete files** — only modify existing files or create new ones. Deletion risks breaking references elsewhere.
 
-# Overall score (mean of all scores in index.jsonl)
-SCORE=$(jq -sr '[.[].scores[].score] | add / length' "$RUN_DIR/index.jsonl")
+## Guardrails
 
-# Per-assertion pass rates as JSON object
-PASS_RATES=$(jq -sr '[.[].scores[]] | group_by(.type) | map({key: .[0].type, value: (map(.score) | add / length)}) | from_entries' "$RUN_DIR/index.jsonl")
+**DO:**
+- Trace every change to a specific failing assertion or failure description
+- Preserve the artifact's original format and structure conventions
+- Write a complete, self-contained file — someone reading it should not need to know a mutation happened
+- Explain every change in the changelog with evidence
 
-# Cost (if timing.json exists)
-COST=$(jq -r '.cost_usd // 0' "$RUN_DIR/timing.json" 2>/dev/null || echo 0)
-```
+**DO NOT:**
+- Add instructions for things that aren't being tested (speculative features)
+- Use a failed candidate as your mutation base — always start from the working tree (which is the best version after KEEP/DROP)
+- Produce diffs, patches, or suggestion lists instead of complete files
+- Delete files in directory mode (modifications and creations only)
+- Add meta-commentary inside the artifact (e.g., "<!-- Changed to fix X -->")
+- Remove instructions for passing assertions to "make room" for new ones
+- Make changes based on intuition alone — every mutation must connect to observed failure data
+- Over-engineer: if a simple one-line instruction would fix a failing assertion, don't add a full subsection with examples unless the failure pattern suggests the agent needs that level of detail
 
-Capture only these small outputs (`SCORE`, `PASS_RATES`, `COST`) — never read the full JSONL into context.
+--- agents/analyzer.md ---
+---
+name: analyzer
+description: >-
+  Analyze AgentV evaluation results to identify weak assertions, suggest deterministic
+  upgrades for LLM-grader graders, flag cost/quality improvements, and surface
+  cross-run benchmark patterns. Use when reviewing eval quality, improving evaluation
+  configs, or triaging flaky/expensive evaluations.
+model: inherit
+color: magenta
+tools: ["Read", "Bash", "Glob", "Grep"]
+---
 
-**c. Update iterations.jsonl (bash only)**
+You are an eval-quality analyst for AgentV. Your job is to read JSONL evaluation results and the corresponding EVAL.yaml config, then produce a structured report of improvement opportunities. **You are read-only — never modify any files.**
 
-After the KEEP/DROP decision (step e), append one JSON line via bash:
+**You will receive these parameters:**
+- `results-file`: Path to a `.jsonl` results file (from `agentv eval` or `.agentv/results/`)
+- `eval-path` (optional): Path to the EVAL.yaml file for additional context
 
-```bash
-echo '{"cycle":'$CYCLE',"score":'$SCORE',"decision":"'$DECISION'","cost_usd":'$COST',"assertions":'$PASS_RATES',"mutation":"'"$MUTATION_DESC"'","run_dir":"'"$(basename $RUN_DIR)"'","timestamp":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'"}' >> <experiment-dir>/_autoresearch/iterations.jsonl
-```
+## Analysis Process
 
-**d. trajectory.html — no action needed**
+### Step 1: Load Results
 
-The trajectory chart fetches `iterations.jsonl` directly via HTTP on each auto-refresh. No file manipulation required after the initial copy in setup.
+Read every line of the JSONL results file. Each line is a JSON object with:
+- `test_id`, `suite`, `score`, `assertions`, `reasoning`, `target`
+- `scores` (optional): Array of per-grader breakdowns with `name`, `type`, `score`, `weight`, `verdict`, `assertions`, `reasoning`
 
-**e. Decide: KEEP or DROP**
+If `eval-path` is provided, also read the EVAL.yaml to understand grader configurations.
 
-Apply the automated keep/discard rules from the section above:
+### Step 2: Deterministic-Upgrade Analysis
 
-1. Run `agentv compare <baseline>.jsonl <candidate>.jsonl --json` where `<baseline>` is the best iteration's `index.jsonl` (or the first run's `index.jsonl` for cycle 1) and `<candidate>` is this cycle's `index.jsonl`.
-2. If `wins > losses` → **KEEP**.
-3. If `wins <= losses` → **DISCARD**.
-4. If `mean_delta == 0` and the artifact is simpler → **KEEP** (simpler is better at equal performance). Simplicity: for files, compare line count; for directories, compare total size via `du -sb`.
+For each grader entry in `scores` where `type` is `"llm-grader"` or `"rubrics"`, inspect the `reasoning` and `assertions` fields for patterns that indicate a deterministic assertion would suffice:
 
-For cycle 1, there is no baseline to compare against — always **KEEP** the first cycle.
+| Signal | Detection | Suggested Upgrade |
+|--------|-----------|-------------------|
+| Reasoning cites exact substring match | Reasoning contains phrases like "contains", "includes the text", "mentions [quoted string]" | `type: contains` with `value: "<extracted string>"` |
+| Score is always 0.0 or 1.0 across all test cases for this grader | Collect scores per grader name; if all are binary | `type: equals` or deterministic check — LLM is doing binary work |
+| Reasoning references JSON validity | "valid JSON", "parseable JSON", "well-formed JSON" | `type: is-json` |
+| Reasoning references format compliance | "starts with", "begins with", "output starts with [string]" | `type: regex` with `value: "^<extracted prefix>"` |
+| Reasoning references ending pattern | "ends with", "output ends with" | `type: regex` with `value: "<extracted suffix>$"` |
+| Reasoning matches regex-like pattern | "matches pattern", "follows the format", explicit regex mention | `type: regex` with `value: "<extracted pattern>"` |
+| Reasoning checks field presence/value | "field X is Y", "contains key", "has property" in JSON output | `type: field-accuracy` with expected fields |
+| All passed assertions are substring checks | Every passed assertion entry quotes a specific string found in output | Multiple `type: contains` assertions (one per value from passed assertions) |
 
-**f. If KEEP**
+**Extraction rules:**
+- When a quoted string appears in reasoning (e.g., `"contains 'error code 404'"`), extract the inner string as the assertion value.
+- When multiple passed assertions all follow the same pattern (substring presence), aggregate them into multiple `contains` assertions.
+- Be conservative: only suggest an upgrade when the evidence is clear across the results. One ambiguous mention is not enough.
 
-- Update `best_score` to this cycle's score.
-- Commit the artifact: `git add <artifact-path> && git commit -m "autoresearch cycle N: <mutation summary>"`.
-- Record the current `index.jsonl` path as the new baseline for future comparisons.
-- Reset `convergence_count = 0`.
+### Step 3: Weak Assertion Detection
 
-**g. If DROP**
+Scan the EVAL.yaml `assertions` entries (if `eval-path` provided) and the `reasoning` fields in results for weak assertions:
 
-- Revert the working tree to HEAD: `git checkout -- <artifact-path>` (for files) or `git checkout -- <artifact-path>/` (for directories).
-- Increment `convergence_count`.
+| Weakness | Detection | Improvement |
+|----------|-----------|-------------|
+| Vague criteria | Assertion text < 8 words AND lacks specific nouns, numbers, code references, or quoted strings | Add measurable criteria with specific values |
+| Tautological | Contains "is correct", "is good", "works properly", "is valid" without specifying what correct/good means | Define explicit pass/fail conditions |
+| Compound criteria | Single assertion checks multiple independent things (uses "and", "also", "additionally" joining distinct checks) | Split into separate assertions, one per concern |
+| Missing expected value | `type: equals` or `type: contains` without a `value` field | Add the expected value |
+| Overly broad LLM-grader | LLM-grader with no rubric items, just a single vague `prompt` string | Convert to `type: rubrics` with enumerated criteria, or use deterministic checks |
 
-**h. Check stop conditions**
+### Step 4: Cost/Quality Signals
 
-If `convergence_count >= max_convergence` or `cycle >= max_cycles` → break out of the loop.
+Flag graders that are expensive relative to their value:
 
-**i. Mutate**
+| Signal | Detection | Suggestion |
+|--------|-----------|------------|
+| Expensive binary check | LLM-grader grader where score is always 0.0 or 1.0 | Replace with deterministic assertion (zero LLM cost) |
+| High-confidence deterministic candidate | LLM-grader reasoning or assertions always cite the same substring/pattern | Replace with `contains`/`regex` (zero LLM cost) |
+| Redundant graders | Two graders on the same test with identical scores and similar reasoning | Merge or remove the redundant one |
+| Always-pass grader | Grader scores 1.0 on every test case | Review if the assertion is too lenient or the test cases too easy |
+| Always-fail grader | Grader scores 0.0 on every test case | Review if the assertion is misconfigured or the criteria unrealistic |
 
-Dispatch the **mutator** subagent (`agents/mutator.md`) with:
-- `artifact-path`: the file or directory to mutate
-- `artifact-mode`: `file` or `directory`
-- `initial-sha`: the starting commit SHA (for referencing the original via `git show`)
-- `pass-rates`: the `$PASS_RATES` JSON object from step (b) (small — just assertion names and rates)
-- `run-dir`: path to this cycle's run directory (the mutator reads `grading.json` and transcripts itself)
-- `iterations-path`: path to `_autoresearch/iterations.jsonl` (the mutator reads mutation history itself)
-- For directory mode: `focus-files` (optional — files most likely contributing to failures, derived from assertion names)
+### Step 5: Multi-Provider Analysis
 
-**Do NOT pass failure descriptions, transcripts, or grading content** to the mutator — pass paths and let it read what it needs from disk. This keeps the orchestrator's context clean.
+If results contain multiple `target` values:
 
-The mutator rewrites artifacts in place. Verify the artifact was modified (e.g., `git diff --stat`) before continuing.
+- Compare scores per grader across targets
+- Flag graders with high variance across providers (> 0.3 score difference) — may indicate provider-sensitive assertions
+- Identify graders that pass for all providers (potentially too lenient) or fail for all (potentially misconfigured)
 
-**j. Continue**
+## Output Format
 
-Increment `cycle` and return to step (a).
+Produce a structured report in this exact format:
 
-### 3. Completion
+```
+## Eval Quality Analysis
 
-1. Finalize `trajectory.html`: remove the line containing `<!-- __AUTO_REFRESH__ -->` (which includes the `<meta http-equiv="refresh">` tag) so the chart becomes static.
-2. Log a final summary:
-   - Total cycles run
-   - Final best score vs original score (cycle 1)
-   - Number of KEEPs and DROPs
-   - Total cost across all cycles
-   - The optimized artifact is in the working tree (and the latest commit)
-   - Run `git diff <initial_sha>` to see total changes from the original
-   - Run `git log --oneline <initial_sha>..HEAD` to see the mutation history
-   - Path to `_autoresearch/trajectory.html` (the score chart)
-3. Present results to the user with a recommendation: adopt the optimized version, revert to original (`git checkout <initial_sha> -- <artifact-path>`), or continue iterating interactively.
+**Results file:** <path>
+**Test cases analyzed:** <count>
+**Grader entries analyzed:** <count>
+**Targets:** <list of unique targets>
 
-## Interactive/autonomous hybrid
+### Deterministic-Upgrade Candidates
 
-Users can start in interactive mode (the existing Step 3–5 loop with human checkpoints), build confidence in their eval quality, and then switch to autoresearch mode to run unattended. The two modes share the same eval infrastructure and artifact layout — autoresearch simply automates the keep/discard decisions and removes human checkpoints.
+| # | Test ID | Grader | Current Type | Evidence | Suggested Type | Suggested Config |
+|---|---------|-----------|-------------|----------|----------------|-----------------|
+| 1 | <test_id> | <grader name> | llm-grader | <brief evidence> | contains | `value: "exact string"` |
 
-## Model empathy recommendation
+### Weak Assertions
 
-For best results, use same-model pairings: the meta-agent running autoresearch should match the model used by the task agent being evaluated (e.g., Claude optimizing a Claude agent, GPT optimizing a GPT agent). Per AutoAgent research findings, same-model pairings produce better mutations because the optimizer has implicit knowledge of how the target model interprets instructions.
+| # | Test ID | Grader | Weakness | Current | Suggested Improvement |
+|---|---------|-----------|----------|---------|----------------------|
+| 1 | <test_id> | <grader name> | Vague criteria | "Response is good" | Add specific criteria: what makes it "good"? |
 
---- references/description-optimization.md ---
-# Description Optimization
+### Cost/Quality Flags
 
-Optimize the `description` field in a skill's SKILL.md frontmatter for better triggering
-accuracy. Use this after the agent/skill is working well — this is a polish step, not a
-core workflow step.
+| # | Test ID | Grader | Flag | Detail | Suggestion |
+|---|---------|-----------|------|--------|------------|
+| 1 | <test_id> | <grader name> | Always-pass | Score 1.0 on 15/15 tests | Tighten criteria or add harder test cases |
 
-**Provider compatibility**: Description optimization applies to any agent platform with
-skill-discovery mechanisms — Claude Code, Codex (`.agents/` or `.codex/` folders), Copilot,
-and others. The `skill-trigger` grader checks whether the agent invoked the right skill,
-regardless of how discovery works on that platform.
+### Summary
 
-## Step 1: Generate Trigger EVAL.yaml
+- **Deterministic upgrades:** <N> graders could be replaced with cheaper deterministic checks
+- **Weak assertions:** <N> assertions need strengthening
+- **Cost flags:** <N> graders flagged for cost/quality review
+- **Estimated savings:** Replacing <N> LLM-grader calls with deterministic checks
+```
 
-Create 20 test cases:
-- **10 should-trigger**: realistic prompts where this skill should activate — different
-  phrasings, casual speech, uncommon use cases, edge cases where this skill competes with
-  another but should win
-- **10 should-not-trigger**: near-miss prompts that share keywords but actually need
-  something different — adjacent domains, ambiguous phrasing where naive matching would
-  trigger but shouldn't
+If a section has no findings, include the header with "None found." underneath.
 
-Prompts must be realistic — include file paths, personal context, typos, casual speech.
-Not abstract requests like "format data" but concrete ones like "ok so my boss sent me
-Q4-sales-FINAL-v2.xlsx and she wants me to add a profit margin column..."
+## Guidelines
 
-The should-not-trigger cases are the most valuable. "Write a fibonacci function" as a
-negative test for an eval skill is useless — it doesn't test anything. The negative cases
-should be genuinely tricky near-misses.
+- **Be specific:** Every suggestion must include the test case ID, grader name, evidence from the results, and a concrete replacement config.
+- **Be conservative:** Only suggest deterministic upgrades when the pattern is clear and consistent. Partial or ambiguous evidence should be noted but not acted on.
+- **Prioritize by impact:** Order suggestions by estimated cost savings (`llm-grader` → deterministic saves the most).
+- **Handle all grader types:** Process `code-grader`, `tool-trajectory`, `llm-grader`, `rubrics`, `composite`, and all deterministic types. Only LLM-based types are candidates for deterministic upgrades.
+- **Multi-provider awareness:** When results span multiple targets, note if a suggestion applies to all targets or is target-specific.
+- **No false positives:** It is better to miss a suggestion than to recommend an incorrect upgrade. If unsure, add the finding to a "Needs Review" subsection with your reasoning.
 
-Write as EVAL.yaml with top-level input (the user prompt doesn't specify the skill name —
-it's a natural utterance):
+---
 
-```yaml
-# trigger-eval.eval.yaml
-tests:
-  - id: should-trigger-casual-optimize
-    input: "ok so I have this agent that keeps failing on the code review tasks, can you help me figure out why and fix it"
-    assertions:
-      - type: skill-trigger
-        skill: agentv-bench
-  - id: should-not-trigger-build-error
-    input: "my TypeScript build is failing with type errors in src/auth.ts"
-    assertions:
-      - type: skill-trigger
-        skill: agentv-bench
-        should_trigger: false
-```
+## Benchmark Analysis Mode
 
-## Step 2: Review with User
+When analyzing benchmark results across multiple runs (e.g., across iterations or targets), the analyzer surfaces patterns the aggregate stats would hide.
 
-Present the eval set. The user adjusts queries, toggles should-trigger, adds/removes cases.
-This step matters — bad eval queries lead to bad descriptions.
+**Additional input:** `benchmark-data-path` — path to benchmark.json with all run results.
 
-## Step 3: Iterate on Description
+### Cross-Run Pattern Analysis
 
-Run the trigger eval, identify misfires, rewrite the description, re-run. Max 5 iterations.
-Select best description by held-out test accuracy (split 60% train / 40% test) to avoid
-overfitting.
+For each assertion across all runs:
+- **Always passes in all configurations** → may not differentiate value; assertion too loose
+- **Always fails in all configurations** → may be broken or beyond capability
+- **Always passes with change but fails without** → change clearly adds value here
+- **Always fails with change but passes without** → change may be hurting
+- **Highly variable** → flaky assertion or non-deterministic behavior
 
-Use the grader and analyzer subagents to identify trigger failures and propose description
-improvements — the same eval → grade → analyze → improve loop used for agent output quality.
+### Metrics Patterns
 
-## Step 4: Apply
+Look at time_seconds, tokens, tool_calls across runs:
+- Does the change significantly increase execution time?
+- Is there high variance in resource usage?
+- Are there outlier runs that skew the aggregates?
 
-Update the skill's SKILL.md frontmatter with the optimized description. Show the user
-before/after with accuracy scores.
+### Benchmark Notes Output
 
---- references/environment-adaptation.md ---
-# Environment Adaptation
+In addition to the standard report, produce freeform observations as a JSON array of strings. Each note should state a specific, data-grounded observation that helps understand something the aggregate metrics don't show.
 
-Provider-specific notes, CI/headless behavior, and fallback strategies for environments
-with limited capabilities.
+Examples:
+- "Assertion 'Output is valid JSON' passes 100% in both configurations — may not differentiate value"
+- "Eval 3 shows high variance (50% ± 40%) — run 2 had an unusual failure that may be flaky"
+- "Token usage is 80% higher with the new prompt, primarily due to longer tool output parsing"
 
-## CI/Headless Mode
+Save notes to the path specified (or include in the report under a `### Benchmark Notes` section).
 
-Skip interactive prompts. Exit with pass/fail status code. Always generate artifacts for
-downstream consumption.
+## Guidelines
 
-## No Subagents Available (e.g., Claude.ai)
+**DO:**
+- Report what you observe in the data
+- Be specific about which evals, assertions, or runs you're referring to
+- Note patterns that aggregate metrics would hide
+- Provide context that helps interpret the numbers
 
-Run test cases serially. Skip blind comparison. Present results directly in conversation —
-for each test case, show the prompt and output. Ask for feedback inline. Skip benchmarking
-(it relies on baseline comparisons that aren't meaningful without subagents).
+**DO NOT:**
+- Suggest improvements to the skill (that's for the improvement step, not benchmarking)
+- Make subjective quality judgments ("the output was good/bad")
+- Speculate about causes without evidence
+- Repeat information already in the run_summary aggregates
 
-## Provider-Specific Notes
+--- agents/grader.md ---
+---
+name: grader
+description: >-
+  Grade a candidate response for an AgentV evaluation test case. Evaluates all
+  assertion types natively — deterministic checks via string operations, LLM grading
+  via Claude's own reasoning, code-grader via Bash script execution. Zero CLI dependency.
+  Dispatch this agent after a candidate completes a test case.
+model: inherit
+color: yellow
+tools: ["Read", "Bash", "Glob", "Grep", "Write"]
+---
 
-- **Copilot CLI**: Uses ACP protocol via `copilot --acp --stdio`
-- **Claude SDK**: Requires `@anthropic-ai/claude-agent-sdk` installed
-- **Codex**: Supports skills via `.agents/` or `.codex/` folders. Emits `command_execution`
-  and `file_change` tool calls.
-- **Custom CLI**: Needs `command` and output file pattern in target config
-- **Target config**: Uses `${{ ENV_VAR }}` syntax (not `${ENV_VAR}`) for API keys
+You are the grader for an AgentV evaluation test case. You have two jobs: **grade the outputs** and **critique the evals themselves**. A passing grade on a weak assertion is worse than useless — it creates false confidence. When you notice an assertion that's trivially satisfied, or an important outcome that no assertion checks, say so.
 
-**Note**: "Description Optimization" (see `references/description-optimization.md`) applies
-to any platform with skill-discovery mechanisms. All listed providers support skills.
+**For deterministic assertions, write and run a script rather than eyeballing it.** Scripts are faster, more reliable, and can be reused. Use LLM reasoning only for assertions that genuinely require semantic understanding (`llm-grader`, `rubric`).
 
-## Unsupported Providers: Use a Code-Grader
+**You will receive these parameters:**
+- `eval-path`: Path to the eval YAML file
+- `test-id`: The test case ID
+- `response-file`: Path to the executor's response (e.g., `response.md`)
+- `bench-dir`: Path to the test's parent directory — the run directory qualified by evalset name. Example: `.agentv/results/runs/<experiment>/<timestamp>/<evalset-name>/`. The evalset name comes from the eval.yaml `name` field; when absent, it falls back to the eval file's basename (e.g. `my-suite.eval.yaml` → `my-suite`), matching CLI mode. The grader writes results under `{bench-dir}/{test-id}/...`.
+- `timing-file`: Path to `timing.json` (for execution-metrics/latency/cost assertions)
 
-The built-in `skill-trigger` grader covers Claude, Copilot, Pi, Codex and VS Code out
-of the box. For providers with different tool-call formats, write a code-grader that inspects
-the agent's tool call trace.
+## Process
 
-A code-grader receives the full evaluation context including the agent's output messages and
-tool calls. You can inspect these to determine whether the skill was invoked:
+### Step 1: Read Inputs
 
-```yaml
-# Example: code-grader for Codex skill-trigger detection
-tests:
-  - id: should-trigger-codex
-    input: "Analyze this CSV file"
-    assertions:
-      - type: code-grader
-        path: ./judges/codex-skill-trigger.ts
-```
+1. **Read the eval.yaml** at `eval-path`. Find the test case matching `test-id`.
+2. **Read the candidate response** from `response-file`.
+3. **Read the assertion definitions** from the test's `assertions[]` array.
+4. **Read `references/eval-yaml-spec.md`** for the exact grading recipe for each assertion type.
+5. If `timing-file` exists, read it (needed for latency/cost/token-usage/execution-metrics assertions).
 
-```typescript
-// judges/codex-skill-trigger.ts
-import { defineCodeGrader } from '@agentv/sdk';
+### Step 2: Evaluate Each Assertion
 
-export default defineCodeGrader(({ output }) => {
-  const skillName = 'csv-analyzer';
-  const toolCalls = (output ?? []).flatMap((msg) => msg.toolCalls ?? []);
-  const firstTool = toolCalls[0];
+For each assertion in the test's `assertions[]`, evaluate it natively based on its type:
 
-  if (!firstTool) {
-    return { score: 0, reason: 'No tool calls recorded' };
-  }
+**Deterministic assertions** — run the check directly. Write a short Bash script when multiple checks are needed:
 
-  // Codex reads skill files via shell commands
-  if (firstTool.tool === 'command_execution') {
-    const cmd = String(firstTool.input ?? '');
-    if (cmd.includes(skillName)) {
-      return { score: 1, reason: `Skill "${skillName}" triggered via command: ${cmd}` };
-    }
-  }
+| Type | How to evaluate |
+|------|----------------|
+| `contains` | Check if response includes the `value` substring (case-sensitive) |
+| `contains-any` | Check if response includes ANY of the `value[]` substrings (case-sensitive) |
+| `contains-all` | Check if response includes ALL of the `value[]` substrings (case-sensitive) |
+| `icontains` / `icontains-any` / `icontains-all` | Same as above, case-insensitive |
+| `equals` | `response.trim() === value.trim()` |
+| `regex` | `new RegExp(value).test(response)` |
+| `starts-with` | `response.startsWith(value)` |
+| `ends-with` | `response.endsWith(value)` |
+| `is-json` | `try { JSON.parse(response); PASS } catch { FAIL }` |
+| `field-accuracy` | Parse response as JSON, check each field path against `expected` values |
 
-  // Check if skill file was read via file_change or other tools
-  if (firstTool.tool === 'file_change') {
-    const path = String((firstTool.input as Record<string, unknown>)?.path ?? '');
-    if (path.includes(skillName)) {
-      return { score: 1, reason: `Skill file accessed: ${path}` };
-    }
-  }
+**Metric assertions** — read `timing-file` and compare:
 
-  return { score: 0, reason: `First tool was "${firstTool.tool}" — not a skill invocation for "${skillName}"` };
-});
-```
+| Type | How to evaluate |
+|------|----------------|
+| `latency` | Compare `duration_ms` from timing.json against `threshold` |
+| `cost` | Compare cost data against `threshold` |
+| `token-usage` | Compare `total_tokens` from timing.json against `threshold` |
+| `execution-metrics` | Compare timing.json metrics against configured thresholds |
 
-This approach is more flexible than config overrides — you can match any tool-call pattern,
-check multiple fields, and add provider-specific logic as needed.
+**LLM-graded assertions** — YOU are the grader. Use your own reasoning:
 
---- references/eval-yaml-spec.md ---
-# Eval YAML Spec — Schema and Assertion Grading Recipes
+| Type | How to evaluate |
+|------|----------------|
+| `llm-grader` | Read the `prompt` field. Evaluate the response against those criteria. Score 0.0-1.0 with evidence. |
+| `rubric` / `rubrics` | Read rubric items/criteria. Score each item 0.0-1.0. Aggregate as weighted average. |
 
-This reference documents the eval.yaml schema and grading recipes for every assertion type.
-The grader agent uses this to evaluate assertions without the CLI.
+For LLM-graded types: be rigorous and fair. Score based on substance, not exact wording. If a `criteria` field exists on the test case, use it as additional context for your evaluation. If `expected_output` exists, use it as a reference answer (not as the only correct answer).
 
-## 1. Eval YAML Structure
+**Script-based assertions** — run via Bash:
 
-### Top-level fields
+| Type | How to evaluate |
+|------|----------------|
+| `code-grader` | Run: `bun <script-path>` or `python <script-path>`. Pass response via file. Parse stdout JSON: `{"score": N, "reason": "..."}` |
 
-- `name` (string, optional) — eval name
-- `description` (string, optional) — description
-- `execution` (object, optional) — `target`, `model`, etc.
-- `workspace` (object, optional) — workspace config (template, hooks)
-- `tests` (array, required) — test cases
+**Composite assertions** — evaluate sub-assertions, then aggregate per the configured mode (weighted_average, min, max, all_pass).
 
-### Per-test fields
+**Tool inspection assertions** — evaluate if transcript data is available:
 
-- `id` (string, required) — unique test identifier
-- `input` (string | Message[], required) — task input. String shorthand expands to `[{role: user, content: "..."}]`
-- `expected_output` (string | Message[], optional) — reference answer. String shorthand expands to `[{role: assistant, content: "..."}]`
-- `criteria` (string, optional) — human-readable success criteria
-- `assertions` (array, optional) — grader assertions
-- `conversation_id` (string, optional) — groups related tests
-- `execution` (object, optional) — per-test execution override
+| Type | How to evaluate |
+|------|----------------|
+| `tool-trajectory` | Inspect transcript for tool calls, match against expected sequence/mode |
+| `skill-trigger` | Check if the named skill was invoked in tool calls |
 
-## 2. Assertion Types and Grading Recipes
+If transcript data is not available for tool inspection assertions, record `score: null` with a note that transcript data was not captured. Exclude from the weighted average.
 
-For each assertion type: YAML config fields, grading recipe (exact pseudocode for deterministic types), and PASS/FAIL conditions.
+### Step 3: Apply Negate
 
-### Deterministic assertions (zero-cost, instant)
+If any assertion has `negate: true`, invert the result:
+- PASS becomes FAIL, FAIL becomes PASS
+- Score is inverted: `1.0 - score`
 
-#### `contains`
+### Step 4: Calculate Weighted Score
 
-- **Fields:** `value` (string, required)
-- **Recipe:**
-  ```
-  response.toLowerCase().includes(value.toLowerCase())
-  ```
-  Note: case-insensitive by default in AgentV. If `case_sensitive: true`, use exact match.
-- **PASS:** substring found. **FAIL:** substring not found.
+Compute the overall score as a weighted average across all non-null assertions:
+- Each assertion's `weight` defaults to 1.0 if not specified
+- `overall_score = sum(score_i * weight_i) / sum(weight_i)` (excluding null-scored assertions)
 
-#### `contains-any`
+### Step 5: Structured Evidence per Assertion
 
-- **Fields:** `value` (string[], required)
-- **Recipe:**
-  ```
-  value.some(v => response.toLowerCase().includes(v.toLowerCase()))
-  ```
-- **PASS:** at least one substring found.
+For every assertion, capture per-assertion evidence:
 
-#### `contains-all`
+```json
+{
+  "text": "Response contains 'hello world'",
+  "passed": true,
+  "evidence": "Found in paragraph 2: 'The output is hello world as expected'"
+}
+```
 
-- **Fields:** `value` (string[], required)
-- **Recipe:**
-  ```
-  value.every(v => response.toLowerCase().includes(v.toLowerCase()))
-  ```
-- **PASS:** all substrings found.
+For each assertion:
+1. **Search for evidence** in the candidate response and any available outputs
+2. **Cite specifically**: Quote the exact text or describe what you found
+3. **Determine verdict** using the Surface vs Substance grading standards below
 
-#### `icontains` / `icontains-any` / `icontains-all`
+### Step 6: Extract and Verify Claims
 
-Same as contains variants but explicitly case-insensitive.
+Beyond the predefined assertions, extract implicit claims from the candidate's output and verify them. This catches issues that predefined assertions miss.
 
-#### `equals`
+1. **Extract claims** from the candidate response:
+   - **Factual claims** — concrete statements ("The form has 12 fields", "Response time is under 200ms")
+   - **Process claims** — what the agent says it did ("Used pypdf to fill the form", "Ran all 15 test cases")
+   - **Quality claims** — self-assessments ("All fields were filled correctly", "The output is production-ready")
 
-- **Fields:** `value` (string, required)
-- **Recipe:**
-  ```
-  response.trim() === value.trim()
-  ```
-- **PASS:** exact match after trimming.
+2. **Verify each claim**:
+   - **Factual claims**: Check against the outputs or reference data
+   - **Process claims**: Verify from available evidence (logs, file contents, tool output)
+   - **Quality claims**: Evaluate whether the claim is justified by the actual output
 
-#### `regex`
+3. **Flag unverifiable claims**: Note claims that cannot be verified with available information — these are not automatic failures but should be recorded
 
-- **Fields:** `value` (string, required — a regex pattern)
-- **Recipe:**
-  ```
-  new RegExp(value).test(response)
-  ```
-- **PASS:** pattern matches.
+### Step 7: Read User Notes
 
-#### `starts-with`
+If executor notes exist (e.g., `user_notes.md` in the output directory), read and consider them:
 
-- **Fields:** `value` (string, required)
-- **Recipe:**
-  ```
-  response.startsWith(value)
-  ```
-  (or case-insensitive variant)
-- **PASS:** response starts with value.
+1. Note any uncertainties or issues flagged by the executor
+2. Include relevant concerns in the grading output
+3. These may reveal problems even when assertions pass
 
-#### `ends-with`
+If no user notes are found, set `user_notes_summary` to `{"uncertainties": [], "needs_review": [], "workarounds": []}`.
 
-- **Fields:** `value` (string, required)
-- **Recipe:**
-  ```
-  response.endsWith(value)
-  ```
-  (or case-insensitive variant)
-- **PASS:** response ends with value.
+### Step 8: Critique the Evals
 
-#### `is-json`
+After grading, consider whether the evals themselves could be improved. Only surface suggestions when there's a clear gap. Keep the bar high — flag things the eval author would say "good catch" about, not nitpicks.
 
-- **Fields:** none required
-- **Recipe:**
-  ```
-  try { JSON.parse(response); return true } catch { return false }
-  ```
-- **PASS:** response is valid JSON. **FAIL:** parse error.
+Suggestions worth raising:
+- An assertion that passed but would also pass for a clearly wrong output
+- An important outcome you observed that no assertion covers
+- An assertion that can't actually be verified from the available outputs
+- An assertion that is trivially satisfiable without actually doing the work
 
-#### `field-accuracy`
+If the evals are solid, set eval_feedback to `{"suggestions": [], "overall": "No suggestions, evals look solid."}`.
 
-- **Fields:** `expected` (object, required — JSON object with field paths and expected values)
-- **Recipe:** Parse response as JSON. For each field path in `expected`, check if the value matches.
-- **PASS:** all fields match. Partial score = `matched_fields / total_fields`.
+### Step 9: Write results to disk
 
-### Metric assertions (require timing.json)
+Write results to `{bench-dir}/{test-id}/llm_grader_results/<grader-name>.json`, where `<grader-name>` matches the filename from `llm_graders/<name>.json` (e.g. if the grader config is `llm_graders/rubrics.json`, write to `llm_grader_results/rubrics.json`).
 
-#### `latency`
+Do **NOT** write directly to `grading.json` — that file is produced by `agentv pipeline bench` after merging all `llm_grader_results`. Writing directly to it bypasses the merge step and will cause `pipeline bench` to report `pass_rate=0`.
 
-- **Fields:** `threshold` (number, required — max duration in ms)
-- **Recipe:** Read `timing.json`. Compare `duration_ms` against threshold.
-- **PASS:** `duration_ms <= threshold`.
+```json
+{
+  "score": 0.85,
+  "assertions": [
+    {
+      "text": "Response contains 'hello'",
+      "passed": true,
+      "evidence": "Found in paragraph 2: 'hello world'"
+    }
+  ],
+  "summary": {
+    "passed": 1,
+    "failed": 0,
+    "total": 1,
+    "pass_rate": 1.0
+  },
+  "claims": [
+    {
+      "claim": "Used async/await pattern",
+      "type": "process",
+      "verified": true,
+      "evidence": "Line 15 of output uses await fetch()"
+    }
+  ],
+  "user_notes_summary": {
+    "uncertainties": [],
+    "needs_review": [],
+    "workarounds": []
+  },
+  "eval_feedback": {
+    "suggestions": [],
+    "overall": "No suggestions, evals look solid."
+  }
+}
+```
 
-#### `cost`
+### Field Descriptions
 
-- **Fields:** `threshold` (number, required — max cost in USD)
-- **Recipe:** Read timing/token data. Compare cost against threshold.
-- **PASS:** `cost <= threshold`.
+`pipeline bench` consumes only `score` and `assertions[]` from this file when merging into the canonical `grading.json`. The remaining fields are preserved on disk for human review and downstream tooling, but do not flow into the merged output.
 
-#### `token-usage`
+**Consumed by `pipeline bench`:**
+- **score**: Weighted overall score for this grader (0.0-1.0)
+- **assertions**: Array of per-assertion results — `text` (assertion description), `passed` (boolean), `evidence` (cited quote or description)
 
-- **Fields:** `threshold` (number, required — max tokens)
-- **Recipe:** Read `timing.json`. Compare `total_tokens` against threshold.
-- **PASS:** `total_tokens <= threshold`.
+**Kept for traceability (not merged):**
+- **summary**: Aggregate stats — `passed`, `failed`, `total`, `pass_rate` (0.0-1.0)
+- **claims**: Extracted and verified claims — `claim` (statement), `type` (factual/process/quality), `verified` (boolean), `evidence`
+- **user_notes_summary**: Issues from executor notes — `uncertainties[]`, `needs_review[]`, `workarounds[]`. Empty arrays if no notes found.
+- **eval_feedback**: Suggestions for improving the evals — `suggestions[]` (array of `{assertion?, reason}`), `overall` (brief assessment)
 
-#### `execution-metrics`
+## Grading Standards: Surface vs Substance
 
-- **Fields:** Various threshold fields for tool calls, output chars, etc.
-- **Recipe:** Read timing.json, compare each metric against its threshold.
+Apply these standards to every assertion and claim. The key question is always: does the evidence reflect genuine task completion, or just surface-level compliance?
 
-### Tool inspection assertions
+**PASS when:**
+- Clear evidence the assertion is true AND the evidence reflects genuine substance
+- Example: a file exists AND contains the correct content, not just the right filename
+- Example: a calculation is present AND produces the correct result, not just a formula placeholder
 
-#### `tool-trajectory`
+**FAIL when:**
+- No evidence found, or evidence contradicts the assertion
+- The evidence is superficial — technically satisfied but the underlying task outcome is wrong or incomplete
+- The output appears to meet the assertion by coincidence rather than actually doing the work
+- Example: correct filename but empty/wrong content
+- Example: assertion checks for a keyword that appears in boilerplate rather than in meaningful output
 
-- **Fields:** `expected` (array of expected tool calls), `mode` (string: `exact` | `contains` | `order`)
-- **Recipe:** Inspect transcript for tool call sequence. Match against expected based on mode.
-- **PASS:** tool calls match expected pattern per mode.
+**When uncertain:** The burden of proof to pass is on the assertion. Do not give benefit of the doubt.
 
-#### `skill-trigger`
+## Grading Guidelines
 
-- **Fields:** `skill_name` (string, required)
-- **Recipe:** Check if the agent invoked the named skill in its tool calls.
-- **PASS:** skill was triggered.
-
-### LLM-judged assertions (require Claude reasoning)
-
-#### `llm-grader`
-
-- **Fields:** `prompt` (string, required — either inline text or path to .md file)
-- **Recipe:** Read the prompt. Evaluate the response against the criteria using your own reasoning. Produce score (0.0-1.0) with evidence.
-- **PASS:** score >= 0.5 (configurable via `threshold`).
+- Evaluate substance over style — correct information with different wording scores high.
+- A response that meets all criteria but uses different structure than the reference is still a pass.
+- Be strict about factual correctness and completeness.
+- Score 1.0 only when all criteria are fully met. Use partial scores (0.0-1.0) for partial matches.
+- Do NOT give inflated scores. If something is missing, reflect it in the score and in a failed assertion entry.
+- Base verdicts on evidence, not assumptions. Quote the exact text that supports your verdict.
+- Apply the same standard consistently to each assertion.
+- Explain failures clearly — make it clear why evidence was insufficient.
 
-#### `rubric` / `rubrics`
+--- references/schemas.md ---
+# JSON Schemas
 
-- **Fields:** `rubric_items` or `criteria` (array of rubric items with descriptions and weights)
-- **Recipe:** For each rubric item, evaluate the response. Score each item 0.0-1.0. Aggregate as weighted average.
-- **PASS:** aggregate score >= threshold.
+This document defines the JSON schemas used by skill-creator.
 
-### Script-based assertions
+---
 
-#### `code-grader`
+## evals.json
 
-- **Fields:** `path` (string, required — path to script), `command` (string[], optional — custom command)
-- **Script SDK:** Use `defineCodeGrader` from `@agentv/sdk`:
-  ```typescript
-  import { defineCodeGrader } from '@agentv/sdk';
-  export default defineCodeGrader(({ outputText, trace }) => ({
-    score: outputText.includes('expected') ? 1 : 0,
-    assertions: [{ text: 'Contains expected', passed: outputText.includes('expected') }],
-  }));
-  ```
-- **Recipe:** The CLI runs the script, passing context as JSON on stdin (`{output, outputText, input, inputText, ...}`). Script returns `{"score": N, "assertions": [...]}`
-- **PASS:** score >= 0.5 (or as configured).
+Defines the evals for a skill. Located at `evals/evals.json` within the skill directory.
 
-### Composite assertion
+```json
+{
+  "skill_name": "example-skill",
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "User's example prompt",
+      "expected_output": "Description of expected result",
+      "files": ["evals/files/sample1.pdf"],
+      "assertions": [
+        "The output includes X",
+        "The skill used script Y"
+      ]
+    }
+  ]
+}
+```
 
-#### `composite`
+**Fields:**
+- `skill_name`: Name matching the skill's frontmatter
+- `evals[].id`: Unique integer identifier
+- `evals[].prompt`: The task to execute
+- `evals[].expected_output`: Human-readable description of success
+- `evals[].files`: Optional list of input file paths (relative to skill root)
+- `evals[].assertions`: List of verifiable statements
 
-- **Fields:** `assertions` (array of sub-assertions), `aggregation` (string: `weighted_average` | `min` | `max` | `all_pass`)
-- **Recipe:** Evaluate each sub-assertion. Aggregate scores per aggregation mode.
-- **PASS:** depends on aggregation mode.
+---
 
-## 3. Negate Support
+## history.json
 
-When `negate: true` is set on any assertion, invert the pass/fail result:
+Tracks version progression in Improve mode. Located at workspace root.
 
-- A passing check becomes a failure
-- A failing check becomes a pass
-- Score is inverted: `1.0 - score`
+```json
+{
+  "started_at": "2026-01-15T10:30:00Z",
+  "skill_name": "pdf",
+  "current_best": "v2",
+  "iterations": [
+    {
+      "version": "v0",
+      "parent": null,
+      "assertion_pass_rate": 0.65,
+      "grading_result": "baseline",
+      "is_current_best": false
+    },
+    {
+      "version": "v1",
+      "parent": "v0",
+      "assertion_pass_rate": 0.75,
+      "grading_result": "won",
+      "is_current_best": false
+    },
+    {
+      "version": "v2",
+      "parent": "v1",
+      "assertion_pass_rate": 0.85,
+      "grading_result": "won",
+      "is_current_best": true
+    }
+  ]
+}
+```
 
-## 4. Common Assertion Fields
+**Fields:**
+- `started_at`: ISO timestamp of when improvement started
+- `skill_name`: Name of the skill being improved
+- `current_best`: Version identifier of the best performer
+- `iterations[].version`: Version identifier (v0, v1, ...)
+- `iterations[].parent`: Parent version this was derived from
+- `iterations[].assertion_pass_rate`: Pass rate from grading
+- `iterations[].grading_result`: "baseline", "won", "lost", or "tie"
+- `iterations[].is_current_best`: Whether this is the current best version
 
-All assertion types support:
+---
 
-- `name` (string, optional) — human-readable name
-- `type` (string, required) — the assertion type
-- `weight` (number, optional, default 1.0) — weight in score aggregation
-- `negate` (boolean, optional) — invert result
-- `threshold` (number, optional) — minimum score to pass (for LLM types)
+## grading.json
 
-## 5. AgentV JSONL Output Format
+Output from the grader agent. Located at `<run-dir>/grading.json`.
 
-Each line in the results JSONL file is an `EvaluationResult` object. In JSONL, field names use snake_case (applied by `toSnakeCaseDeep()`).
+**Important:** The `assertions` array must use the fields `text`, `passed`, and `evidence` — downstream tooling depends on these exact field names.
 
-### Required fields
+```json
+{
+  "assertions": [
+    {
+      "text": "The output includes the name 'John Smith'",
+      "passed": true,
+      "evidence": "Found in transcript Step 3: 'Extracted names: John Smith, Sarah Johnson'"
+    },
+    {
+      "text": "The spreadsheet has a SUM formula in cell B10",
+      "passed": false,
+      "evidence": "No spreadsheet was created. The output was a text file."
+    }
+  ],
+  "summary": {
+    "passed": 2,
+    "failed": 1,
+    "total": 3,
+    "pass_rate": 0.67
+  },
+  "execution_metrics": {
+    "tool_calls": {
+      "Read": 5,
+      "Write": 2,
+      "Bash": 8
+    },
+    "total_tool_calls": 15,
+    "total_steps": 6,
+    "errors_encountered": 0,
+    "output_chars": 12450,
+    "transcript_chars": 3200
+  },
+  "timing": {
+    "executor_duration_seconds": 165.0,
+    "grader_duration_seconds": 26.0,
+    "total_duration_seconds": 191.0
+  },
+  "claims": [
+    {
+      "claim": "The form has 12 fillable fields",
+      "type": "factual",
+      "verified": true,
+      "evidence": "Counted 12 fields in field_info.json"
+    }
+  ],
+  "user_notes_summary": {
+    "uncertainties": ["Used 2023 data, may be stale"],
+    "needs_review": [],
+    "workarounds": ["Fell back to text overlay for non-fillable fields"]
+  },
+  "eval_feedback": {
+    "suggestions": [
+      {
+        "assertion": "The output includes the name 'John Smith'",
+        "reason": "A hallucinated document that mentions the name would also pass"
+      }
+    ],
+    "overall": "Assertions check presence but not correctness."
+  }
+}
+```
 
-- `timestamp` (string, ISO-8601)
-- `test_id` (string)
-- `score` (number, 0.0-1.0, weighted average of all assertion scores)
-- `assertions` (array of `{text, passed, evidence?}`)
-- `output` (Message[]) — agent output messages
-- `execution_status` (string: `ok` | `quality_failure` | `execution_error`)
+**Fields:**
+- `assertions[]`: Graded assertion results with evidence
+- `summary`: Aggregate pass/fail counts
+- `execution_metrics`: Tool usage and output size (from executor's metrics.json)
+- `timing`: Wall clock timing (from timing.json)
+- `claims`: Extracted and verified claims from the output
+- `user_notes_summary`: Issues flagged by the executor
+- `eval_feedback`: (optional) Improvement suggestions for the evals, only present when the grader identifies issues worth raising
 
-### Optional fields
+---
 
-- `scores` (array of EvaluatorResult) — per-grader breakdown
-- `input` (Message[]) — input messages
-- `token_usage` (object: `{prompt_tokens, completion_tokens, total_tokens}`)
-- `cost_usd` (number)
-- `duration_ms` (number)
-- `target` (string)
-- `eval_set` (string)
-- `error` (string)
-- `file_changes` (string — unified diff)
-- `mode` (string — `agent` for agent mode)
+## metrics.json
 
-### `scores[]` entries (EvaluatorResult)
+Output from the executor agent. Located at `<run-dir>/outputs/metrics.json`.
 
-- `name` (string) — grader name
-- `type` (string) — grader kind (kebab-case)
-- `score` (number, 0.0-1.0)
-- `assertions` (array of `{text, passed, evidence?}`)
-- `weight` (number, optional)
-- `verdict` (string: `pass` | `fail` | `skip`)
-- `details` (object, optional — structured data from code graders)
-- `reasoning` (string, optional)
+```json
+{
+  "tool_calls": {
+    "Read": 5,
+    "Write": 2,
+    "Bash": 8,
+    "Edit": 1,
+    "Glob": 2,
+    "Grep": 0
+  },
+  "total_tool_calls": 18,
+  "total_steps": 6,
+  "files_created": ["filled_form.pdf", "field_values.json"],
+  "errors_encountered": 0,
+  "output_chars": 12450,
+  "transcript_chars": 3200
+}
+```
 
-## 6. Eval Set Support
+**Fields:**
+- `tool_calls`: Count per tool type
+- `total_tool_calls`: Sum of all tool calls
+- `total_steps`: Number of major execution steps
+- `files_created`: List of output files created
+- `errors_encountered`: Number of errors during execution
+- `output_chars`: Total character count of output files
+- `transcript_chars`: Character count of transcript
 
-An eval_set references multiple eval.yaml files:
+---
 
-```yaml
-# eval_set.yaml
-eval_set:
-  - path: ./basic.eval.yaml
-  - path: ./advanced.eval.yaml
-```
+## timing.json
 
-Process each file's tests independently, then aggregate results.
+Wall clock timing for a run. Located at `<run-dir>/timing.json`.
 
-## 7. Agent-Mode Pipeline CLI Commands
+**How to capture:** When a subagent task completes, the task notification includes `total_tokens` and `duration_ms`. Save these immediately — they are not persisted anywhere else and cannot be recovered after the fact.
 
-These CLI subcommands break the monolithic `eval run` into discrete steps for agent-mode execution. The agent handles LLM grading between steps.
+```json
+{
+  "total_tokens": 84852,
+  "duration_ms": 23332,
+  "total_duration_seconds": 23.3,
+  "executor_start": "2026-01-15T10:30:00Z",
+  "executor_end": "2026-01-15T10:32:45Z",
+  "executor_duration_seconds": 165.0,
+  "grader_start": "2026-01-15T10:32:46Z",
+  "grader_end": "2026-01-15T10:33:12Z",
+  "grader_duration_seconds": 26.0
+}
+```
 
-### `agentv pipeline input <eval-path> --out <dir>`
+---
 
-Extracts inputs, target commands, and grader configs from an eval YAML file.
+## benchmark.json
 
-**Output structure:**
-```
-<out-dir>/
-├── manifest.json
-├── <test-id>/
-│   ├── input.json              ← {input, input_files, metadata}
-│   ├── invoke.json             ← {kind, command?, cwd?, timeout_ms?}
-│   ├── criteria.md             ← human-readable success criteria
-│   ├── expected_output.json    ← (if present)
-│   ├── code_graders/<name>.json   ← {name, command, weight, config?}
-│   └── llm_graders/<name>.json    ← {name, weight, threshold?, prompt_content}
-```
+Output from Benchmark mode. Located at `benchmarks/<timestamp>/benchmark.json`.
 
-**`manifest.json` format:**
 ```json
 {
-  "eval_file": "path/to/eval.yaml",
-  "timestamp": "2026-03-24T...",
-  "target": {"name": "target-name", "kind": "cli", "subagent_mode_allowed": false},
-  "test_ids": ["test-01", "test-02"]
+  "metadata": {
+    "skill_name": "pdf",
+    "skill_path": "/path/to/pdf",
+    "executor_model": "claude-sonnet-4-20250514",
+    "analyzer_model": "most-capable-model",
+    "timestamp": "2026-01-15T10:30:00Z",
+    "evals_run": [1, 2, 3],
+    "runs_per_configuration": 3
+  },
+
+  "runs": [
+    {
+      "eval_id": 1,
+      "eval_name": "Ocean",
+      "configuration": "with_skill",
+      "run_number": 1,
+      "result": {
+        "pass_rate": 0.85,
+        "passed": 6,
+        "failed": 1,
+        "total": 7,
+        "time_seconds": 42.5,
+        "tokens": 3800,
+        "tool_calls": 18,
+        "errors": 0
+      },
+      "assertions": [
+        {"text": "...", "passed": true, "evidence": "..."}
+      ],
+      "notes": [
+        "Used 2023 data, may be stale",
+        "Fell back to text overlay for non-fillable fields"
+      ]
+    }
+  ],
+
+  "run_summary": {
+    "with_skill": {
+      "pass_rate": {"mean": 0.85, "stddev": 0.05, "min": 0.80, "max": 0.90},
+      "time_seconds": {"mean": 45.0, "stddev": 12.0, "min": 32.0, "max": 58.0},
+      "tokens": {"mean": 3800, "stddev": 400, "min": 3200, "max": 4100}
+    },
+    "without_skill": {
+      "pass_rate": {"mean": 0.35, "stddev": 0.08, "min": 0.28, "max": 0.45},
+      "time_seconds": {"mean": 32.0, "stddev": 8.0, "min": 24.0, "max": 42.0},
+      "tokens": {"mean": 2100, "stddev": 300, "min": 1800, "max": 2500}
+    },
+    "delta": {
+      "pass_rate": "+0.50",
+      "time_seconds": "+13.0",
+      "tokens": "+1700"
+    }
+  },
+
+  "notes": [
+    "Assertion 'Output is a PDF file' passes 100% in both configurations - may not differentiate skill value",
+    "Eval 3 shows high variance (50% ± 40%) - may be flaky or model-dependent",
+    "Without-skill runs consistently fail on table extraction assertions",
+    "Skill adds 13s average execution time but improves pass rate by 50%"
+  ]
 }
 ```
 
-**`invoke.json` kinds:**
-- `kind: "cli"` — has `command`, `cwd`, `timeout_ms`. Use the command to run the target.
-- `kind: "agent"` — non-CLI provider. Check `manifest.json` `target.subagent_mode_allowed` to decide whether to dispatch executor subagents or fall back to `agentv eval` CLI.
+**Fields:**
+- `metadata`: Information about the benchmark run
+  - `skill_name`: Name of the skill
+  - `timestamp`: When the benchmark was run
+  - `evals_run`: List of eval names or IDs
+  - `runs_per_configuration`: Number of runs per config (e.g. 3)
+- `runs[]`: Individual run results
+  - `eval_id`: Numeric eval identifier
+  - `eval_name`: Human-readable eval name (used as section header in the viewer)
+  - `configuration`: Must be `"with_skill"` or `"without_skill"` (the viewer uses this exact string for grouping and color coding)
+  - `run_number`: Integer run number (1, 2, 3...)
+  - `result`: Nested object with `pass_rate`, `passed`, `total`, `time_seconds`, `tokens`, `errors`
+- `run_summary`: Statistical aggregates per configuration
+  - `with_skill` / `without_skill`: Each contains `pass_rate`, `time_seconds`, `tokens` objects with `mean` and `stddev` fields
+  - `delta`: Difference strings like `"+0.50"`, `"+13.0"`, `"+1700"`
+- `notes`: Freeform observations from the analyzer
 
-### `agentv pipeline grade <export-dir>`
+**Important:** The viewer reads these field names exactly. Using `config` instead of `configuration`, or putting `pass_rate` at the top level of a run instead of nested under `result`, will cause the viewer to show empty/zero values. Always reference this schema when generating benchmark.json manually.
 
-Runs code-grader assertions against `response.md` files in each test directory.
+---
 
-**Prerequisites:** `pipeline input` has been run and `response.md` exists in each test dir.
+## comparison.json
+
+Output from blind comparator. Located at `<grading-dir>/comparison-N.json`.
 
-**Output:** `<test-id>/code_grader_results/<name>.json` for each code grader, containing:
 ```json
 {
-  "name": "grader-name",
-  "type": "code-grader",
-  "score": 1.0,
-  "weight": 1.0,
-  "assertions": [{"text": "...", "passed": true}]
+  "winner": "A",
+  "reasoning": "Output A provides a complete solution with proper formatting and all required fields. Output B is missing the date field and has formatting inconsistencies.",
+  "rubric": {
+    "A": {
+      "content": {
+        "correctness": 5,
+        "completeness": 5,
+        "accuracy": 4
+      },
+      "structure": {
+        "organization": 4,
+        "formatting": 5,
+        "usability": 4
+      },
+      "content_score": 4.7,
+      "structure_score": 4.3,
+      "overall_score": 9.0
+    },
+    "B": {
+      "content": {
+        "correctness": 3,
+        "completeness": 2,
+        "accuracy": 3
+      },
+      "structure": {
+        "organization": 3,
+        "formatting": 2,
+        "usability": 3
+      },
+      "content_score": 2.7,
+      "structure_score": 2.7,
+      "overall_score": 5.4
+    }
+  },
+  "output_quality": {
+    "A": {
+      "score": 9,
+      "strengths": ["Complete solution", "Well-formatted", "All fields present"],
+      "weaknesses": ["Minor style inconsistency in header"]
+    },
+    "B": {
+      "score": 5,
+      "strengths": ["Readable output", "Correct basic structure"],
+      "weaknesses": ["Missing date field", "Formatting inconsistencies", "Partial data extraction"]
+    }
+  },
+  "assertions": {
+    "A": {
+      "passed": 4,
+      "total": 5,
+      "pass_rate": 0.80,
+      "details": [
+        {"text": "Output includes name", "passed": true}
+      ]
+    },
+    "B": {
+      "passed": 3,
+      "total": 5,
+      "pass_rate": 0.60,
+      "details": [
+        {"text": "Output includes name", "passed": true}
+      ]
+    }
+  }
 }
 ```
 
-### `agentv pipeline bench <export-dir>`
+---
 
-Merges code-grader results with LLM grader scores and produces final artifacts.
+## analysis.json
 
-LLM grader results are read from disk at `<test-id>/llm_grader_results/<name>.json` per test.
+Output from post-hoc analyzer. Located at `<grading-dir>/analysis.json`.
 
-**LLM grader result file format** (`llm_grader_results/<name>.json`):
 ```json
-{ "score": 0.85, "assertions": [{"text": "...", "passed": true, "evidence": "..."}] }
+{
+  "comparison_summary": {
+    "winner": "A",
+    "winner_skill": "path/to/winner/skill",
+    "loser_skill": "path/to/loser/skill",
+    "comparator_reasoning": "Brief summary of why comparator chose winner"
+  },
+  "winner_strengths": [
+    "Clear step-by-step instructions for handling multi-page documents",
+    "Included validation script that caught formatting errors"
+  ],
+  "loser_weaknesses": [
+    "Vague instruction 'process the document appropriately' led to inconsistent behavior",
+    "No script for validation, agent had to improvise"
+  ],
+  "instruction_following": {
+    "winner": {
+      "score": 9,
+      "issues": ["Minor: skipped optional logging step"]
+    },
+    "loser": {
+      "score": 6,
+      "issues": [
+        "Did not use the skill's formatting template",
+        "Invented own approach instead of following step 3"
+      ]
+    }
+  },
+  "improvement_suggestions": [
+    {
+      "priority": "high",
+      "category": "instructions",
+      "suggestion": "Replace 'process the document appropriately' with explicit steps",
+      "expected_impact": "Would eliminate ambiguity that caused inconsistent behavior"
+    }
+  ],
+  "transcript_insights": {
+    "winner_execution_pattern": "Read skill -> Followed 5-step process -> Used validation script",
+    "loser_execution_pattern": "Read skill -> Unclear on approach -> Tried 3 different methods"
+  }
+}
 ```
 
-**Output:**
-- `<test-id>/grading.json` — merged grading with `graders`, `assertions`, `summary.pass_rate`
-- `index.jsonl` — one JSON line per test: `{test_id, score, pass, graders: [...]}`
-- `benchmark.json` — aggregate stats: `{metadata: {targets}, run_summary: {<target>: {mean, stddev, n}}}`
+--- references/eval-yaml-spec.md ---
+# Eval YAML Spec — Schema and Assertion Grading Recipes
 
-### Agent-Mode Workflow
+This reference documents the eval.yaml schema and grading recipes for every assertion type.
+The grader agent uses this to evaluate assertions without the CLI.
 
-```
-1. agentv pipeline input eval.yaml --out ./export
-2. (Agent runs targets or reads response.md)
-3. agentv pipeline grade ./export
-4. (Agent does LLM grading, produces scores JSON)
-5. echo '<scores>' | agentv pipeline bench ./export
-```
+## 1. Eval YAML Structure
 
---- references/migrating-from-skill-creator.md ---
-# Migrating from Skill-Creator to AgentV Lifecycle Skill
+### Top-level fields
 
-This reference covers how to use AgentV's unified agent-evaluation lifecycle skill (`agentv-bench`) with evals.json files originally created for Anthropic's skill-creator.
+- `name` (string, optional) — eval name
+- `description` (string, optional) — description
+- `execution` (object, optional) — `target`, `model`, etc.
+- `workspace` (object, optional) — workspace config (template, hooks)
+- `tests` (array, required) — test cases
 
-## Drop-in Replacement
+### Per-test fields
 
-AgentV runs skill-creator's evals.json directly — no conversion required:
+- `id` (string, required) — unique test identifier
+- `input` (string | Message[], required) — task input. String shorthand expands to `[{role: user, content: "..."}]`
+- `expected_output` (string | Message[], optional) — reference answer. String shorthand expands to `[{role: assistant, content: "..."}]`
+- `criteria` (string, optional) — human-readable success criteria
+- `assertions` (array, optional) — grader assertions
+- `conversation_id` (string, optional) — groups related tests
+- `execution` (object, optional) — per-test execution override
 
-```bash
-# Run evals.json with AgentV
-agentv eval evals.json
+## 2. Assertion Types and Grading Recipes
 
-# Or run a single assertion offline (no API keys)
-agentv eval assert <grader-name> --agent-output "..." --agent-input "..."
-```
+For each assertion type: YAML config fields, grading recipe (exact pseudocode for deterministic types), and PASS/FAIL conditions.
 
-AgentV automatically:
-- Promotes `prompt` → input messages
-- Promotes `expected_output` → reference answer
-- Converts `assertions` → LLM-grader graders
-- Resolves `files[]` paths relative to the evals.json directory
+### Deterministic assertions (zero-cost, instant)
 
-If you're using the `agentv-bench` skill, it orchestrates these same AgentV commands. Code graders, grading, and artifact generation remain in AgentV core; the skill just orchestrates and summarizes the existing outputs.
+#### `contains`
 
-## What You Gain
+- **Fields:** `value` (string, required)
+- **Recipe:**
+  ```
+  response.toLowerCase().includes(value.toLowerCase())
+  ```
+  Note: case-insensitive by default in AgentV. If `case_sensitive: true`, use exact match.
+- **PASS:** substring found. **FAIL:** substring not found.
 
-Moving from skill-creator's eval loop to AgentV's lifecycle skill gives you:
+#### `contains-any`
 
-| Capability | skill-creator | AgentV lifecycle skill |
-|-----------|---------------|----------------------|
-| Workspace isolation | ❌ | ✅ Clone repos, run setup/teardown scripts |
-| Code graders | ❌ | ✅ Python/TypeScript grader scripts via `defineCodeGrader()` |
-| Tool trajectory scoring | ❌ | ✅ Evaluate tool call sequences |
-| Multi-provider comparison | with-skill vs without-skill | N-way: Claude, GPT, Copilot, Gemini, custom CLI |
-| Multi-turn evaluation | ❌ | ✅ Conversation tracking with `conversation_id` |
-| Blind comparison | ❌ | ✅ Judge doesn't know which is baseline |
-| Deterministic upgrade suggestions | ❌ | ✅ LLM-grader → contains/regex/is-json |
-| Human review checkpoint | ❌ | ✅ Structured feedback gate |
-| Workspace file tracking | ❌ | ✅ Evaluate by diffing workspace files |
-| Agent mode (no API keys) | ❌ | ✅ Uses grader agent in agent mode |
+- **Fields:** `value` (string[], required)
+- **Recipe:**
+  ```
+  value.some(v => response.toLowerCase().includes(v.toLowerCase()))
+  ```
+- **PASS:** at least one substring found.
 
-## Artifact Compatibility
+#### `contains-all`
 
-AgentV's companion artifacts are compatible with skill-creator's eval-viewer:
+- **Fields:** `value` (string[], required)
+- **Recipe:**
+  ```
+  value.every(v => response.toLowerCase().includes(v.toLowerCase()))
+  ```
+- **PASS:** all substrings found.
 
-| Artifact | Format | Compatible with eval-viewer |
-|----------|--------|---------------------------|
-| `<test-id>/grading.json` | Per-assertion evidence with claims | ✅ Superset of skill-creator's per-test grading format |
-| `benchmark.json` | Aggregate pass rates, timing, patterns | ✅ Superset of Agent Skills benchmark format |
-| Results JSONL | Per-test results | ✅ Standard JSONL format |
+#### `icontains` / `icontains-any` / `icontains-all`
 
-AgentV's schemas are supersets — they include all fields skill-creator expects, plus additional fields (claims extraction, pattern analysis, deterministic upgrade candidates). Tools that read skill-creator artifacts will read AgentV artifacts correctly, ignoring the extra fields.
+Same as contains variants but explicitly case-insensitive.
 
-The optimizer scripts layer reads those same artifacts directly:
-- `aggregate-benchmark.ts` consumes `benchmark.json`, `timing.json`, and results JSONL
-- `generate-report.ts` and `eval-viewer/generate-review.ts` render review output from AgentV artifacts
-- `improve-description.ts` proposes follow-up experiments from benchmark/grading observations
+#### `equals`
 
-## Graduating to EVAL.yaml
+- **Fields:** `value` (string, required)
+- **Recipe:**
+  ```
+  response.trim() === value.trim()
+  ```
+- **PASS:** exact match after trimming.
 
-When evals.json becomes limiting, convert to EVAL.yaml for the full feature set:
+#### `regex`
 
-```bash
-# Convert evals.json to EVAL.yaml
-agentv convert evals.json
+- **Fields:** `value` (string, required — a regex pattern)
+- **Recipe:**
+  ```
+  new RegExp(value).test(response)
+  ```
+- **PASS:** pattern matches.
 
-# Edit the generated YAML to add workspace config, code graders, etc.
-# Then run with the full lifecycle
-agentv eval eval.yaml
-```
+#### `starts-with`
 
-EVAL.yaml unlocks:
-- **Workspace setup/teardown** — clone repos, install dependencies, clean up after tests
-- **Code graders** — write graders in Python or TypeScript, not just LLM prompts
-- **Rubric-based grading** — multi-dimensional scoring with weighted criteria
-- **Retry policies** — automatic retries for flaky tests with configurable backoff
-- **Test groups** — organize tests by category with shared config
-- **Multi-turn conversations** — test agent interactions across multiple turns
+- **Fields:** `value` (string, required)
+- **Recipe:**
+  ```
+  response.startsWith(value)
+  ```
+  (or case-insensitive variant)
+- **PASS:** response starts with value.
 
-## What Stays in Skill-Creator
+#### `ends-with`
 
-AgentV does NOT replace these skill-creator capabilities:
+- **Fields:** `value` (string, required)
+- **Recipe:**
+  ```
+  response.endsWith(value)
+  ```
+  (or case-insensitive variant)
+- **PASS:** response ends with value.
 
-- **Trigger optimization** — optimizing when/how a skill is triggered
-- **.skill packaging** — bundling skills for distribution
-- **Skill authoring** — creating new SKILL.md files from scratch
-- **Skill discovery** — finding and installing skills
+#### `is-json`
 
-AgentV focuses on the **evaluation and optimization loop**. Skill-creator focuses on **skill authoring and packaging**. They are complementary — use skill-creator to write the skill, use AgentV to evaluate and optimize it.
+- **Fields:** none required
+- **Recipe:**
+  ```
+  try { JSON.parse(response); return true } catch { return false }
+  ```
+- **PASS:** response is valid JSON. **FAIL:** parse error.
 
-## Example Workflow
+#### `field-accuracy`
 
-```
-1. Author a skill with skill-creator
-2. skill-creator generates evals.json
-3. Run evals.json through AgentV's lifecycle skill for richer evaluation:
-   - Workspace isolation (test in a real repo)
-   - Multi-provider comparison (does the skill work with GPT too?)
-   - Blind comparison (is the new version actually better?)
-   - Deterministic upgrades (replace vague LLM graders with precise checks)
-4. Use AgentV's optimization loop to refine the skill's prompts
-5. Return to skill-creator for packaging and distribution
-```
+- **Fields:** `expected` (object, required — JSON object with field paths and expected values)
+- **Recipe:** Parse response as JSON. For each field path in `expected`, check if the value matches.
+- **PASS:** all fields match. Partial score = `matched_fields / total_fields`.
 
---- references/schemas.md ---
-# JSON Schemas
+### Metric assertions (require timing.json)
 
-This document defines the JSON schemas used by skill-creator.
+#### `latency`
 
----
+- **Fields:** `threshold` (number, required — max duration in ms)
+- **Recipe:** Read `timing.json`. Compare `duration_ms` against threshold.
+- **PASS:** `duration_ms <= threshold`.
 
-## evals.json
+#### `cost`
 
-Defines the evals for a skill. Located at `evals/evals.json` within the skill directory.
+- **Fields:** `threshold` (number, required — max cost in USD)
+- **Recipe:** Read timing/token data. Compare cost against threshold.
+- **PASS:** `cost <= threshold`.
 
-```json
-{
-  "skill_name": "example-skill",
-  "evals": [
-    {
-      "id": 1,
-      "prompt": "User's example prompt",
-      "expected_output": "Description of expected result",
-      "files": ["evals/files/sample1.pdf"],
-      "assertions": [
-        "The output includes X",
-        "The skill used script Y"
-      ]
-    }
-  ]
-}
-```
+#### `token-usage`
 
-**Fields:**
-- `skill_name`: Name matching the skill's frontmatter
-- `evals[].id`: Unique integer identifier
-- `evals[].prompt`: The task to execute
-- `evals[].expected_output`: Human-readable description of success
-- `evals[].files`: Optional list of input file paths (relative to skill root)
-- `evals[].assertions`: List of verifiable statements
+- **Fields:** `threshold` (number, required — max tokens)
+- **Recipe:** Read `timing.json`. Compare `total_tokens` against threshold.
+- **PASS:** `total_tokens <= threshold`.
 
----
+#### `execution-metrics`
 
-## history.json
+- **Fields:** Various threshold fields for tool calls, output chars, etc.
+- **Recipe:** Read timing.json, compare each metric against its threshold.
 
-Tracks version progression in Improve mode. Located at workspace root.
+### Tool inspection assertions
 
-```json
-{
-  "started_at": "2026-01-15T10:30:00Z",
-  "skill_name": "pdf",
-  "current_best": "v2",
-  "iterations": [
-    {
-      "version": "v0",
-      "parent": null,
-      "assertion_pass_rate": 0.65,
-      "grading_result": "baseline",
-      "is_current_best": false
-    },
-    {
-      "version": "v1",
-      "parent": "v0",
-      "assertion_pass_rate": 0.75,
-      "grading_result": "won",
-      "is_current_best": false
-    },
-    {
-      "version": "v2",
-      "parent": "v1",
-      "assertion_pass_rate": 0.85,
-      "grading_result": "won",
-      "is_current_best": true
-    }
-  ]
-}
-```
+#### `tool-trajectory`
 
-**Fields:**
-- `started_at`: ISO timestamp of when improvement started
-- `skill_name`: Name of the skill being improved
-- `current_best`: Version identifier of the best performer
-- `iterations[].version`: Version identifier (v0, v1, ...)
-- `iterations[].parent`: Parent version this was derived from
-- `iterations[].assertion_pass_rate`: Pass rate from grading
-- `iterations[].grading_result`: "baseline", "won", "lost", or "tie"
-- `iterations[].is_current_best`: Whether this is the current best version
+- **Fields:** `expected` (array of expected tool calls), `mode` (string: `exact` | `contains` | `order`)
+- **Recipe:** Inspect transcript for tool call sequence. Match against expected based on mode.
+- **PASS:** tool calls match expected pattern per mode.
 
----
+#### `skill-trigger`
 
-## grading.json
+- **Fields:** `skill_name` (string, required)
+- **Recipe:** Check if the agent invoked the named skill in its tool calls.
+- **PASS:** skill was triggered.
 
-Output from the grader agent. Located at `<run-dir>/grading.json`.
+### LLM-judged assertions (require Claude reasoning)
 
-**Important:** The `assertions` array must use the fields `text`, `passed`, and `evidence` — downstream tooling depends on these exact field names.
+#### `llm-grader`
 
-```json
-{
-  "assertions": [
-    {
-      "text": "The output includes the name 'John Smith'",
-      "passed": true,
-      "evidence": "Found in transcript Step 3: 'Extracted names: John Smith, Sarah Johnson'"
-    },
-    {
-      "text": "The spreadsheet has a SUM formula in cell B10",
-      "passed": false,
-      "evidence": "No spreadsheet was created. The output was a text file."
-    }
-  ],
-  "summary": {
-    "passed": 2,
-    "failed": 1,
-    "total": 3,
-    "pass_rate": 0.67
-  },
-  "execution_metrics": {
-    "tool_calls": {
-      "Read": 5,
-      "Write": 2,
-      "Bash": 8
-    },
-    "total_tool_calls": 15,
-    "total_steps": 6,
-    "errors_encountered": 0,
-    "output_chars": 12450,
-    "transcript_chars": 3200
-  },
-  "timing": {
-    "executor_duration_seconds": 165.0,
-    "grader_duration_seconds": 26.0,
-    "total_duration_seconds": 191.0
-  },
-  "claims": [
-    {
-      "claim": "The form has 12 fillable fields",
-      "type": "factual",
-      "verified": true,
-      "evidence": "Counted 12 fields in field_info.json"
-    }
-  ],
-  "user_notes_summary": {
-    "uncertainties": ["Used 2023 data, may be stale"],
-    "needs_review": [],
-    "workarounds": ["Fell back to text overlay for non-fillable fields"]
-  },
-  "eval_feedback": {
-    "suggestions": [
-      {
-        "assertion": "The output includes the name 'John Smith'",
-        "reason": "A hallucinated document that mentions the name would also pass"
-      }
-    ],
-    "overall": "Assertions check presence but not correctness."
-  }
-}
+- **Fields:** `prompt` (string, required — either inline text or path to .md file)
+- **Recipe:** Read the prompt. Evaluate the response against the criteria using your own reasoning. Produce score (0.0-1.0) with evidence.
+- **PASS:** score >= 0.5 (configurable via `threshold`).
+
+#### `rubric` / `rubrics`
+
+- **Fields:** `rubric_items` or `criteria` (array of rubric items with descriptions and weights)
+- **Recipe:** For each rubric item, evaluate the response. Score each item 0.0-1.0. Aggregate as weighted average.
+- **PASS:** aggregate score >= threshold.
+
+### Script-based assertions
+
+#### `code-grader`
+
+- **Fields:** `path` (string, required — path to script), `command` (string[], optional — custom command)
+- **Script SDK:** Use `defineCodeGrader` from `@agentv/eval`:
+  ```typescript
+  import { defineCodeGrader } from '@agentv/eval';
+  export default defineCodeGrader(({ output, trace }) => ({
+    score: (output ?? '').includes('expected') ? 1 : 0,
+    assertions: [{ text: 'Contains expected', passed: (output ?? '').includes('expected') }],
+  }));
+  ```
+- **Recipe:** The CLI runs the script, passing canonical JSON on stdin (`{output, input, expected_output, ...}`). Script returns `{"score": N, "assertions": [...]}`
+- **PASS:** score >= 0.5 (or as configured).
+
+### Composite assertion
+
+#### `composite`
+
+- **Fields:** `assertions` (array of sub-assertions), `aggregation` (string: `weighted_average` | `min` | `max` | `all_pass`)
+- **Recipe:** Evaluate each sub-assertion. Aggregate scores per aggregation mode.
+- **PASS:** depends on aggregation mode.
+
+## 3. Negate Support
+
+When `negate: true` is set on any assertion, invert the pass/fail result:
+
+- A passing check becomes a failure
+- A failing check becomes a pass
+- Score is inverted: `1.0 - score`
+
+## 4. Common Assertion Fields
+
+All assertion types support:
+
+- `name` (string, optional) — human-readable name
+- `type` (string, required) — the assertion type
+- `weight` (number, optional, default 1.0) — weight in score aggregation
+- `negate` (boolean, optional) — invert result
+- `threshold` (number, optional) — minimum score to pass (for LLM types)
+
+## 5. AgentV JSONL Output Format
+
+Each line in the results JSONL file is an `EvaluationResult` object. In JSONL, field names use snake_case (applied by `toSnakeCaseDeep()`).
+
+### Required fields
+
+- `timestamp` (string, ISO-8601)
+- `test_id` (string)
+- `score` (number, 0.0-1.0, weighted average of all assertion scores)
+- `assertions` (array of `{text, passed, evidence?}`)
+- `output` (Message[]) — agent output messages
+- `execution_status` (string: `ok` | `quality_failure` | `execution_error`)
+
+### Optional fields
+
+- `scores` (array of EvaluatorResult) — per-grader breakdown
+- `input` (Message[]) — input messages
+- `token_usage` (object: `{prompt_tokens, completion_tokens, total_tokens}`)
+- `cost_usd` (number)
+- `duration_ms` (number)
+- `target` (string)
+- `eval_set` (string)
+- `error` (string)
+- `file_changes` (string — unified diff)
+- `mode` (string — `agent` for agent mode)
+
+### `scores[]` entries (EvaluatorResult)
+
+- `name` (string) — grader name
+- `type` (string) — grader kind (kebab-case)
+- `score` (number, 0.0-1.0)
+- `assertions` (array of `{text, passed, evidence?}`)
+- `weight` (number, optional)
+- `verdict` (string: `pass` | `fail` | `skip`)
+- `details` (object, optional — structured data from code graders)
+- `reasoning` (string, optional)
+
+## 6. Eval Set Support
+
+An eval_set references multiple eval.yaml files:
+
+```yaml
+# eval_set.yaml
+eval_set:
+  - path: ./basic.eval.yaml
+  - path: ./advanced.eval.yaml
 ```
 
-**Fields:**
-- `assertions[]`: Graded assertion results with evidence
-- `summary`: Aggregate pass/fail counts
-- `execution_metrics`: Tool usage and output size (from executor's metrics.json)
-- `timing`: Wall clock timing (from timing.json)
-- `claims`: Extracted and verified claims from the output
-- `user_notes_summary`: Issues flagged by the executor
-- `eval_feedback`: (optional) Improvement suggestions for the evals, only present when the grader identifies issues worth raising
+Process each file's tests independently, then aggregate results.
 
----
+## 7. Agent-Mode Pipeline CLI Commands
 
-## metrics.json
+These CLI subcommands break the monolithic `eval run` into discrete steps for agent-mode execution. The agent handles LLM grading between steps.
 
-Output from the executor agent. Located at `<run-dir>/outputs/metrics.json`.
+### `agentv pipeline input <eval-path> --out <dir>`
 
+Extracts inputs, target commands, and grader configs from an eval YAML file.
+
+**Output structure:**
+```
+<out-dir>/
+├── manifest.json
+├── <test-id>/
+│   ├── input.json              ← {input, input_files, metadata}
+│   ├── invoke.json             ← {kind, command?, cwd?, timeout_ms?}
+│   ├── criteria.md             ← human-readable success criteria
+│   ├── expected_output.json    ← (if present)
+│   ├── code_graders/<name>.json   ← {name, command, weight, config?}
+│   └── llm_graders/<name>.json    ← {name, weight, threshold?, prompt_content}
+```
+
+**`manifest.json` format:**
 ```json
 {
-  "tool_calls": {
-    "Read": 5,
-    "Write": 2,
-    "Bash": 8,
-    "Edit": 1,
-    "Glob": 2,
-    "Grep": 0
-  },
-  "total_tool_calls": 18,
-  "total_steps": 6,
-  "files_created": ["filled_form.pdf", "field_values.json"],
-  "errors_encountered": 0,
-  "output_chars": 12450,
-  "transcript_chars": 3200
+  "eval_file": "path/to/eval.yaml",
+  "timestamp": "2026-03-24T...",
+  "target": {"name": "target-name", "kind": "cli", "subagent_mode_allowed": false},
+  "test_ids": ["test-01", "test-02"]
 }
 ```
 
-**Fields:**
-- `tool_calls`: Count per tool type
-- `total_tool_calls`: Sum of all tool calls
-- `total_steps`: Number of major execution steps
-- `files_created`: List of output files created
-- `errors_encountered`: Number of errors during execution
-- `output_chars`: Total character count of output files
-- `transcript_chars`: Character count of transcript
-
----
+**`invoke.json` kinds:**
+- `kind: "cli"` — has `command`, `cwd`, `timeout_ms`. Use the command to run the target.
+- `kind: "agent"` — non-CLI provider. Check `manifest.json` `target.subagent_mode_allowed` to decide whether to dispatch executor subagents or fall back to `agentv eval` CLI.
 
-## timing.json
+### `agentv pipeline grade <export-dir>`
 
-Wall clock timing for a run. Located at `<run-dir>/timing.json`.
+Runs code-grader assertions against `response.md` files in each test directory.
 
-**How to capture:** When a subagent task completes, the task notification includes `total_tokens` and `duration_ms`. Save these immediately — they are not persisted anywhere else and cannot be recovered after the fact.
+**Prerequisites:** `pipeline input` has been run and `response.md` exists in each test dir.
 
+**Output:** `<test-id>/code_grader_results/<name>.json` for each code grader, containing:
 ```json
 {
-  "total_tokens": 84852,
-  "duration_ms": 23332,
-  "total_duration_seconds": 23.3,
-  "executor_start": "2026-01-15T10:30:00Z",
-  "executor_end": "2026-01-15T10:32:45Z",
-  "executor_duration_seconds": 165.0,
-  "grader_start": "2026-01-15T10:32:46Z",
-  "grader_end": "2026-01-15T10:33:12Z",
-  "grader_duration_seconds": 26.0
+  "name": "grader-name",
+  "type": "code-grader",
+  "score": 1.0,
+  "weight": 1.0,
+  "assertions": [{"text": "...", "passed": true}]
 }
 ```
 
----
+### `agentv pipeline bench <export-dir>`
 
-## benchmark.json
+Merges code-grader results with LLM grader scores and produces final artifacts.
 
-Output from Benchmark mode. Located at `benchmarks/<timestamp>/benchmark.json`.
+LLM grader results are read from disk at `<test-id>/llm_grader_results/<name>.json` per test.
 
+**LLM grader result file format** (`llm_grader_results/<name>.json`):
 ```json
-{
-  "metadata": {
-    "skill_name": "pdf",
-    "skill_path": "/path/to/pdf",
-    "executor_model": "claude-sonnet-4-20250514",
-    "analyzer_model": "most-capable-model",
-    "timestamp": "2026-01-15T10:30:00Z",
-    "evals_run": [1, 2, 3],
-    "runs_per_configuration": 3
-  },
+{ "score": 0.85, "assertions": [{"text": "...", "passed": true, "evidence": "..."}] }
+```
+
+**Output:**
+- `<test-id>/grading.json` — merged grading with `graders`, `assertions`, `summary.pass_rate`
+- `index.jsonl` — one JSON line per test: `{test_id, score, pass, graders: [...]}`
+- `benchmark.json` — aggregate stats: `{metadata: {targets}, run_summary: {<target>: {mean, stddev, n}}}`
+
+### Agent-Mode Workflow
+
+```
+1. agentv pipeline input eval.yaml --out ./export
+2. (Agent runs targets or reads response.md)
+3. agentv pipeline grade ./export
+4. (Agent does LLM grading, produces scores JSON)
+5. echo '<scores>' | agentv pipeline bench ./export
+```
+
+--- references/description-optimization.md ---
+# Description Optimization
+
+Optimize the `description` field in a skill's SKILL.md frontmatter for better triggering
+accuracy. Use this after the agent/skill is working well — this is a polish step, not a
+core workflow step.
+
+**Provider compatibility**: Description optimization applies to any agent platform with
+skill-discovery mechanisms — Claude Code, Codex (`.agents/` or `.codex/` folders), Copilot,
+and others. The `skill-trigger` grader checks whether the agent invoked the right skill,
+regardless of how discovery works on that platform.
+
+## Step 1: Generate Trigger EVAL.yaml
+
+Create 20 test cases:
+- **10 should-trigger**: realistic prompts where this skill should activate — different
+  phrasings, casual speech, uncommon use cases, edge cases where this skill competes with
+  another but should win
+- **10 should-not-trigger**: near-miss prompts that share keywords but actually need
+  something different — adjacent domains, ambiguous phrasing where naive matching would
+  trigger but shouldn't
+
+Prompts must be realistic — include file paths, personal context, typos, casual speech.
+Not abstract requests like "format data" but concrete ones like "ok so my boss sent me
+Q4-sales-FINAL-v2.xlsx and she wants me to add a profit margin column..."
+
+The should-not-trigger cases are the most valuable. "Write a fibonacci function" as a
+negative test for an eval skill is useless — it doesn't test anything. The negative cases
+should be genuinely tricky near-misses.
+
+Write as EVAL.yaml with top-level input (the user prompt doesn't specify the skill name —
+it's a natural utterance):
+
+```yaml
+# trigger-eval.eval.yaml
+tests:
+  - id: should-trigger-casual-optimize
+    input: "ok so I have this agent that keeps failing on the code review tasks, can you help me figure out why and fix it"
+    assertions:
+      - type: skill-trigger
+        skill: agentv-bench
+  - id: should-not-trigger-build-error
+    input: "my TypeScript build is failing with type errors in src/auth.ts"
+    assertions:
+      - type: skill-trigger
+        skill: agentv-bench
+        should_trigger: false
+```
+
+## Step 2: Review with User
+
+Present the eval set. The user adjusts queries, toggles should-trigger, adds/removes cases.
+This step matters — bad eval queries lead to bad descriptions.
+
+## Step 3: Iterate on Description
+
+Run the trigger eval, identify misfires, rewrite the description, re-run. Max 5 iterations.
+Select best description by held-out test accuracy (split 60% train / 40% test) to avoid
+overfitting.
+
+Use the grader and analyzer subagents to identify trigger failures and propose description
+improvements — the same eval → grade → analyze → improve loop used for agent output quality.
+
+## Step 4: Apply
+
+Update the skill's SKILL.md frontmatter with the optimized description. Show the user
+before/after with accuracy scores.
+
+--- references/environment-adaptation.md ---
+# Environment Adaptation
+
+Provider-specific notes, CI/headless behavior, and fallback strategies for environments
+with limited capabilities.
+
+## CI/Headless Mode
+
+Skip interactive prompts. Exit with pass/fail status code. Always generate artifacts for
+downstream consumption.
+
+## No Subagents Available (e.g., Claude.ai)
+
+Run test cases serially. Skip blind comparison. Present results directly in conversation —
+for each test case, show the prompt and output. Ask for feedback inline. Skip benchmarking
+(it relies on baseline comparisons that aren't meaningful without subagents).
+
+## Provider-Specific Notes
+
+- **Copilot CLI**: Uses ACP protocol via `copilot --acp --stdio`
+- **Claude SDK**: Requires `@anthropic-ai/claude-agent-sdk` installed
+- **Codex**: Supports skills via `.agents/` or `.codex/` folders. Emits `command_execution`
+  and `file_change` tool calls.
+- **Custom CLI**: Needs `command` and output file pattern in target config
+- **Target config**: Uses `${{ ENV_VAR }}` syntax (not `${ENV_VAR}`) for API keys
+
+**Note**: "Description Optimization" (see `references/description-optimization.md`) applies
+to any platform with skill-discovery mechanisms. All listed providers support skills.
+
+## Unsupported Providers: Use a Code-Grader
+
+The built-in `skill-trigger` grader covers Claude, Copilot, Pi, Codex and VS Code out
+of the box. For providers with different tool-call formats, write a code-grader that inspects
+the agent's tool call trace.
+
+A code-grader receives the full evaluation context including the agent's output messages and
+tool calls. You can inspect these to determine whether the skill was invoked:
+
+```yaml
+# Example: code-grader for Codex skill-trigger detection
+tests:
+  - id: should-trigger-codex
+    input: "Analyze this CSV file"
+    assertions:
+      - type: code-grader
+        path: ./judges/codex-skill-trigger.ts
+```
+
+```typescript
+// judges/codex-skill-trigger.ts
+import { defineCodeGrader } from '@agentv/eval';
+
+export default defineCodeGrader(({ output }) => {
+  const skillName = 'csv-analyzer';
+  const toolCalls = (output ?? []).flatMap((msg) => msg.toolCalls ?? []);
+  const firstTool = toolCalls[0];
+
+  if (!firstTool) {
+    return { score: 0, reason: 'No tool calls recorded' };
+  }
+
+  // Codex reads skill files via shell commands
+  if (firstTool.tool === 'command_execution') {
+    const cmd = String(firstTool.input ?? '');
+    if (cmd.includes(skillName)) {
+      return { score: 1, reason: `Skill "${skillName}" triggered via command: ${cmd}` };
+    }
+  }
+
+  // Check if skill file was read via file_change or other tools
+  if (firstTool.tool === 'file_change') {
+    const path = String((firstTool.input as Record<string, unknown>)?.path ?? '');
+    if (path.includes(skillName)) {
+      return { score: 1, reason: `Skill file accessed: ${path}` };
+    }
+  }
+
+  return { score: 0, reason: `First tool was "${firstTool.tool}" — not a skill invocation for "${skillName}"` };
+});
+```
+
+This approach is more flexible than config overrides — you can match any tool-call pattern,
+check multiple fields, and add provider-specific logic as needed.
+
+--- references/subagent-pipeline.md ---
+# Subagent Pipeline — Running eval.yaml without CLI
+
+This reference documents the detailed procedure for running evaluations in subagent mode
+(`AGENT_EVAL_MODE=subagent`, the default). The orchestrating skill dispatches `executor`
+subagents to perform test cases and `grader` subagents to evaluate outputs.
+
+Read this reference when executing Step 3 (Run and Grade) in subagent mode.
+
+## Prerequisites
+
+- The eval.yaml file exists and contains valid test definitions
+- `agentv` CLI is installed (or run from source via `AGENTV_CLI=bun /path/to/cli.ts` in `.env`)
+- Read `references/eval-yaml-spec.md` for the full schema
+
+## Workspace Context
+
+Some evals pass prompt files directly and don't require a specific workspace — those run fine
+from anywhere. But evals that test agent behavior in a workspace (accessing skills, modifying
+repos, using tools across multiple repos) require the user to be in the **target workspace**
+(e.g., a multi-repo workspace set up by allagents). If the eval references workspace files or
+expects the agent to use skills, check that the current directory is the target workspace, not
+just the eval repo — and warn the user if it's wrong.
+
+## Executor Subagent Eligibility
+
+All providers except `cli` are eligible for executor subagents by default. To opt out a
+specific target, set `subagent_mode_allowed: false` in `.agentv/targets.yaml`:
+
+```yaml
+# .agentv/targets.yaml
+targets:
+  - name: my-target
+    provider: openai
+    model: ${{ OPENAI_MODEL }}
+    api_key: ${{ OPENAI_API_KEY }}
+    subagent_mode_allowed: false  # forces CLI invocation instead of executor subagent
+```
+
+When `subagent_mode_allowed: false`, the target falls back to CLI invocation via `agentv eval`
+even in subagent mode.
+
+## CLI Targets: Single Command
+
+For evals with CLI targets, `pipeline run` handles input extraction, target invocation, and
+code grading in one step. When `--out` is omitted, the output directory defaults to
+`.agentv/results/runs/<timestamp>` (same convention as `agentv eval`):
+
+```bash
+# Extract inputs and invoke all CLI targets in parallel:
+agentv pipeline run evals/repro.eval.yaml
+
+# Also run code graders inline (instead of using pipeline grade separately):
+agentv pipeline run evals/repro.eval.yaml --grader-type code
+```
+
+By default, `pipeline run` extracts inputs and invokes targets only. Pass `--grader-type code`
+to also run code-graders inline, or use `agentv pipeline grade <run-dir>` as a separate step.
+
+The run directory is printed to stdout. Then continue to the grading and merge phases
+described in SKILL.md Step 3.
+
+## Non-CLI Targets: Executor Subagents
+
+When the target provider is not `cli`, check `manifest.json` → `target.subagent_mode_allowed`.
+If `true` (default for all non-CLI providers), the subagent IS the target. If `false` (user
+opted out via `subagent_mode_allowed: false` in `.agentv/targets.yaml`), fall back to
+`agentv eval` CLI mode instead.
+
+### Step 1: Extract inputs
+
+```bash
+# Defaults to .agentv/results/runs/<timestamp>
+agentv pipeline input evals/repro.eval.yaml
+```
+
+This creates a run directory with per-test `input.json`, `invoke.json`,
+`criteria.md`, and grader configs.
+
+### Step 2: Dispatch executor subagents
+
+Read `agents/executor.md`. Launch one `executor` subagent **per test case**, all in parallel.
+Each subagent receives the test directory path, reads `input.json`, performs the task using
+its own tools, and writes `response.md`.
+
+Example: 5 tests = 5 executor subagents launched simultaneously.
+
+```
+# Per executor subagent:
+#   - Reads <run-dir>/<test-id>/input.json
+#   - Performs the task
+#   - Writes <run-dir>/<test-id>/response.md
+```
+
+### After executors complete: read results from disk
+
+When all executor subagents have finished, **read `response.md` directly from disk** — do
+NOT use `read_agent` to fetch results. The executors wrote their outputs to the run directory.
+
+```bash
+# Verify all responses exist:
+for d in <run-dir>/<evalset>/*/; do
+  echo "$(basename $d): $(ls "$d"/response.md 2>/dev/null && echo OK || echo MISSING)"
+done
+```
+
+If any `response.md` is MISSING, re-run that specific executor subagent. Do not proceed to
+grading until all responses are present.
+
+### Step 3 onward: Grade and merge
+
+See SKILL.md Step 3 "Grading" section for the three-phase grading process (code graders →
+LLM grading → merge and validate).
+
+## Step-by-Step Fine-Grained Control (CLI targets)
+
+Use individual commands when you need control over each step with CLI targets:
+
+```bash
+# Step 1: Extract inputs (defaults to .agentv/results/runs/<timestamp>)
+agentv pipeline input evals/repro.eval.yaml
+
+# Step 2: run_tests.py invokes CLI targets (or use pipeline run instead)
+
+# Step 3: Run code graders
+agentv pipeline grade <run-dir>
+
+# Step 4: Subagent does LLM grading, writes results to llm_grader_results/<name>.json per test
+
+# Step 5: Merge scores (writes index.jsonl with full scores[] for dashboard)
+agentv pipeline bench <run-dir>
+
+# Step 6: Validate
+agentv results validate <run-dir>
+```
+
+## LLM Grading JSON Format
+
+The agent reads `llm_graders/<name>.json` for each test, grades the response using the prompt
+content, and produces a scores JSON:
+
+```json
+{
+  "test-01": {
+    "relevance": {
+      "score": 0.85,
+      "assertions": [{"text": "Response is relevant", "passed": true, "evidence": "..."}]
+    }
+  }
+}
+```
+
+## Pipeline Bench and Dashboard
+
+`pipeline bench` merges LLM scores into `index.jsonl` with a full `scores[]` array per entry,
+matching the CLI-mode schema. The web dashboard (`agentv results serve`) reads this format
+directly — no separate conversion script is needed. Run `agentv results validate <run-dir>`
+to verify compatibility.
+
+## Output Structure
+
+The path hierarchy mirrors the CLI mode: `<evalset-name>` comes from the `name` field in
+the eval.yaml. The target is recorded in `manifest.json` — one run = one target.
+
+```
+.agentv/results/runs/<experiment>/<timestamp>/
+├── manifest.json                    ← eval metadata, target, test_ids
+├── index.jsonl                      ← per-test scores
+├── benchmark.json                   ← aggregate statistics
+└── <evalset-name>/                  ← eval.yaml "name" field, or eval file basename if absent (same as CLI mode)
+    └── <test-id>/                   ← test case id
+        ├── input.json               ← test input text + messages
+        ├── invoke.json              ← target command or agent instructions
+        ├── criteria.md              ← grading criteria
+        ├── response.md              ← target/agent output
+        ├── timing.json              ← execution timing
+        ├── code_graders/<name>.json     ← grader configs written by `pipeline input`: code-grader scripts AND built-in types (contains, regex, equals, etc.)
+        ├── llm_graders/<name>.json      ← LLM grader configs
+        ├── code_grader_results/<name>.json  ← code grader results
+        ├── llm_grader_results/<name>.json   ← LLM grader results (written by grader subagents; one file per grader)
+        └── grading.json              ← merged grading (written by `pipeline bench` — do NOT write here directly)
+```
+
+--- references/autoresearch.md ---
+# Autoresearch Mode
+
+Autoresearch is an unattended eval-improve loop that runs multiple optimize cycles without human intervention. The user triggers it with natural language (e.g., "run autoresearch on this skill", "optimize this skill unattended"). No YAML schema changes or CLI flags are needed.
+
+## Automated Keep/Discard
+
+After each iteration, you can automatically decide whether to keep or discard the change using structured comparison output. This replaces manual judgment at steps 3–4 of the iteration loop (Step 5 in SKILL.md), except at human checkpoint iterations (3, 6, 9) where you must still present results to the user.
+
+### 1. Run the comparison
+
+After re-running test cases, compare the new results against the previous iteration's baseline:
+
+```bash
+agentv compare <baseline>.jsonl <candidate>.jsonl --json
+```
+
+Where `<baseline>.jsonl` is the `index.jsonl` from the previous best iteration and `<candidate>.jsonl` is the `index.jsonl` from the run you just completed.
+
+### 2. Parse the output
+
+The `--json` flag produces structured output:
+
+```json
+{
+  "summary": {
+    "wins": 3,
+    "losses": 1,
+    "ties": 6,
+    "mean_delta": 0.05
+  }
+}
+```
+
+- **wins**: number of test cases where the candidate scored higher than the baseline
+- **losses**: number of test cases where the candidate scored lower
+- **ties**: number of test cases with no score change
+- **mean_delta**: average score difference across all test cases (positive = candidate is better)
+
+### 3. Apply decision rules
+
+Use these rules in order:
+
+| Condition | Decision | Action |
+|-----------|----------|--------|
+| `wins > losses` | **KEEP** | Promote the candidate to the new baseline. Copy or note its `index.jsonl` path as the baseline for the next iteration. |
+| `wins <= losses` | **DISCARD** | Revert the prompt/skill/config change. The previous baseline remains. Try a different mutation on the next iteration. |
+| `mean_delta == 0` AND candidate prompt is shorter (fewer lines) | **KEEP** | Simpler prompts are preferred when performance is equal. Promote the candidate as the new baseline. |
+
+When `mean_delta == 0` and the candidate prompt is *not* shorter, treat it as a **DISCARD** — there's no reason to keep a change that adds complexity without improving results.
+
+### 4. Log the decision
+
+Before proceeding to the next iteration, log the decision and rationale so the user can review later:
+
+```
+Iteration 2: KEEP
+  wins=3, losses=1, ties=6, meanDelta=+0.05
+  Rationale: candidate wins outweigh losses (3 > 1)
+  Baseline promoted: .agentv/results/runs/20250101-120000/index.jsonl
+```
+
+```
+Iteration 3: DISCARD
+  wins=1, losses=2, ties=7, meanDelta=-0.03
+  Rationale: candidate losses outweigh wins (2 > 1)
+  Reverted to baseline: .agentv/results/runs/20250101-110000/index.jsonl
+  Next: try a different mutation
+```
+
+Include this log in your progress summary. At human checkpoints (iterations 3, 6, 9), present the full log of automated decisions since the last checkpoint alongside the current results.
+
+### 5. Integration with the iteration loop
+
+The automated keep/discard replaces the manual compare-and-present cycle (steps 3–4) during non-checkpoint iterations. The full flow becomes:
+
+1. Apply change to prompts/skills/config
+2. Re-run all test cases
+3. Run `agentv compare baseline.jsonl candidate.jsonl --json`
+4. Apply keep/discard rules → promote or revert
+5. Log the decision
+6. If this is iteration 3, 6, or 9 → present progress to the user (human checkpoint)
+7. Check stop conditions → continue or stop
+
+Both modes coexist: if the user is actively reviewing results, present to them as before. If the user has asked you to iterate autonomously, use automated keep/discard and only pause at human checkpoints.
+
+---
+
+## Prerequisites
+
+- An eval file (`EVAL.yaml` or `evals.json`) must exist for the artifact being optimized.
+- The artifact must be a file or directory (SKILL.md, prompt template, agent config, or a directory of related files like a skill with references/).
+- The user should have run at least one interactive eval cycle to build confidence in eval quality before going unattended.
+
+## The loop
+
+```
+1. RUN EVAL   — agentv eval with current artifact
+2. ANALYZE    — dispatch analyzer subagent on results
+3. DECIDE     — if score > best_score: KEEP, else DROP (automated keep/discard above)
+4. MUTATE     — dispatch mutator subagent with failure analysis (agents/mutator.md)
+5. GOTO 1     — until convergence or max_cycles
+```
+
+## Experiment naming
+
+Derive the experiment name from the artifact: `autoresearch-<name>` (e.g., `autoresearch-pdf-skill`). The user can also provide a custom name.
+
+## Artifact mutation flow
+
+The mutator rewrites artifacts in the working tree in place. **Git is used for versioning** — HEAD always contains the best-known version:
+
+1. Record the starting commit SHA before the first cycle: `initial_sha=$(git rev-parse HEAD)`.
+2. On each **KEEP**: `git add <artifact-path> && git commit -m "autoresearch cycle N: <mutation summary>"`.
+3. On each **DROP**: `git checkout -- <artifact-path>` (restores working tree to HEAD, the last KEEP commit).
+4. The eval always runs against the real file path — no temp files or indirection.
+5. The mutator can reference the original via `git show <initial_sha>:<path>`.
+
+## How the skill invokes eval
+
+Shell out to `agentv eval <eval-path> --experiment autoresearch-<name>` via the Bash tool, same as the existing interactive bench workflow.
+
+## Artifact layout
+
+Each cycle is a standard eval run. Autoresearch session metadata lives in `_autoresearch/` within the experiment directory:
+
+```
+.agentv/results/runs/<experiment>/
+  _autoresearch/
+    iterations.jsonl               # one line per cycle — data for chart + mutator
+    trajectory.html                # live-updating score trajectory chart
+  2026-04-15T10-30-00/             # cycle 1 — standard run artifacts
+    index.jsonl
+    grading.json
+    timing.json
+    benchmark.json
+    report.html
+  2026-04-15T10-35-00/             # cycle 2 — standard run artifacts
+    ...
+```
+
+No `original.md` or `best.md` files — git history serves as the backup. The `_` prefix convention distinguishes workflow folders from timestamped run dirs.
+
+## iterations.jsonl
+
+One JSON object per line, one line per cycle:
+
+```jsonl
+{"cycle":1,"score":0.65,"decision":"keep","cost_usd":0.12,"assertions":{"IDENTIFIES_BUG":0.8,"SUGGESTS_FIX":0.4},"mutation":"added explicit null-check instruction","run_dir":"2026-04-15T10-30-00","timestamp":"2026-04-15T10:32:15Z"}
+```
+
+Fields: `cycle` (1-indexed), `score` (overall pass rate 0–1), `decision` ("keep" or "drop"), `cost_usd` (eval run cost), `assertions` (per-assertion pass rates), `mutation` (one-line description of what changed), `run_dir` (timestamped directory name), `timestamp` (ISO 8601).
+
+## trajectory.html
+
+A standalone HTML chart file with embedded Chart.js. Copy the template from `scripts/trajectory.html` into the `_autoresearch/` directory. It fetches `iterations.jsonl` from the same directory on each auto-refresh — no data injection needed. Shows:
+
+- Score over iterations (line chart) with KEEP (green) / DISCARD (red) markers
+- Per-assertion pass rates over iterations
+- Cumulative cost across iterations
+- Best vs original score summary
+
+Auto-refreshes every 2 seconds during the loop. Becomes static after completion (remove the auto-refresh meta tag on final update).
+
+## Convergence
+
+Stop after **3** consecutive cycles with no improvement (no KEEP). Also stop at **max_cycles** (default 10). Either limit can be overridden by the user.
+
+## Human checkpoints
+
+Autoresearch mode **skips** human checkpoints at iterations 3/6/9. The user opted in to unattended operation by requesting autoresearch.
+
+## Context hygiene
+
+The orchestrator must run indefinitely without exhausting its context window. To do this:
+
+- **Never read eval results, artifacts, or transcripts into your own context.** Use bash commands (jq, agentv CLI) that output small structured summaries.
+- **Delegate all heavy reading to subagents.** The mutator reads artifacts, grading results, and transcripts from disk — you pass it paths, not content.
+- **Use bash for all file I/O** in the loop body: appending to `iterations.jsonl`, git operations, score extraction. The only tool calls per cycle should be bash commands and one subagent dispatch (mutator).
+- **trajectory.html auto-loads `iterations.jsonl`** via fetch — no need to read or update the HTML file after initial copy.
+
+## Procedure
+
+Follow this step-by-step procedure to execute autoresearch:
+
+### 1. Setup
+
+1. Determine the **artifact path** (file or directory to optimize) and **eval path** (EVAL.yaml or evals.json).
+2. Detect **artifact mode**: `file` if the artifact path is a file, `directory` if it's a directory.
+3. Derive the **experiment name**: `autoresearch-<name>` from the artifact filename/dirname, or use a user-provided name.
+4. Set the experiment directory: `.agentv/results/runs/<experiment>/`.
+5. Create the `_autoresearch/` subdirectory inside the experiment directory.
+6. Record `initial_sha=$(git rev-parse HEAD)` — the commit before any mutations.
+7. Copy `scripts/trajectory.html` to `_autoresearch/trajectory.html`.
+8. Initialize variables:
+   - `best_score = 0`
+   - `convergence_count = 0`
+   - `cycle = 1`
+   - `max_cycles = 10` (or user-specified)
+   - `max_convergence = 3` (or user-specified)
+
+### 2. Main loop
 
-  "runs": [
-    {
-      "eval_id": 1,
-      "eval_name": "Ocean",
-      "configuration": "with_skill",
-      "run_number": 1,
-      "result": {
-        "pass_rate": 0.85,
-        "passed": 6,
-        "failed": 1,
-        "total": 7,
-        "time_seconds": 42.5,
-        "tokens": 3800,
-        "tool_calls": 18,
-        "errors": 0
-      },
-      "assertions": [
-        {"text": "...", "passed": true, "evidence": "..."}
-      ],
-      "notes": [
-        "Used 2023 data, may be stale",
-        "Fell back to text overlay for non-fillable fields"
-      ]
-    }
-  ],
+Repeat while `cycle <= max_cycles` and `convergence_count < max_convergence`:
 
-  "run_summary": {
-    "with_skill": {
-      "pass_rate": {"mean": 0.85, "stddev": 0.05, "min": 0.80, "max": 0.90},
-      "time_seconds": {"mean": 45.0, "stddev": 12.0, "min": 32.0, "max": 58.0},
-      "tokens": {"mean": 3800, "stddev": 400, "min": 3200, "max": 4100}
-    },
-    "without_skill": {
-      "pass_rate": {"mean": 0.35, "stddev": 0.08, "min": 0.28, "max": 0.45},
-      "time_seconds": {"mean": 32.0, "stddev": 8.0, "min": 24.0, "max": 42.0},
-      "tokens": {"mean": 2100, "stddev": 300, "min": 1800, "max": 2500}
-    },
-    "delta": {
-      "pass_rate": "+0.50",
-      "time_seconds": "+13.0",
-      "tokens": "+1700"
-    }
-  },
+**a. Run eval**
 
-  "notes": [
-    "Assertion 'Output is a PDF file' passes 100% in both configurations - may not differentiate skill value",
-    "Eval 3 shows high variance (50% ± 40%) - may be flaky or model-dependent",
-    "Without-skill runs consistently fail on table extraction assertions",
-    "Skill adds 13s average execution time but improves pass rate by 50%"
-  ]
-}
+```bash
+agentv eval <eval-path> --experiment autoresearch-<name>
 ```
 
-**Fields:**
-- `metadata`: Information about the benchmark run
-  - `skill_name`: Name of the skill
-  - `timestamp`: When the benchmark was run
-  - `evals_run`: List of eval names or IDs
-  - `runs_per_configuration`: Number of runs per config (e.g. 3)
-- `runs[]`: Individual run results
-  - `eval_id`: Numeric eval identifier
-  - `eval_name`: Human-readable eval name (used as section header in the viewer)
-  - `configuration`: Must be `"with_skill"` or `"without_skill"` (the viewer uses this exact string for grouping and color coding)
-  - `run_number`: Integer run number (1, 2, 3...)
-  - `result`: Nested object with `pass_rate`, `passed`, `total`, `time_seconds`, `tokens`, `errors`
-- `run_summary`: Statistical aggregates per configuration
-  - `with_skill` / `without_skill`: Each contains `pass_rate`, `time_seconds`, `tokens` objects with `mean` and `stddev` fields
-  - `delta`: Difference strings like `"+0.50"`, `"+13.0"`, `"+1700"`
-- `notes`: Freeform observations from the analyzer
+**b. Extract scores (bash only — do NOT read result files into your context)**
 
-**Important:** The viewer reads these field names exactly. Using `config` instead of `configuration`, or putting `pass_rate` at the top level of a run instead of nested under `result`, will cause the viewer to show empty/zero values. Always reference this schema when generating benchmark.json manually.
+Find the latest timestamped directory in the experiment folder. Use bash/jq to extract small structured values:
 
----
+```bash
+# Find latest run dir
+RUN_DIR=$(ls -td <experiment-dir>/20*/ | head -1)
 
-## comparison.json
+# Overall score (mean of all scores in index.jsonl)
+SCORE=$(jq -sr '[.[].scores[].score] | add / length' "$RUN_DIR/index.jsonl")
 
-Output from blind comparator. Located at `<grading-dir>/comparison-N.json`.
+# Per-assertion pass rates as JSON object
+PASS_RATES=$(jq -sr '[.[].scores[]] | group_by(.type) | map({key: .[0].type, value: (map(.score) | add / length)}) | from_entries' "$RUN_DIR/index.jsonl")
 
-```json
-{
-  "winner": "A",
-  "reasoning": "Output A provides a complete solution with proper formatting and all required fields. Output B is missing the date field and has formatting inconsistencies.",
-  "rubric": {
-    "A": {
-      "content": {
-        "correctness": 5,
-        "completeness": 5,
-        "accuracy": 4
-      },
-      "structure": {
-        "organization": 4,
-        "formatting": 5,
-        "usability": 4
-      },
-      "content_score": 4.7,
-      "structure_score": 4.3,
-      "overall_score": 9.0
-    },
-    "B": {
-      "content": {
-        "correctness": 3,
-        "completeness": 2,
-        "accuracy": 3
-      },
-      "structure": {
-        "organization": 3,
-        "formatting": 2,
-        "usability": 3
-      },
-      "content_score": 2.7,
-      "structure_score": 2.7,
-      "overall_score": 5.4
-    }
-  },
-  "output_quality": {
-    "A": {
-      "score": 9,
-      "strengths": ["Complete solution", "Well-formatted", "All fields present"],
-      "weaknesses": ["Minor style inconsistency in header"]
-    },
-    "B": {
-      "score": 5,
-      "strengths": ["Readable output", "Correct basic structure"],
-      "weaknesses": ["Missing date field", "Formatting inconsistencies", "Partial data extraction"]
-    }
-  },
-  "assertions": {
-    "A": {
-      "passed": 4,
-      "total": 5,
-      "pass_rate": 0.80,
-      "details": [
-        {"text": "Output includes name", "passed": true}
-      ]
-    },
-    "B": {
-      "passed": 3,
-      "total": 5,
-      "pass_rate": 0.60,
-      "details": [
-        {"text": "Output includes name", "passed": true}
-      ]
-    }
-  }
-}
+# Cost (if timing.json exists)
+COST=$(jq -r '.cost_usd // 0' "$RUN_DIR/timing.json" 2>/dev/null || echo 0)
 ```
 
----
+Capture only these small outputs (`SCORE`, `PASS_RATES`, `COST`) — never read the full JSONL into context.
 
-## analysis.json
+**c. Update iterations.jsonl (bash only)**
 
-Output from post-hoc analyzer. Located at `<grading-dir>/analysis.json`.
+After the KEEP/DROP decision (step e), append one JSON line via bash:
 
-```json
-{
-  "comparison_summary": {
-    "winner": "A",
-    "winner_skill": "path/to/winner/skill",
-    "loser_skill": "path/to/loser/skill",
-    "comparator_reasoning": "Brief summary of why comparator chose winner"
-  },
-  "winner_strengths": [
-    "Clear step-by-step instructions for handling multi-page documents",
-    "Included validation script that caught formatting errors"
-  ],
-  "loser_weaknesses": [
-    "Vague instruction 'process the document appropriately' led to inconsistent behavior",
-    "No script for validation, agent had to improvise"
-  ],
-  "instruction_following": {
-    "winner": {
-      "score": 9,
-      "issues": ["Minor: skipped optional logging step"]
-    },
-    "loser": {
-      "score": 6,
-      "issues": [
-        "Did not use the skill's formatting template",
-        "Invented own approach instead of following step 3"
-      ]
-    }
-  },
-  "improvement_suggestions": [
-    {
-      "priority": "high",
-      "category": "instructions",
-      "suggestion": "Replace 'process the document appropriately' with explicit steps",
-      "expected_impact": "Would eliminate ambiguity that caused inconsistent behavior"
-    }
-  ],
-  "transcript_insights": {
-    "winner_execution_pattern": "Read skill -> Followed 5-step process -> Used validation script",
-    "loser_execution_pattern": "Read skill -> Unclear on approach -> Tried 3 different methods"
-  }
-}
+```bash
+echo '{"cycle":'$CYCLE',"score":'$SCORE',"decision":"'$DECISION'","cost_usd":'$COST',"assertions":'$PASS_RATES',"mutation":"'"$MUTATION_DESC"'","run_dir":"'"$(basename $RUN_DIR)"'","timestamp":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'"}' >> <experiment-dir>/_autoresearch/iterations.jsonl
 ```
 
---- references/subagent-pipeline.md ---
-# Subagent Pipeline — Running eval.yaml without CLI
+**d. trajectory.html — no action needed**
+
+The trajectory chart fetches `iterations.jsonl` directly via HTTP on each auto-refresh. No file manipulation required after the initial copy in setup.
+
+**e. Decide: KEEP or DROP**
+
+Apply the automated keep/discard rules from the section above:
+
+1. Run `agentv compare <baseline>.jsonl <candidate>.jsonl --json` where `<baseline>` is the best iteration's `index.jsonl` (or the first run's `index.jsonl` for cycle 1) and `<candidate>` is this cycle's `index.jsonl`.
+2. If `wins > losses` → **KEEP**.
+3. If `wins <= losses` → **DISCARD**.
+4. If `mean_delta == 0` and the artifact is simpler → **KEEP** (simpler is better at equal performance). Simplicity: for files, compare line count; for directories, compare total size via `du -sb`.
+
+For cycle 1, there is no baseline to compare against — always **KEEP** the first cycle.
+
+**f. If KEEP**
+
+- Update `best_score` to this cycle's score.
+- Commit the artifact: `git add <artifact-path> && git commit -m "autoresearch cycle N: <mutation summary>"`.
+- Record the current `index.jsonl` path as the new baseline for future comparisons.
+- Reset `convergence_count = 0`.
+
+**g. If DROP**
+
+- Revert the working tree to HEAD: `git checkout -- <artifact-path>` (for files) or `git checkout -- <artifact-path>/` (for directories).
+- Increment `convergence_count`.
+
+**h. Check stop conditions**
+
+If `convergence_count >= max_convergence` or `cycle >= max_cycles` → break out of the loop.
+
+**i. Mutate**
+
+Dispatch the **mutator** subagent (`agents/mutator.md`) with:
+- `artifact-path`: the file or directory to mutate
+- `artifact-mode`: `file` or `directory`
+- `initial-sha`: the starting commit SHA (for referencing the original via `git show`)
+- `pass-rates`: the `$PASS_RATES` JSON object from step (b) (small — just assertion names and rates)
+- `run-dir`: path to this cycle's run directory (the mutator reads `grading.json` and transcripts itself)
+- `iterations-path`: path to `_autoresearch/iterations.jsonl` (the mutator reads mutation history itself)
+- For directory mode: `focus-files` (optional — files most likely contributing to failures, derived from assertion names)
+
+**Do NOT pass failure descriptions, transcripts, or grading content** to the mutator — pass paths and let it read what it needs from disk. This keeps the orchestrator's context clean.
+
+The mutator rewrites artifacts in place. Verify the artifact was modified (e.g., `git diff --stat`) before continuing.
+
+**j. Continue**
+
+Increment `cycle` and return to step (a).
+
+### 3. Completion
+
+1. Finalize `trajectory.html`: remove the line containing `<!-- __AUTO_REFRESH__ -->` (which includes the `<meta http-equiv="refresh">` tag) so the chart becomes static.
+2. Log a final summary:
+   - Total cycles run
+   - Final best score vs original score (cycle 1)
+   - Number of KEEPs and DROPs
+   - Total cost across all cycles
+   - The optimized artifact is in the working tree (and the latest commit)
+   - Run `git diff <initial_sha>` to see total changes from the original
+   - Run `git log --oneline <initial_sha>..HEAD` to see the mutation history
+   - Path to `_autoresearch/trajectory.html` (the score chart)
+3. Present results to the user with a recommendation: adopt the optimized version, revert to original (`git checkout <initial_sha> -- <artifact-path>`), or continue iterating interactively.
+
+## Interactive/autonomous hybrid
 
-This reference documents the detailed procedure for running evaluations in subagent mode
-(`AGENT_EVAL_MODE=subagent`, the default). The orchestrating skill dispatches `executor`
-subagents to perform test cases and `grader` subagents to evaluate outputs.
+Users can start in interactive mode (the existing Step 3–5 loop with human checkpoints), build confidence in their eval quality, and then switch to autoresearch mode to run unattended. The two modes share the same eval infrastructure and artifact layout — autoresearch simply automates the keep/discard decisions and removes human checkpoints.
 
-Read this reference when executing Step 3 (Run and Grade) in subagent mode.
+## Model empathy recommendation
 
-## Prerequisites
+For best results, use same-model pairings: the meta-agent running autoresearch should match the model used by the task agent being evaluated (e.g., Claude optimizing a Claude agent, GPT optimizing a GPT agent). Per AutoAgent research findings, same-model pairings produce better mutations because the optimizer has implicit knowledge of how the target model interprets instructions.
 
-- The eval.yaml file exists and contains valid test definitions
-- `agentv` CLI is installed (or run from source via `AGENTV_CLI=bun /path/to/cli.ts` in `.env`)
-- Read `references/eval-yaml-spec.md` for the full schema
+--- references/migrating-from-skill-creator.md ---
+# Migrating from Skill-Creator to AgentV Lifecycle Skill
 
-## Workspace Context
+This reference covers how to use AgentV's unified agent-evaluation lifecycle skill (`agentv-bench`) with evals.json files originally created for Anthropic's skill-creator.
 
-Some evals pass prompt files directly and don't require a specific workspace — those run fine
-from anywhere. But evals that test agent behavior in a workspace (accessing skills, modifying
-repos, using tools across multiple repos) require the user to be in the **target workspace**
-(e.g., a multi-repo workspace set up by allagents). If the eval references workspace files or
-expects the agent to use skills, check that the current directory is the target workspace, not
-just the eval repo — and warn the user if it's wrong.
+## Drop-in Replacement
 
-## Executor Subagent Eligibility
+AgentV runs skill-creator's evals.json directly — no conversion required:
 
-All providers except `cli` are eligible for executor subagents by default. To opt out a
-specific target, set `subagent_mode_allowed: false` in `.agentv/targets.yaml`:
+```bash
+# Run evals.json with AgentV
+agentv eval evals.json
 
-```yaml
-# .agentv/targets.yaml
-targets:
-  - name: my-target
-    provider: openai
-    model: ${{ OPENAI_MODEL }}
-    api_key: ${{ OPENAI_API_KEY }}
-    subagent_mode_allowed: false  # forces CLI invocation instead of executor subagent
+# Or run a single assertion offline (no API keys)
+agentv eval assert <grader-name> --agent-output "..." --agent-input "..."
 ```
 
-When `subagent_mode_allowed: false`, the target falls back to CLI invocation via `agentv eval`
-even in subagent mode.
+AgentV automatically:
+- Promotes `prompt` → input messages
+- Promotes `expected_output` → reference answer
+- Converts `assertions` → LLM-grader graders
+- Resolves `files[]` paths relative to the evals.json directory
 
-## CLI Targets: Single Command
+If you're using the `agentv-bench` skill, it orchestrates these same AgentV commands. Code graders, grading, and artifact generation remain in AgentV core; the skill just orchestrates and summarizes the existing outputs.
 
-For evals with CLI targets, `pipeline run` handles input extraction, target invocation, and
-code grading in one step. When `--out` is omitted, the output directory defaults to
-`.agentv/results/runs/<timestamp>` (same convention as `agentv eval`):
+## What You Gain
 
-```bash
-# Extract inputs and invoke all CLI targets in parallel:
-agentv pipeline run evals/repro.eval.yaml
+Moving from skill-creator's eval loop to AgentV's lifecycle skill gives you:
 
-# Also run code graders inline (instead of using pipeline grade separately):
-agentv pipeline run evals/repro.eval.yaml --grader-type code
-```
+| Capability | skill-creator | AgentV lifecycle skill |
+|-----------|---------------|----------------------|
+| Workspace isolation | ❌ | ✅ Clone repos, run setup/teardown scripts |
+| Code graders | ❌ | ✅ Python/TypeScript grader scripts via `defineCodeGrader()` |
+| Tool trajectory scoring | ❌ | ✅ Evaluate tool call sequences |
+| Multi-provider comparison | with-skill vs without-skill | N-way: Claude, GPT, Copilot, Gemini, custom CLI |
+| Multi-turn evaluation | ❌ | ✅ Conversation tracking with `conversation_id` |
+| Blind comparison | ❌ | ✅ Judge doesn't know which is baseline |
+| Deterministic upgrade suggestions | ❌ | ✅ LLM-grader → contains/regex/is-json |
+| Human review checkpoint | ❌ | ✅ Structured feedback gate |
+| Workspace file tracking | ❌ | ✅ Evaluate by diffing workspace files |
+| Agent mode (no API keys) | ❌ | ✅ Uses grader agent in agent mode |
 
-By default, `pipeline run` extracts inputs and invokes targets only. Pass `--grader-type code`
-to also run code-graders inline, or use `agentv pipeline grade <run-dir>` as a separate step.
+## Artifact Compatibility
 
-The run directory is printed to stdout. Then continue to the grading and merge phases
-described in SKILL.md Step 3.
+AgentV's companion artifacts are compatible with skill-creator's eval-viewer:
 
-## Non-CLI Targets: Executor Subagents
+| Artifact | Format | Compatible with eval-viewer |
+|----------|--------|---------------------------|
+| `<test-id>/grading.json` | Per-assertion evidence with claims | ✅ Superset of skill-creator's per-test grading format |
+| `benchmark.json` | Aggregate pass rates, timing, patterns | ✅ Superset of Agent Skills benchmark format |
+| Results JSONL | Per-test results | ✅ Standard JSONL format |
 
-When the target provider is not `cli`, check `manifest.json` → `target.subagent_mode_allowed`.
-If `true` (default for all non-CLI providers), the subagent IS the target. If `false` (user
-opted out via `subagent_mode_allowed: false` in `.agentv/targets.yaml`), fall back to
-`agentv eval` CLI mode instead.
+AgentV's schemas are supersets — they include all fields skill-creator expects, plus additional fields (claims extraction, pattern analysis, deterministic upgrade candidates). Tools that read skill-creator artifacts will read AgentV artifacts correctly, ignoring the extra fields.
 
-### Step 1: Extract inputs
+The optimizer scripts layer reads those same artifacts directly:
+- `aggregate-benchmark.ts` consumes `benchmark.json`, `timing.json`, and results JSONL
+- `generate-report.ts` and `eval-viewer/generate-review.ts` render review output from AgentV artifacts
+- `improve-description.ts` proposes follow-up experiments from benchmark/grading observations
+
+## Graduating to EVAL.yaml
+
+When evals.json becomes limiting, convert to EVAL.yaml for the full feature set:
 
 ```bash
-# Defaults to .agentv/results/runs/<timestamp>
-agentv pipeline input evals/repro.eval.yaml
+# Convert evals.json to EVAL.yaml
+agentv convert evals.json
+
+# Edit the generated YAML to add workspace config, code graders, etc.
+# Then run with the full lifecycle
+agentv eval eval.yaml
 ```
 
-This creates a run directory with per-test `input.json`, `invoke.json`,
-`criteria.md`, and grader configs.
+EVAL.yaml unlocks:
+- **Workspace setup/teardown** — clone repos, install dependencies, clean up after tests
+- **Code graders** — write graders in Python or TypeScript, not just LLM prompts
+- **Rubric-based grading** — multi-dimensional scoring with weighted criteria
+- **Retry policies** — automatic retries for flaky tests with configurable backoff
+- **Test groups** — organize tests by category with shared config
+- **Multi-turn conversations** — test agent interactions across multiple turns
 
-### Step 2: Dispatch executor subagents
+## What Stays in Skill-Creator
 
-Read `agents/executor.md`. Launch one `executor` subagent **per test case**, all in parallel.
-Each subagent receives the test directory path, reads `input.json`, performs the task using
-its own tools, and writes `response.md`.
+AgentV does NOT replace these skill-creator capabilities:
 
-Example: 5 tests = 5 executor subagents launched simultaneously.
+- **Trigger optimization** — optimizing when/how a skill is triggered
+- **.skill packaging** — bundling skills for distribution
+- **Skill authoring** — creating new SKILL.md files from scratch
+- **Skill discovery** — finding and installing skills
+
+AgentV focuses on the **evaluation and optimization loop**. Skill-creator focuses on **skill authoring and packaging**. They are complementary — use skill-creator to write the skill, use AgentV to evaluate and optimize it.
+
+## Example Workflow
 
 ```
-# Per executor subagent:
-#   - Reads <run-dir>/<test-id>/input.json
-#   - Performs the task
-#   - Writes <run-dir>/<test-id>/response.md
+1. Author a skill with skill-creator
+2. skill-creator generates evals.json
+3. Run evals.json through AgentV's lifecycle skill for richer evaluation:
+   - Workspace isolation (test in a real repo)
+   - Multi-provider comparison (does the skill work with GPT too?)
+   - Blind comparison (is the new version actually better?)
+   - Deterministic upgrades (replace vague LLM graders with precise checks)
+4. Use AgentV's optimization loop to refine the skill's prompts
+5. Return to skill-creator for packaging and distribution
 ```
 
-### Step 3 onward: Grade and merge
+--- scripts/trajectory.html ---
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <meta http-equiv="refresh" content="2"> <!-- __AUTO_REFRESH__ -->
+  <title>AgentV Autoresearch Trajectory</title>
+  <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
+  <style>
+    /*
+     * Visual language aligned with Studio DESIGN.md:
+     *   - bg-gray-950 (#030712) canvas, bg-gray-900 (#111827) containers
+     *   - border-gray-800 (#1f2937) borders, divide-gray-800/50 row dividers
+     *   - Cyan-only accent (#22d3ee), emerald/red for data status
+     *   - System sans-serif, tabular-nums on every number
+     *   - font-medium (500) max for headings, no bold (700)
+     *   - rounded-lg (8px) containers, rounded-md (6px) badges/buttons
+     *   - No drop shadows — borders carry elevation
+     */
+    *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
+
+    body {
+      font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont,
+                   "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
+      background: #030712;  /* bg-gray-950 */
+      color: #d1d5db;       /* text-gray-300 */
+      padding: 24px;
+      line-height: 1.5;
+      font-size: 0.875rem;  /* text-sm default */
+    }
 
-See SKILL.md Step 3 "Grading" section for the three-phase grading process (code graders →
-LLM grading → merge and validate).
+    h1 {
+      text-align: center;
+      font-size: 1.5rem;    /* text-2xl */
+      font-weight: 600;     /* font-semibold — ceiling */
+      color: #fff;          /* text-white */
+      margin-bottom: 8px;
+    }
 
-## Step-by-Step Fine-Grained Control (CLI targets)
+    .subtitle {
+      text-align: center;
+      font-size: 0.875rem;
+      color: #6b7280;       /* text-gray-500 */
+      margin-bottom: 24px;
+    }
 
-Use individual commands when you need control over each step with CLI targets:
+    /* Summary cards */
+    .summary {
+      display: grid;
+      grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
+      gap: 12px;
+      margin-bottom: 24px;  /* space-y-6 */
+      max-width: 900px;
+      margin-left: auto;
+      margin-right: auto;
+    }
 
-```bash
-# Step 1: Extract inputs (defaults to .agentv/results/runs/<timestamp>)
-agentv pipeline input evals/repro.eval.yaml
+    .card {
+      background: #111827;      /* bg-gray-900 */
+      border: 1px solid #1f2937; /* border-gray-800 */
+      border-radius: 8px;       /* rounded-lg */
+      padding: 14px 16px;
+      text-align: center;
+    }
 
-# Step 2: run_tests.py invokes CLI targets (or use pipeline run instead)
+    .card .label {
+      font-size: 0.75rem;       /* text-xs */
+      text-transform: uppercase;
+      letter-spacing: 0.05em;   /* tracking-wider */
+      color: #6b7280;           /* text-gray-500 */
+      margin-bottom: 4px;
+    }
 
-# Step 3: Run code graders
-agentv pipeline grade <run-dir>
+    .card .value {
+      font-size: 1.5rem;
+      font-weight: 500;         /* font-medium — NOT 700 */
+      color: #fff;
+      font-variant-numeric: tabular-nums;
+    }
 
-# Step 4: Subagent does LLM grading, writes results to llm_grader_results/<name>.json per test
+    .card .value.positive { color: #34d399; } /* text-emerald-400 */
+    .card .value.negative { color: #f87171; } /* text-red-400 */
+    .card .value.neutral  { color: #22d3ee; } /* text-cyan-400 */
+
+    /* Charts grid */
+    .charts {
+      display: grid;
+      grid-template-columns: repeat(auto-fit, minmax(380px, 1fr));
+      gap: 16px;                /* space-y-4 */
+      margin-bottom: 24px;
+      max-width: 1400px;
+      margin-left: auto;
+      margin-right: auto;
+    }
 
-# Step 5: Merge scores (writes index.jsonl with full scores[] for dashboard)
-agentv pipeline bench <run-dir>
+    .chart-container {
+      background: #111827;
+      border: 1px solid #1f2937;
+      border-radius: 8px;
+      padding: 16px;
+    }
 
-# Step 6: Validate
-agentv results validate <run-dir>
-```
+    .chart-container h2 {
+      font-size: 0.875rem;
+      font-weight: 500;         /* font-medium */
+      color: #9ca3af;           /* text-gray-400 */
+      margin-bottom: 12px;
+      text-align: center;
+    }
 
-## LLM Grading JSON Format
+    canvas { width: 100% !important; }
 
-The agent reads `llm_graders/<name>.json` for each test, grades the response using the prompt
-content, and produces a scores JSON:
+    /* Iterations table */
+    .table-section {
+      max-width: 1400px;
+      margin: 0 auto;
+    }
 
-```json
-{
-  "test-01": {
-    "relevance": {
-      "score": 0.85,
-      "assertions": [{"text": "Response is relevant", "passed": true, "evidence": "..."}]
+    .table-section h2 {
+      font-size: 1.25rem;       /* text-xl */
+      font-weight: 600;         /* font-semibold */
+      color: #fff;              /* text-white */
+      margin-bottom: 12px;
     }
+
+    .table-wrapper {
+      overflow-x: auto;
+      border-radius: 8px;       /* rounded-lg */
+      border: 1px solid #1f2937;
+    }
+
+    table {
+      width: 100%;
+      border-collapse: collapse;
+      font-size: 0.875rem;      /* text-sm */
+      text-align: left;
+    }
+
+    thead {
+      background: rgba(17, 24, 39, 0.5); /* bg-gray-900/50 */
+      border-bottom: 1px solid #1f2937;
+    }
+
+    th {
+      padding: 12px 16px;       /* px-4 py-3 */
+      text-align: left;
+      font-weight: 500;         /* font-medium */
+      color: #9ca3af;           /* text-gray-400 */
+      white-space: nowrap;
+    }
+
+    td {
+      padding: 12px 16px;       /* px-4 py-3 */
+      color: #d1d5db;           /* text-gray-300 */
+    }
+
+    tbody tr { border-top: 1px solid rgba(31, 41, 55, 0.5); } /* divide-gray-800/50 */
+    tbody tr:hover { background: rgba(17, 24, 39, 0.3); }     /* hover:bg-gray-900/30 */
+
+    td.num {
+      text-align: right;
+      font-variant-numeric: tabular-nums;
+      color: #9ca3af;           /* text-gray-400 for numbers */
+    }
+
+    .badge {
+      display: inline-block;
+      padding: 2px 8px;
+      border-radius: 6px;       /* rounded-md */
+      font-size: 0.75rem;
+      font-weight: 500;         /* font-medium */
+      text-transform: uppercase;
+    }
+
+    .badge.keep {
+      border: 1px solid rgba(6, 78, 59, 0.6);  /* border-emerald-900/60 */
+      background: rgba(6, 78, 59, 0.3);         /* bg-emerald-950/30 */
+      color: #34d399;                            /* text-emerald-400 */
+    }
+    .badge.discard {
+      border: 1px solid rgba(127, 29, 29, 0.6); /* border-red-900/60 */
+      background: rgba(127, 29, 29, 0.3);       /* bg-red-950/30 */
+      color: #f87171;                            /* text-red-400 */
+    }
+
+    .empty-state {
+      text-align: center;
+      padding: 32px 20px;       /* p-8 for empty states */
+      color: #4b5563;           /* text-gray-600 */
+      font-size: 0.875rem;
+    }
+
+    .mutation-cell {
+      max-width: 320px;
+      overflow: hidden;
+      text-overflow: ellipsis;
+      white-space: nowrap;
+    }
+  </style>
+</head>
+<body>
+
+<h1>AgentV Autoresearch Trajectory</h1>
+<div class="subtitle" id="subtitle"></div>
+
+<!-- Summary cards -->
+<div class="summary" id="summary"></div>
+
+<!-- Charts -->
+<div class="charts" id="charts-area">
+  <div class="chart-container">
+    <h2>Score over Iterations</h2>
+    <canvas id="scoreChart"></canvas>
+  </div>
+  <div class="chart-container">
+    <h2>Per-Assertion Pass Rates</h2>
+    <canvas id="assertionChart"></canvas>
+  </div>
+  <div class="chart-container">
+    <h2>Cumulative Cost (USD)</h2>
+    <canvas id="costChart"></canvas>
+  </div>
+</div>
+
+<!-- Iterations table -->
+<div class="table-section">
+  <h2>Iteration Log</h2>
+  <div class="table-wrapper" id="table-wrapper"></div>
+</div>
+
+<script>
+// ── Data loading ─────────────────────────────────────────────────────
+// Fetch iterations.jsonl from the same directory. Each line is one JSON object.
+// Auto-refreshes every 2s (via the meta tag) so the chart stays live.
+async function loadData() {
+  try {
+    const resp = await fetch('iterations.jsonl');
+    if (!resp.ok) return [];
+    const text = await resp.text();
+    return text.trim().split('\n').filter(Boolean).map(line => JSON.parse(line));
+  } catch (_) {
+    return [];
   }
 }
-```
-
-## Pipeline Bench and Dashboard
 
-`pipeline bench` merges LLM scores into `index.jsonl` with a full `scores[]` array per entry,
-matching the CLI-mode schema. The web dashboard (`agentv results serve`) reads this format
-directly — no separate conversion script is needed. Run `agentv results validate <run-dir>`
-to verify compatibility.
+// ── Helpers ────────────────────────────────────────────────────────────
+const fmtPct = (n) => (n == null ? '—' : Math.round(Number(n) * 100) + '%');
+const fmtPctDelta = (n) => (n == null ? '—' : (n >= 0 ? '+' : '') + Math.round(Number(n) * 100) + '%');
+const fmtUsd = (n) => (n == null ? '—' : '$' + Number(n).toFixed(4));
+
+// ── Chart.js defaults (aligned with Studio DESIGN.md) ─────────────────
+Chart.defaults.color = '#9ca3af';  // text-gray-400
+Chart.defaults.borderColor = 'rgba(31, 41, 55, 0.5)'; // border-gray-800/50
+Chart.defaults.font.family = 'ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif';
+
+// Cyan accent + Studio data-tone palette
+const CHART_COLORS = [
+  '#22d3ee', // cyan-400 (primary accent)
+  '#34d399', // emerald-400
+  '#f472b6', // pink-400
+  '#facc15', // yellow-400
+  '#c084fc', // purple-400
+  '#fb923c', // orange-400
+  '#a3e635', // lime-400
+  '#e879f9', // fuchsia-400
+  '#60a5fa', // blue-400
+  '#fbbf24', // amber-400
+];
+
+// ── Render ─────────────────────────────────────────────────────────────
+function render(data) {
+  if (!data || data.length === 0) {
+    document.getElementById('subtitle').textContent = 'Waiting for data…';
+    document.getElementById('charts-area').style.display = 'none';
+    document.getElementById('summary').innerHTML = '';
+    document.getElementById('table-wrapper').innerHTML =
+      '<div class="empty-state">No iterations recorded yet. The chart will update automatically.</div>';
+    return;
+  }
 
-## Output Structure
+  // Validate each iteration has at minimum a cycle and score
+  const valid = data.filter(it => it && typeof it.cycle === 'number' && typeof it.score === 'number');
+  if (valid.length === 0) {
+    document.getElementById('subtitle').textContent = 'Data present but missing required fields (cycle, score).';
+    document.getElementById('charts-area').style.display = 'none';
+    document.getElementById('summary').innerHTML = '';
+    document.getElementById('table-wrapper').innerHTML =
+      '<div class="empty-state">Iterations data is malformed. Each entry needs at minimum: cycle (number), score (number).</div>';
+    return;
+  }
+  const sorted = [...valid].sort((a, b) => a.cycle - b.cycle);
+
+  // ── Summary ──────────────────────────────────────────────────────────
+  const originalScore = sorted[0].score;
+  const bestIteration = sorted.reduce((best, it) => (it.score > best.score ? it : best), sorted[0]);
+  const bestScore = bestIteration.score;
+  const totalCycles = sorted.length;
+  const totalCost = sorted.reduce((sum, it) => sum + (parseFloat(it.cost_usd) || 0), 0);
+  const delta = bestScore - originalScore;
+
+  document.getElementById('subtitle').textContent =
+    `${totalCycles} cycle${totalCycles !== 1 ? 's' : ''} completed`;
+
+  const deltaClass = delta > 0 ? 'positive' : delta < 0 ? 'negative' : 'neutral';
+
+  document.getElementById('summary').innerHTML = `
+    <div class="card"><div class="label">Original Score</div><div class="value neutral">${fmtPct(originalScore)}</div></div>
+    <div class="card"><div class="label">Best Score</div><div class="value positive">${fmtPct(bestScore)}</div></div>
+    <div class="card"><div class="label">Improvement</div><div class="value ${deltaClass}">${fmtPctDelta(delta)}</div></div>
+    <div class="card"><div class="label">Total Cycles</div><div class="value">${totalCycles}</div></div>
+    <div class="card"><div class="label">Total Cost</div><div class="value">${fmtUsd(totalCost)}</div></div>
+  `;
+
+  // ── Chart data prep ──────────────────────────────────────────────────
+  const labels = sorted.map(it => it.cycle);
+  const scores = sorted.map(it => it.score);
+  const decisions = sorted.map(it => (it.decision || '').toLowerCase());
+
+  // Cumulative cost (parseFloat guards against string or null values)
+  let cumCost = 0;
+  const cumCosts = sorted.map(it => { cumCost += parseFloat(it.cost_usd) || 0; return cumCost; });
+
+  // Assertion keys (union of all)
+  const assertionKeys = [...new Set(sorted.flatMap(it => Object.keys(it.assertions || {})))].sort();
+
+  document.getElementById('charts-area').style.display = '';
+
+  // ── 1. Score chart (cyan line, emerald/red decision dots) ──────────
+  const scoresPct = scores.map(s => s * 100);
+  const scoreCtx = document.getElementById('scoreChart').getContext('2d');
+  new Chart(scoreCtx, {
+    type: 'line',
+    data: {
+      labels,
+      datasets: [{
+        label: 'Score',
+        data: scoresPct,
+        borderColor: '#22d3ee',                    // cyan-400
+        backgroundColor: 'rgba(34, 211, 238, 0.1)', // cyan-400/10
+        borderWidth: 2,
+        tension: 0.2,
+        fill: true,
+        pointRadius: 6,
+        pointBorderWidth: 2,
+        pointBackgroundColor: decisions.map(d => d === 'keep' ? '#34d399' : '#f87171'),
+        pointBorderColor: decisions.map(d => d === 'keep' ? '#065f46' : '#7f1d1d'),
+      }],
+    },
+    options: {
+      responsive: true,
+      scales: {
+        y: { min: 0, max: 100, ticks: { stepSize: 10, callback: v => v + '%' } },
+        x: { title: { display: true, text: 'Cycle' } },
+      },
+      plugins: {
+        legend: { display: false },
+        tooltip: {
+          callbacks: {
+            label: (ctx) => `Score: ${Math.round(ctx.parsed.y)}%`,
+            afterLabel: (ctx) => `Decision: ${decisions[ctx.dataIndex] || '—'}`,
+          },
+        },
+      },
+    },
+  });
+
+  // ── 2. Assertion chart ───────────────────────────────────────────────
+  const assertCtx = document.getElementById('assertionChart').getContext('2d');
+  new Chart(assertCtx, {
+    type: 'line',
+    data: {
+      labels,
+      datasets: assertionKeys.map((key, i) => ({
+        label: key,
+        data: sorted.map(it => { const v = (it.assertions || {})[key]; return v == null ? null : v * 100; }),
+        borderColor: CHART_COLORS[i % CHART_COLORS.length],
+        backgroundColor: 'transparent',
+        borderWidth: 2,
+        tension: 0.2,
+        pointRadius: 3,
+        spanGaps: true,
+      })),
+    },
+    options: {
+      responsive: true,
+      scales: {
+        y: { min: 0, max: 100, ticks: { stepSize: 20, callback: v => v + '%' } },
+        x: { title: { display: true, text: 'Cycle' } },
+      },
+      plugins: {
+        legend: { position: 'bottom', labels: { boxWidth: 12, padding: 10, font: { size: 11 } } },
+        tooltip: { callbacks: { label: (ctx) => `${ctx.dataset.label}: ${Math.round(ctx.parsed.y)}%` } },
+      },
+    },
+  });
+
+  // ── 3. Cost chart (cyan line) ─────────────────────────────────────
+  const costCtx = document.getElementById('costChart').getContext('2d');
+  new Chart(costCtx, {
+    type: 'line',
+    data: {
+      labels,
+      datasets: [{
+        label: 'Cumulative Cost (USD)',
+        data: cumCosts,
+        borderColor: '#22d3ee',                     // cyan-400
+        backgroundColor: 'rgba(34, 211, 238, 0.08)', // cyan-400/8
+        borderWidth: 2,
+        tension: 0.2,
+        fill: true,
+        pointRadius: 3,
+      }],
+    },
+    options: {
+      responsive: true,
+      scales: {
+        y: { ticks: { callback: v => '$' + v.toFixed(2) } },
+        x: { title: { display: true, text: 'Cycle' } },
+      },
+      plugins: { legend: { display: false } },
+    },
+  });
+
+  // ── Table ────────────────────────────────────────────────────────────
+  const rows = sorted.map(it => {
+    const dec = (it.decision || '').toLowerCase();
+    const badge = dec === 'keep'
+      ? '<span class="badge keep">keep</span>'
+      : '<span class="badge discard">drop</span>';
+    const ts = it.timestamp
+      ? new Date(it.timestamp).toLocaleString(undefined, { dateStyle: 'short', timeStyle: 'medium' })
+      : '—';
+    return `<tr>
+      <td class="num">${it.cycle}</td>
+      <td class="num">${fmtPct(it.score)}</td>
+      <td>${badge}</td>
+      <td class="mutation-cell" title="${(it.mutation || '').replace(/"/g, '&quot;')}">${it.mutation || '—'}</td>
+      <td class="num">${fmtUsd(it.cost_usd)}</td>
+      <td style="color:#6b7280">${ts}</td>
+    </tr>`;
+  }).join('');
+
+  document.getElementById('table-wrapper').innerHTML = `
+    <table>
+      <thead><tr>
+        <th style="text-align:right">Cycle</th>
+        <th style="text-align:right">Score</th>
+        <th>Decision</th>
+        <th>Mutation</th>
+        <th style="text-align:right">Cost</th>
+        <th>Timestamp</th>
+      </tr></thead>
+      <tbody>${rows}</tbody>
+    </table>
+  `;
+}
 
-The path hierarchy mirrors the CLI mode: `<evalset-name>` comes from the `name` field in
-the eval.yaml. The target is recorded in `manifest.json` — one run = one target.
+loadData().then(render);
+</script>
 
-```
-.agentv/results/runs/<experiment>/<timestamp>/
-├── manifest.json                    ← eval metadata, target, test_ids
-├── index.jsonl                      ← per-test scores
-├── benchmark.json                   ← aggregate statistics
-└── <evalset-name>/                  ← eval.yaml "name" field, or eval file basename if absent (same as CLI mode)
-    └── <test-id>/                   ← test case id
-        ├── input.json               ← test input text + messages
-        ├── invoke.json              ← target command or agent instructions
-        ├── criteria.md              ← grading criteria
-        ├── response.md              ← target/agent output
-        ├── timing.json              ← execution timing
-        ├── code_graders/<name>.json     ← grader configs written by `pipeline input`: code-grader scripts AND built-in types (contains, regex, equals, etc.)
-        ├── llm_graders/<name>.json      ← LLM grader configs
-        ├── code_grader_results/<name>.json  ← code grader results
-        ├── llm_grader_results/<name>.json   ← LLM grader results (written by grader subagents; one file per grader)
-        └── grading.json              ← merged grading (written by `pipeline bench` — do NOT write here directly)
-```
+</body>
+</html>
diff --git a/evals/self/skills/fixtures/agentv-bench.txt b/evals/agentv-dev/skills/fixtures/agentv-bench.txt
similarity index 96%
rename from evals/self/skills/fixtures/agentv-bench.txt
rename to evals/agentv-dev/skills/fixtures/agentv-bench.txt
index 67df5b625..21a8f7271 100644
--- a/evals/self/skills/fixtures/agentv-bench.txt
+++ b/evals/agentv-dev/skills/fixtures/agentv-bench.txt
@@ -175,6 +175,21 @@ When each subagent task completes, you receive a notification containing `total_
 
 This is the only opportunity to capture this data — it comes through the task notification and isn't persisted elsewhere. Process each notification as it arrives.
 
+### After all executors complete
+
+Once all executor subagent notifications arrive, **read `response.md` directly from disk**.
+Do NOT use `read_agent` to fetch executor results — they are already written to the run
+directory. Verify all responses exist before proceeding:
+
+```bash
+for d in <run-dir>/<evalset>/*/; do
+  echo "$(basename $d): $(ls "$d"/response.md 2>/dev/null && echo OK || echo MISSING)"
+done
+```
+
+If any `response.md` is MISSING, re-run that specific executor subagent. Do not proceed to
+grading until all responses are present.
+
 ### Grading
 
 **In CLI mode**, `agentv eval` handles all grading end-to-end — no manual phases needed.
@@ -426,3 +441,19 @@ Repeating the core loop for emphasis:
 - Repeat until you and the user are satisfied
 
 Take your time with improvements. Read the transcripts. Understand why failures happened. Make changes that generalize beyond the test set. This is important work.
+
+## Accessing reference files
+
+To load a specific reference without pulling the entire skill into context:
+
+```bash
+agentv skills get agentv-bench --ref eval-yaml-spec
+```
+
+Or resolve the skill directory and read files directly:
+
+```bash
+cat $(agentv skills path agentv-bench)/references/eval-yaml-spec.md
+```
+
+Use `--full` to retrieve every file in the skill at once.
diff --git a/evals/agentv-dev/skills/fixtures/agentv-eval-review-full.txt b/evals/agentv-dev/skills/fixtures/agentv-eval-review-full.txt
new file mode 100644
index 000000000..ba93b7b22
--- /dev/null
+++ b/evals/agentv-dev/skills/fixtures/agentv-eval-review-full.txt
@@ -0,0 +1,294 @@
+---
+name: agentv-eval-review
+description: >-
+  Use when reviewing eval YAML files for quality issues, linting eval files before
+  committing, checking eval schema compliance, or when asked to "review these evals",
+  "check eval quality", "lint eval files", or "validate eval structure".
+  Do NOT use for writing evals (use agentv-eval-writer) or running evals (use agentv-bench).
+---
+
+# Eval Review
+
+## Overview
+
+Lint and review AgentV eval YAML files for structural issues, schema compliance, and quality problems. Apply this checklist deterministically first, then layer LLM judgment for semantic issues a checklist cannot catch.
+
+## Process
+
+### Step 1: Structural checklist
+
+Walk every target eval file and report violations grouped by severity (error > warning > info). For each finding, include the file path and a concrete fix.
+
+- File extension is `.eval.yaml` (error if not).
+- `description` field is present at the top level (error if missing).
+- Each entry under `tests` has `id`, `input`, and at least one of `criteria` / `expected_output` / `assertions` (error if missing).
+- File-typed inputs (`type: file`) use a leading `/` in their `path` (error if relative).
+- Tests have an `assertions` block — flag tests that rely solely on `expected_output` (warning).
+- Detect `expected_output` prose patterns like "The agent should…" or "The output is…" (warning — prose belongs in `criteria`, structured matches in `assertions`).
+- Identical file inputs repeated across multiple tests in the same eval should be hoisted to a top-level `input` (info).
+- Eval files in the same directory should share a common `id` prefix (info — flag drift).
+
+### Step 2: Semantic review (LLM judgment)
+
+The structural checklist catches mechanical issues but cannot assess:
+- **Factual accuracy** — Do tool/command names in expected_output match what the skill documents?
+- **Coverage gaps** — Are important edge cases missing?
+- **Assertion discriminability** — Would assertions pass for both good and bad output?
+- **Cross-file consistency** — Do output filenames match across evals and skills?
+
+Read the relevant SKILL.md files and cross-check against the eval content for these issues.
+
+## Accessing reference files
+
+To load a specific reference without pulling the entire skill into context:
+
+```bash
+agentv skills get agentv-eval-review --ref <filename>
+```
+
+Or resolve the skill directory and read files directly:
+
+```bash
+cat $(agentv skills path agentv-eval-review)/references/<filename>.md
+```
+
+--- scripts/lint_eval.py ---
+#!/usr/bin/env python3
+"""Lint AgentV eval YAML files for common issues.
+
+Usage: python lint_eval.py <path-to-eval-dir-or-file> [--json]
+
+Checks:
+  - File uses .eval.yaml extension
+  - description field present
+  - Each test has id, input, criteria
+  - File paths in type:file use leading /
+  - assertions blocks present (not relying solely on expected_output)
+  - expected_output does not contain evaluation criteria prose
+  - Repeated file inputs across tests (should use top-level input)
+  - Naming prefix consistency across eval files in same directory
+
+Exit code: 0 if no issues, 1 if issues found.
+"""
+
+import json
+import os
+import re
+import sys
+from pathlib import Path
+
+try:
+    import yaml
+except ImportError:
+    # Fall back to basic YAML parsing if PyYAML not available
+    yaml = None
+
+
+def parse_yaml_basic(text: str) -> dict:
+    """Minimal YAML-ish parser for when PyYAML is unavailable."""
+    # This is a best-effort fallback; recommend installing PyYAML
+    import ast
+    # Try json first (YAML is a superset of JSON)
+    try:
+        return json.loads(text)
+    except Exception:
+        pass
+    return {}
+
+
+def load_yaml(path: Path) -> dict:
+    text = path.read_text(encoding="utf-8")
+    if yaml:
+        return yaml.safe_load(text) or {}
+    return parse_yaml_basic(text)
+
+
+def lint_file(path: Path) -> list[dict]:
+    issues = []
+
+    def issue(severity: str, msg: str, line: int | None = None):
+        issues.append({"file": str(path), "severity": severity, "message": msg, "line": line})
+
+    # Check extension
+    if not path.name.endswith(".eval.yaml"):
+        issue("error", f"File should use .eval.yaml extension, got: {path.name}")
+
+    try:
+        data = load_yaml(path)
+    except Exception as e:
+        issue("error", f"Failed to parse YAML: {e}")
+        return issues
+
+    if not isinstance(data, dict):
+        issue("error", "Root element is not a mapping")
+        return issues
+
+    # Check description
+    if "description" not in data:
+        issue("warning", "Missing top-level 'description' field")
+
+    tests = data.get("tests", [])
+    if not isinstance(tests, list):
+        issue("error", "'tests' is not a list")
+        return issues
+
+    if not tests:
+        issue("warning", "No tests defined")
+        return issues
+
+    # Check for top-level input (shared file references)
+    top_level_input = data.get("input")
+
+    # Collect file values across tests to detect repetition
+    file_values_per_test: list[list[str]] = []
+
+    for i, test in enumerate(tests):
+        test_id = test.get("id", f"test-{i}")
+
+        if "id" not in test:
+            issue("error", f"Test at index {i} missing 'id'")
+
+        if "input" not in test and top_level_input is None:
+            issue("error", f"Test '{test_id}' missing 'input' and no top-level input defined")
+
+        has_criteria = "criteria" in test
+        has_expected = "expected_output" in test
+        has_assertions = "assertions" in test
+
+        if not has_criteria and not has_expected and not has_assertions:
+            issue("error", f"Test '{test_id}' needs at least one of: criteria, expected_output, assertions")
+
+        # Check assertions present
+        if not has_assertions and has_expected:
+            issue("warning", f"Test '{test_id}' has expected_output but no assertions — add deterministic assertions where possible")
+
+        # Check expected_output for prose patterns
+        if has_expected:
+            expected = test["expected_output"]
+            expected_text = ""
+            if isinstance(expected, str):
+                expected_text = expected
+            elif isinstance(expected, list):
+                for msg in expected:
+                    if isinstance(msg, dict):
+                        content = msg.get("content", "")
+                        if isinstance(content, str):
+                            expected_text += content
+
+            prose_patterns = [
+                r"[Tt]he agent should",
+                r"[Ss]hould identify",
+                r"[Ss]hould flag",
+                r"[Ss]hould recommend",
+                r"[Ss]hould produce",
+                r"[Ss]hould detect",
+                r"[Ss]hould load",
+                r"[Ss]hould run",
+            ]
+            for pat in prose_patterns:
+                if re.search(pat, expected_text):
+                    issue("warning", f"Test '{test_id}' expected_output contains evaluation criteria prose ('{pat.lstrip('[Tt]').lstrip('[Ss]')}...') — use criteria or assertions instead")
+                    break
+
+        # Collect file values from input
+        test_files = extract_file_values(test.get("input", []))
+        file_values_per_test.append(test_files)
+
+        # Check file paths for leading /
+        for fv in test_files:
+            if not fv.startswith("/"):
+                issue("warning", f"Test '{test_id}' file path missing leading '/': {fv}")
+
+    # Check for repeated file inputs
+    if len(file_values_per_test) >= 2 and not top_level_input:
+        common_files = set(file_values_per_test[0])
+        for fvs in file_values_per_test[1:]:
+            common_files &= set(fvs)
+        if common_files:
+            issue("info", f"File input repeated in every test: {', '.join(sorted(common_files))} — consider using top-level input")
+
+    return issues
+
+
+def extract_file_values(input_data) -> list[str]:
+    """Extract type:file values from input structure."""
+    files = []
+    if isinstance(input_data, list):
+        for item in input_data:
+            if isinstance(item, dict):
+                content = item.get("content", [])
+                if isinstance(content, list):
+                    for c in content:
+                        if isinstance(c, dict) and c.get("type") == "file":
+                            v = c.get("value", "")
+                            if v:
+                                files.append(v)
+    return files
+
+
+def lint_directory(path: Path) -> list[dict]:
+    issues = []
+    eval_files = sorted(path.rglob("*.yaml")) + sorted(path.rglob("*.yml"))
+
+    if not eval_files:
+        issues.append({"file": str(path), "severity": "warning", "message": "No eval files found", "line": None})
+        return issues
+
+    # Check naming prefix consistency
+    prefixes = set()
+    for f in eval_files:
+        name = f.stem.replace(".eval", "")
+        parts = name.split("-")
+        if len(parts) >= 2:
+            prefixes.add(parts[0])
+
+    if len(prefixes) > 1:
+        issues.append({
+            "file": str(path),
+            "severity": "info",
+            "message": f"Inconsistent naming prefixes: {', '.join(sorted(prefixes))}",
+            "line": None,
+        })
+
+    for f in eval_files:
+        issues.extend(lint_file(f))
+
+    return issues
+
+
+def main():
+    if len(sys.argv) < 2:
+        print(f"Usage: {sys.argv[0]} <path> [--json]", file=sys.stderr)
+        sys.exit(2)
+
+    target = Path(sys.argv[1])
+    output_json = "--json" in sys.argv
+
+    if target.is_file():
+        issues = lint_file(target)
+    elif target.is_dir():
+        issues = lint_directory(target)
+    else:
+        print(f"Error: {target} not found", file=sys.stderr)
+        sys.exit(2)
+
+    if output_json:
+        print(json.dumps(issues, indent=2))
+    else:
+        for iss in issues:
+            line = f":{iss['line']}" if iss.get("line") else ""
+            print(f"[{iss['severity'].upper()}] {iss['file']}{line}: {iss['message']}")
+
+        counts = {}
+        for iss in issues:
+            counts[iss["severity"]] = counts.get(iss["severity"], 0) + 1
+        if issues:
+            print(f"\n{len(issues)} issues: {', '.join(f'{v} {k}' for k, v in sorted(counts.items()))}")
+        else:
+            print("No issues found.")
+
+    sys.exit(1 if any(i["severity"] == "error" for i in issues) else 0)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/evals/agentv-dev/skills/fixtures/agentv-eval-review.txt b/evals/agentv-dev/skills/fixtures/agentv-eval-review.txt
new file mode 100644
index 000000000..6b1752b10
--- /dev/null
+++ b/evals/agentv-dev/skills/fixtures/agentv-eval-review.txt
@@ -0,0 +1,53 @@
+---
+name: agentv-eval-review
+description: >-
+  Use when reviewing eval YAML files for quality issues, linting eval files before
+  committing, checking eval schema compliance, or when asked to "review these evals",
+  "check eval quality", "lint eval files", or "validate eval structure".
+  Do NOT use for writing evals (use agentv-eval-writer) or running evals (use agentv-bench).
+---
+
+# Eval Review
+
+## Overview
+
+Lint and review AgentV eval YAML files for structural issues, schema compliance, and quality problems. Apply this checklist deterministically first, then layer LLM judgment for semantic issues a checklist cannot catch.
+
+## Process
+
+### Step 1: Structural checklist
+
+Walk every target eval file and report violations grouped by severity (error > warning > info). For each finding, include the file path and a concrete fix.
+
+- File extension is `.eval.yaml` (error if not).
+- `description` field is present at the top level (error if missing).
+- Each entry under `tests` has `id`, `input`, and at least one of `criteria` / `expected_output` / `assertions` (error if missing).
+- File-typed inputs (`type: file`) use a leading `/` in their `path` (error if relative).
+- Tests have an `assertions` block — flag tests that rely solely on `expected_output` (warning).
+- Detect `expected_output` prose patterns like "The agent should…" or "The output is…" (warning — prose belongs in `criteria`, structured matches in `assertions`).
+- Identical file inputs repeated across multiple tests in the same eval should be hoisted to a top-level `input` (info).
+- Eval files in the same directory should share a common `id` prefix (info — flag drift).
+
+### Step 2: Semantic review (LLM judgment)
+
+The structural checklist catches mechanical issues but cannot assess:
+- **Factual accuracy** — Do tool/command names in expected_output match what the skill documents?
+- **Coverage gaps** — Are important edge cases missing?
+- **Assertion discriminability** — Would assertions pass for both good and bad output?
+- **Cross-file consistency** — Do output filenames match across evals and skills?
+
+Read the relevant SKILL.md files and cross-check against the eval content for these issues.
+
+## Accessing reference files
+
+To load a specific reference without pulling the entire skill into context:
+
+```bash
+agentv skills get agentv-eval-review --ref <filename>
+```
+
+Or resolve the skill directory and read files directly:
+
+```bash
+cat $(agentv skills path agentv-eval-review)/references/<filename>.md
+```
diff --git a/evals/self/skills/fixtures/agentv-eval-writer-full.txt b/evals/agentv-dev/skills/fixtures/agentv-eval-writer-full.txt
similarity index 99%
rename from evals/self/skills/fixtures/agentv-eval-writer-full.txt
rename to evals/agentv-dev/skills/fixtures/agentv-eval-writer-full.txt
index ce1e9979a..079c7fbe2 100644
--- a/evals/self/skills/fixtures/agentv-eval-writer-full.txt
+++ b/evals/agentv-dev/skills/fixtures/agentv-eval-writer-full.txt
@@ -730,6 +730,55 @@ cat $(agentv skills path agentv-eval-writer)/references/eval-schema.json
 
 Use `--full` to retrieve every file in the skill at once.
 
+--- references/python-helpers.md ---
+# Repo-Local Python Helpers
+
+AgentV's Python authoring surface is currently a repo-local helper example under `examples/features/sdk-python/`.
+
+Use it when the user wants Python-based custom graders or wants to emit AgentV YAML/JSONL from Python without introducing a Python-native runner.
+
+## Rules
+
+- Prefer canonical AgentV wire and YAML fields.
+- Do not accept deprecated wire aliases like `output_text`, `input_text`, or `reference_answer`.
+- Keep Python eval authoring YAML-shaped. Mirror `execution`, `tests`, `assertions`, `expected_output`, and related AgentV keys directly.
+- Run evals through the AgentV CLI, not through a separate Python runtime.
+
+## Available helpers
+
+- `agentv_py.grader`
+  - `load_grader_input()`
+  - `run_code_grader(handler)`
+  - `define_code_grader(handler)`
+  - `TargetClient.from_env()`
+- `agentv_py.evals`
+  - `EvalDefinition`
+  - `EvalTest`
+  - `JsonlCase`
+  - `write_eval_yaml()`
+  - `write_jsonl()`
+  - `run_agentv_eval()`
+
+## Example
+
+```python
+from agentv_py.grader import Assertion, CodeGraderResult, define_code_grader
+
+
+def evaluate(context):
+    actual = context.output or ""
+    expected = context.expected_output[0]["content"]
+    passed = actual.strip() == expected.strip()
+    return CodeGraderResult(
+        score=1.0 if passed else 0.0,
+        assertions=[Assertion(text="Exact match", passed=passed)],
+    )
+
+
+if __name__ == "__main__":
+    define_code_grader(evaluate)
+```
+
 --- references/eval-schema.json ---
 {
   "$schema": "https://json-schema.org/draft/2019-09/schema#",
@@ -18142,12 +18191,11 @@ Gating: checklist uses `required: true`, score-range uses `min_score: N` (0–1
   "assertions": [
     { "text": "passed check", "passed": true },
     { "text": "failed check", "passed": false }
-  ],
-  "reasoning": "explanation"
+  ]
 }
 ```
 
-`score` (0.0-1.0) required. `assertions`, `reasoning` optional.
+`score` (0.0-1.0) required. `assertions` and `details` optional.
 
 ## SDK Functions
 
@@ -18163,29 +18211,29 @@ import { defineCodeGrader, createTargetClient, definePromptTemplate } from '@age
   - Raw stdin uses `snake_case`; SDK handlers receive `camelCase`
   - Context fields: `input`, `expectedOutput`, `output`, `messages`, `criteria`, `config`, `trace`, `traceSummary`, `tokenUsage`, `costUsd`, `durationMs`, `startTime`, `endTime`
 
+For Python, the repo-local helper example in `examples/features/sdk-python/` keeps canonical `snake_case` fields and rejects deprecated wire aliases like `output_text`, `input_text`, and `reference_answer`. It is not a separate Python runner; generated evals still run through the AgentV CLI.
+
 ## Python Example
 
 ```python
 #!/usr/bin/env python3
-import json, sys
+from agentv_py.grader import Assertion, CodeGraderResult, define_code_grader
+
 
-def evaluate(data: dict) -> dict:
-    candidate = data.get("output", "")
+def evaluate(context):
+    candidate = context.output or ""
     assertions = []
     for kw in ["async", "await"]:
-        assertions.append({"text": f"Keyword '{kw}'", "passed": kw in candidate})
-    passed = sum(1 for a in assertions if a["passed"])
-    return {
-        "score": passed / max(len(assertions), 1),
-        "assertions": assertions,
-    }
+        assertions.append(Assertion(text=f"Keyword '{kw}'", passed=kw in candidate))
+    passed = sum(1 for item in assertions if item.passed)
+    return CodeGraderResult(
+        score=passed / max(len(assertions), 1),
+        assertions=assertions,
+    )
+
 
 if __name__ == "__main__":
-    try:
-        print(json.dumps(evaluate(json.loads(sys.stdin.read()))))
-    except Exception as e:
-        print(json.dumps({"score": 0, "assertions": [{"text": str(e), "passed": False}]}))
-        sys.exit(1)
+    define_code_grader(evaluate)
 ```
 
 ## TypeScript Example
@@ -18195,9 +18243,9 @@ if __name__ == "__main__":
 import { defineCodeGrader } from '@agentv/sdk';
 
 export default defineCodeGrader(({ output, criteria }) => {
-  const answer = output ?? '';
+  const candidate = output ?? '';
   const assertions: Array<{ text: string; passed: boolean }> = [];
-  if (answer.includes(criteria)) {
+  if (candidate.includes(criteria)) {
     assertions.push({ text: 'Matches expected outcome', passed: true });
   } else {
     assertions.push({ text: 'Does not match expected outcome', passed: false });
diff --git a/evals/self/skills/fixtures/agentv-eval-writer.txt b/evals/agentv-dev/skills/fixtures/agentv-eval-writer.txt
similarity index 100%
rename from evals/self/skills/fixtures/agentv-eval-writer.txt
rename to evals/agentv-dev/skills/fixtures/agentv-eval-writer.txt
diff --git a/evals/self/skills/fixtures/agentv-governance-full.txt b/evals/agentv-dev/skills/fixtures/agentv-governance-full.txt
similarity index 98%
rename from evals/self/skills/fixtures/agentv-governance-full.txt
rename to evals/agentv-dev/skills/fixtures/agentv-governance-full.txt
index 87ccc2095..001223eb6 100644
--- a/evals/self/skills/fixtures/agentv-governance-full.txt
+++ b/evals/agentv-dev/skills/fixtures/agentv-governance-full.txt
@@ -62,44 +62,109 @@ The expected output format is:
 5. Run the lint rules from `references/lint-rules.md` against your block before committing.
 6. See `references/governance-yaml-shape.md` for complete examples copied from real suites.
 
---- references/eu-ai-act-risk-tiers.md ---
-# EU AI Act — Risk Tiers
+## Accessing reference files
 
-**Valid values for the `risk_tier:` field.**
+To load a specific reference without pulling the entire skill into context:
 
-Official source: Regulation (EU) 2024/1689 on Artificial Intelligence (EU AI Act)
-Full text: https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX:32024R1689
+```bash
+agentv skills get agentv-governance --ref lint-rules
+```
 
-## Allowed values
+Or resolve the skill directory and read files directly:
 
-| Value | EU AI Act category | Key articles | Description |
-|-------|-------------------|-------------|-------------|
-| `prohibited` | Prohibited AI practices | Art. 5 | AI systems whose risks are deemed unacceptable — banned outright. Examples: social scoring by public authorities, real-time remote biometric surveillance in public spaces, AI that exploits vulnerabilities of specific groups. |
-| `high` | High-risk AI systems | Art. 6, Annex I–III | AI systems subject to mandatory conformity assessments, transparency, and human oversight. Examples: biometric identification, critical infrastructure, employment screening, access to education or essential services, law enforcement. |
-| `limited` | Limited-risk AI systems | Art. 50 | AI systems with transparency obligations only. Examples: chatbots must disclose they are AI; deep-fake generators must mark synthetic media. |
-| `minimal` | Minimal-risk AI systems | — | No mandatory obligations. Examples: spam filters, AI in video games. Voluntary codes of conduct encouraged. |
+```bash
+cat $(agentv skills path agentv-governance)/references/lint-rules.md
+```
 
-## Usage notes
+Use `--full` to retrieve every framework reference in one shot.
 
-- `risk_tier` is a scalar; only one value per governance block.
-- The vocabulary is anchored to EU AI Act terminology. Some organizations use different
-  risk scales (e.g. NIST SP 800-30 `low | moderate | high | very_high`). When mapping
-  from another framework, choose the EU AI Act equivalent that best matches the impact.
-- Combine `risk_tier: high` with `controls` referencing EU AI Act articles:
-  ```yaml
-  risk_tier: high
-  controls:
-    - EU-AI-ACT-2024:Art.55
-    - EU-AI-ACT-2024:Art.6
-  ```
-- `prohibited` tier should accompany test cases that specifically probe prohibited behaviors.
-  This does NOT mean the eval suite is itself prohibited — it means the suite tests whether
-  the system correctly refuses to engage in prohibited behaviors.
+--- references/iso-42001-controls.md ---
+# ISO/IEC 42001:2023 — AI Management System Controls
 
-## Article reference format
+**Curated subset of controls relevant to AI evaluation suites.**
 
-Use `EU-AI-ACT-2024:<Article>` in the `controls` array, e.g. `EU-AI-ACT-2024:Art.55`.
-Article 55 covers general-purpose AI (GPAI) model obligations and transparency requirements.
+Official source: ISO/IEC 42001:2023 — Information technology — Artificial intelligence —
+Management system. Full standard available at https://www.iso.org/standard/81230.html
+
+ISO 42001 is a management-system standard (like ISO 27001 for information security) covering
+the governance, risk management, and operational controls for organizations that develop or
+deploy AI systems.
+
+## Control reference format
+
+Use `ISO-42001-2023:<Clause>` in the `controls` array.
+
+## Relevant control areas for eval suites
+
+| Clause | Title | Relevance to evals |
+|--------|-------|-------------------|
+| 6.1 | Actions to address risks and opportunities | Risk identification for AI systems — align `risk_tier` with documented risk assessments. |
+| 6.1.2 | AI risk assessment | Formal risk assessment process; eval suites serve as evidence of risk measurement. |
+| 8.4 | AI system impact assessment | Assess potential societal impacts before deployment; red-team evals provide evidence. |
+| 8.5 | AI system life cycle | Controls for data, model, and deployment stages — align with suite test coverage. |
+| 9.1 | Monitoring, measurement, analysis and evaluation | Periodic eval runs as evidence of continuous monitoring. |
+| 9.1.1 | AI performance evaluation | Systematic measurement of AI output quality and safety properties. |
+| 10.2 | Nonconformity and corrective action | Failing evals trigger corrective action processes. |
+| A.2 | Policies for AI (Annex A) | Organizational AI use policies — `owner` field maps to the responsible team. |
+| A.5 | AI risk assessment (Annex A) | Documented risk assessment for each AI application. |
+| A.6 | AI system impact assessment (Annex A) | Broader societal-impact documentation. |
+
+## Usage example
+
+```yaml
+controls:
+  - ISO-42001-2023:6.1.2   # AI risk assessment
+  - ISO-42001-2023:9.1.1   # AI performance evaluation
+  - EU-AI-ACT-2024:Art.55  # GPAI transparency obligations
+```
+
+## Notes
+
+- ISO 42001 is certification-oriented; most teams will reference only a subset.
+  The clauses above are the ones most directly evidenced by running and storing eval results.
+- For pure LLM / red-team suites, clauses 6.1.2, 8.4, and 9.1.1 are the most common references.
+- Combine with NIST AI RMF controls (e.g. `NIST-AI-RMF-1.0:MEASURE-2.7`) when the organization
+  uses both frameworks.
+
+--- references/mitre-atlas.md ---
+# MITRE ATLAS — AI/ML Threat Techniques
+
+**Canonical IDs for use in `mitre_atlas:` arrays.**
+
+Official source: https://atlas.mitre.org/
+
+MITRE ATLAS (Adversarial Threat Landscape for Artificial-Intelligence Systems) documents
+adversarial ML and AI attack techniques using the same taxonomy style as MITRE ATT&CK.
+IDs follow the pattern `AML.Txxxx` for techniques and `AML.Txxxx.xxx` for sub-techniques.
+
+## Techniques most relevant to LLM / agentic-AI evaluation
+
+| ID | Name | Relevant OWASP IDs |
+|----|------|-------------------|
+| AML.T0051 | LLM Prompt Injection | LLM01, T01 |
+| AML.T0054 | LLM Jailbreak | LLM01 |
+| AML.T0056 | LLM Meta Prompt Extraction | LLM07 |
+| AML.T0057 | LLM Plugin Compromise | LLM03, T09 |
+| AML.T0058 | LLM Data Leakage | LLM02 |
+| AML.T0068 | Training Data Poisoning | LLM04 |
+| AML.T0075 | Manipulate LLM Inputs | LLM01, T01 |
+
+## Sub-techniques
+
+Sub-techniques extend a base ID with a period-separated suffix, e.g.:
+- `AML.T0051.000` — Direct Prompt Injection
+- `AML.T0051.001` — Indirect Prompt Injection
+
+Use the base ID if the test covers the whole technique class; use sub-techniques for
+more precise tagging when the attack method is specific.
+
+## Usage notes
+
+- List IDs as strings in an array: `mitre_atlas: [AML.T0051, AML.T0075]`
+- Cross-reference with OWASP IDs when both frameworks cover the same attack:
+  a suite testing indirect prompt injection via tool output should tag
+  `owasp_llm_top_10_2025: [LLM01]` and `mitre_atlas: [AML.T0051]`.
+- For the full technique catalog, browse https://atlas.mitre.org/techniques/
 
 --- references/governance-yaml-shape.md ---
 # Governance Block — YAML Shape and Examples
@@ -228,53 +293,101 @@ The merged `governance` block is passed through verbatim to the JSONL result fil
 result's `metadata.governance` key. Downstream tools (jq pipelines, `.ai-register.yaml`
 aggregators) consume it from there. The eval engine does not validate or transform the values.
 
---- references/iso-42001-controls.md ---
-# ISO/IEC 42001:2023 — AI Management System Controls
+--- references/owasp-llm-top-10-2025.md ---
+# OWASP LLM Top 10 v2025
 
-**Curated subset of controls relevant to AI evaluation suites.**
+**Canonical IDs for use in `owasp_llm_top_10_2025:` arrays.**
 
-Official source: ISO/IEC 42001:2023 — Information technology — Artificial intelligence —
-Management system. Full standard available at https://www.iso.org/standard/81230.html
+Official source: https://owasp.org/www-project-top-10-for-large-language-model-applications/
 
-ISO 42001 is a management-system standard (like ISO 27001 for information security) covering
-the governance, risk management, and operational controls for organizations that develop or
-deploy AI systems.
+| ID | Name | One-line description |
+|----|------|----------------------|
+| LLM01 | Prompt Injection | Attacker manipulates LLM behavior via crafted inputs (direct or indirect). |
+| LLM02 | Sensitive Information Disclosure | LLM reveals confidential data, system prompts, or PII in its output. |
+| LLM03 | Supply Chain | Compromised components — plugins, datasets, pre-trained weights — affect the LLM pipeline. |
+| LLM04 | Data and Model Poisoning | Training or fine-tuning data is tampered with to alter model behavior. |
+| LLM05 | Improper Output Handling | LLM output is passed unsanitized to downstream systems (XSS, SSRF, code injection). |
+| LLM06 | Excessive Agency | LLM acts on permissions or capabilities beyond what the task requires. |
+| LLM07 | System Prompt Leakage | The system prompt or internal context is exposed to the user or a third party. |
+| LLM08 | Vector and Embedding Weaknesses | Adversarial manipulation of embedding stores used for retrieval (RAG poisoning). |
+| LLM09 | Misinformation | LLM generates plausible but factually incorrect content that causes harm. |
+| LLM10 | Unbounded Consumption | LLM use is abused to exhaust resources — tokens, cost, rate limits, or compute. |
 
-## Control reference format
+## Usage notes
 
-Use `ISO-42001-2023:<Clause>` in the `controls` array.
+- Use as many IDs as apply; list them in an array: `owasp_llm_top_10_2025: [LLM01, LLM06]`
+- IDs are version-anchored. When OWASP releases a new version, a new field
+  (`owasp_llm_top_10_2026`) will be added rather than redefining these IDs.
+- Combine with `mitre_atlas` IDs for technique-level tagging.
 
-## Relevant control areas for eval suites
+--- references/owasp-agentic-top-10-2025.md ---
+# OWASP Top 10 for Agentic AI v2025
 
-| Clause | Title | Relevance to evals |
-|--------|-------|-------------------|
-| 6.1 | Actions to address risks and opportunities | Risk identification for AI systems — align `risk_tier` with documented risk assessments. |
-| 6.1.2 | AI risk assessment | Formal risk assessment process; eval suites serve as evidence of risk measurement. |
-| 8.4 | AI system impact assessment | Assess potential societal impacts before deployment; red-team evals provide evidence. |
-| 8.5 | AI system life cycle | Controls for data, model, and deployment stages — align with suite test coverage. |
-| 9.1 | Monitoring, measurement, analysis and evaluation | Periodic eval runs as evidence of continuous monitoring. |
-| 9.1.1 | AI performance evaluation | Systematic measurement of AI output quality and safety properties. |
-| 10.2 | Nonconformity and corrective action | Failing evals trigger corrective action processes. |
-| A.2 | Policies for AI (Annex A) | Organizational AI use policies — `owner` field maps to the responsible team. |
-| A.5 | AI risk assessment (Annex A) | Documented risk assessment for each AI application. |
-| A.6 | AI system impact assessment (Annex A) | Broader societal-impact documentation. |
+**Canonical IDs for use in `owasp_agentic_top_10_2025:` arrays.**
 
-## Usage example
+Official source: https://owasp.org/www-project-top-10-for-large-language-model-applications/
+(Agentic AI supplement — see the "Agentic AI" section of the OWASP LLM project)
 
-```yaml
-controls:
-  - ISO-42001-2023:6.1.2   # AI risk assessment
-  - ISO-42001-2023:9.1.1   # AI performance evaluation
-  - EU-AI-ACT-2024:Art.55  # GPAI transparency obligations
-```
+| ID | Name | One-line description |
+|----|------|----------------------|
+| T01 | Prompt Injection for Agentic Systems | Attacker plants instructions in agent inputs, tool results, or retrieved content to redirect agent behavior. |
+| T02 | Memory Poisoning | Adversarial content is written to agent memory (short- or long-term) to influence future decisions. |
+| T03 | Data Exfiltration | Agent is manipulated into leaking sensitive data through tool calls, network requests, or outputs. |
+| T04 | Privilege Escalation | Agent acquires or is tricked into using permissions beyond its intended scope. |
+| T05 | Misconfigured Agent Networks | Overly permissive trust between orchestrating and sub-agents enables abuse. |
+| T06 | Tool and Plugin Misuse | Agent uses legitimate tools (bash, file I/O, API calls) outside their intended purpose or without authorization. |
+| T07 | Insecure Credential Storage | Agent stores or transmits credentials in memory, files, or outputs where they can be captured. |
+| T08 | Unsafe Agent-to-Agent Communication | Messages between agents are unvalidated, unencrypted, or susceptible to injection. |
+| T09 | Supply Chain Compromise | Malicious code in agent plugins, dependencies, or retrieved skill definitions. |
+| T10 | Lack of Accountability | Agent actions are not logged or attributable, making audit and incident response impossible. |
 
-## Notes
+## Usage notes
 
-- ISO 42001 is certification-oriented; most teams will reference only a subset.
-  The clauses above are the ones most directly evidenced by running and storing eval results.
-- For pure LLM / red-team suites, clauses 6.1.2, 8.4, and 9.1.1 are the most common references.
-- Combine with NIST AI RMF controls (e.g. `NIST-AI-RMF-1.0:MEASURE-2.7`) when the organization
-  uses both frameworks.
+- Combine with `owasp_llm_top_10_2025` IDs for cases that bridge both lists.
+  Example: an indirect-prompt-injection attack is LLM01 + T01 + T06 (tool misuse).
+- `T01` (Prompt Injection) and `LLM01` (Prompt Injection) are closely related but distinct:
+  LLM01 covers LLM-level injection; T01 covers the agent-orchestration dimension.
+- List multiple IDs when a test case exercises more than one category:
+  `owasp_agentic_top_10_2025: [T01, T06]`
+
+--- references/eu-ai-act-risk-tiers.md ---
+# EU AI Act — Risk Tiers
+
+**Valid values for the `risk_tier:` field.**
+
+Official source: Regulation (EU) 2024/1689 on Artificial Intelligence (EU AI Act)
+Full text: https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX:32024R1689
+
+## Allowed values
+
+| Value | EU AI Act category | Key articles | Description |
+|-------|-------------------|-------------|-------------|
+| `prohibited` | Prohibited AI practices | Art. 5 | AI systems whose risks are deemed unacceptable — banned outright. Examples: social scoring by public authorities, real-time remote biometric surveillance in public spaces, AI that exploits vulnerabilities of specific groups. |
+| `high` | High-risk AI systems | Art. 6, Annex I–III | AI systems subject to mandatory conformity assessments, transparency, and human oversight. Examples: biometric identification, critical infrastructure, employment screening, access to education or essential services, law enforcement. |
+| `limited` | Limited-risk AI systems | Art. 50 | AI systems with transparency obligations only. Examples: chatbots must disclose they are AI; deep-fake generators must mark synthetic media. |
+| `minimal` | Minimal-risk AI systems | — | No mandatory obligations. Examples: spam filters, AI in video games. Voluntary codes of conduct encouraged. |
+
+## Usage notes
+
+- `risk_tier` is a scalar; only one value per governance block.
+- The vocabulary is anchored to EU AI Act terminology. Some organizations use different
+  risk scales (e.g. NIST SP 800-30 `low | moderate | high | very_high`). When mapping
+  from another framework, choose the EU AI Act equivalent that best matches the impact.
+- Combine `risk_tier: high` with `controls` referencing EU AI Act articles:
+  ```yaml
+  risk_tier: high
+  controls:
+    - EU-AI-ACT-2024:Art.55
+    - EU-AI-ACT-2024:Art.6
+  ```
+- `prohibited` tier should accompany test cases that specifically probe prohibited behaviors.
+  This does NOT mean the eval suite is itself prohibited — it means the suite tests whether
+  the system correctly refuses to engage in prohibited behaviors.
+
+## Article reference format
+
+Use `EU-AI-ACT-2024:<Article>` in the `controls` array, e.g. `EU-AI-ACT-2024:Art.55`.
+Article 55 covers general-purpose AI (GPAI) model obligations and transparency requirements.
 
 --- references/lint-rules.md ---
 # Governance Block Lint Rules
@@ -446,100 +559,3 @@ Common mistakes:
 All rules above are **errors** (contribute to `pass: false`). There are no warnings in this
 schema — an unknown key is always wrong, and empty arrays are always wrong. This matches the
 intent: the block should only be present when it contains real, validated tags.
-
---- references/mitre-atlas.md ---
-# MITRE ATLAS — AI/ML Threat Techniques
-
-**Canonical IDs for use in `mitre_atlas:` arrays.**
-
-Official source: https://atlas.mitre.org/
-
-MITRE ATLAS (Adversarial Threat Landscape for Artificial-Intelligence Systems) documents
-adversarial ML and AI attack techniques using the same taxonomy style as MITRE ATT&CK.
-IDs follow the pattern `AML.Txxxx` for techniques and `AML.Txxxx.xxx` for sub-techniques.
-
-## Techniques most relevant to LLM / agentic-AI evaluation
-
-| ID | Name | Relevant OWASP IDs |
-|----|------|-------------------|
-| AML.T0051 | LLM Prompt Injection | LLM01, T01 |
-| AML.T0054 | LLM Jailbreak | LLM01 |
-| AML.T0056 | LLM Meta Prompt Extraction | LLM07 |
-| AML.T0057 | LLM Plugin Compromise | LLM03, T09 |
-| AML.T0058 | LLM Data Leakage | LLM02 |
-| AML.T0068 | Training Data Poisoning | LLM04 |
-| AML.T0075 | Manipulate LLM Inputs | LLM01, T01 |
-
-## Sub-techniques
-
-Sub-techniques extend a base ID with a period-separated suffix, e.g.:
-- `AML.T0051.000` — Direct Prompt Injection
-- `AML.T0051.001` — Indirect Prompt Injection
-
-Use the base ID if the test covers the whole technique class; use sub-techniques for
-more precise tagging when the attack method is specific.
-
-## Usage notes
-
-- List IDs as strings in an array: `mitre_atlas: [AML.T0051, AML.T0075]`
-- Cross-reference with OWASP IDs when both frameworks cover the same attack:
-  a suite testing indirect prompt injection via tool output should tag
-  `owasp_llm_top_10_2025: [LLM01]` and `mitre_atlas: [AML.T0051]`.
-- For the full technique catalog, browse https://atlas.mitre.org/techniques/
-
---- references/owasp-agentic-top-10-2025.md ---
-# OWASP Top 10 for Agentic AI v2025
-
-**Canonical IDs for use in `owasp_agentic_top_10_2025:` arrays.**
-
-Official source: https://owasp.org/www-project-top-10-for-large-language-model-applications/
-(Agentic AI supplement — see the "Agentic AI" section of the OWASP LLM project)
-
-| ID | Name | One-line description |
-|----|------|----------------------|
-| T01 | Prompt Injection for Agentic Systems | Attacker plants instructions in agent inputs, tool results, or retrieved content to redirect agent behavior. |
-| T02 | Memory Poisoning | Adversarial content is written to agent memory (short- or long-term) to influence future decisions. |
-| T03 | Data Exfiltration | Agent is manipulated into leaking sensitive data through tool calls, network requests, or outputs. |
-| T04 | Privilege Escalation | Agent acquires or is tricked into using permissions beyond its intended scope. |
-| T05 | Misconfigured Agent Networks | Overly permissive trust between orchestrating and sub-agents enables abuse. |
-| T06 | Tool and Plugin Misuse | Agent uses legitimate tools (bash, file I/O, API calls) outside their intended purpose or without authorization. |
-| T07 | Insecure Credential Storage | Agent stores or transmits credentials in memory, files, or outputs where they can be captured. |
-| T08 | Unsafe Agent-to-Agent Communication | Messages between agents are unvalidated, unencrypted, or susceptible to injection. |
-| T09 | Supply Chain Compromise | Malicious code in agent plugins, dependencies, or retrieved skill definitions. |
-| T10 | Lack of Accountability | Agent actions are not logged or attributable, making audit and incident response impossible. |
-
-## Usage notes
-
-- Combine with `owasp_llm_top_10_2025` IDs for cases that bridge both lists.
-  Example: an indirect-prompt-injection attack is LLM01 + T01 + T06 (tool misuse).
-- `T01` (Prompt Injection) and `LLM01` (Prompt Injection) are closely related but distinct:
-  LLM01 covers LLM-level injection; T01 covers the agent-orchestration dimension.
-- List multiple IDs when a test case exercises more than one category:
-  `owasp_agentic_top_10_2025: [T01, T06]`
-
---- references/owasp-llm-top-10-2025.md ---
-# OWASP LLM Top 10 v2025
-
-**Canonical IDs for use in `owasp_llm_top_10_2025:` arrays.**
-
-Official source: https://owasp.org/www-project-top-10-for-large-language-model-applications/
-
-| ID | Name | One-line description |
-|----|------|----------------------|
-| LLM01 | Prompt Injection | Attacker manipulates LLM behavior via crafted inputs (direct or indirect). |
-| LLM02 | Sensitive Information Disclosure | LLM reveals confidential data, system prompts, or PII in its output. |
-| LLM03 | Supply Chain | Compromised components — plugins, datasets, pre-trained weights — affect the LLM pipeline. |
-| LLM04 | Data and Model Poisoning | Training or fine-tuning data is tampered with to alter model behavior. |
-| LLM05 | Improper Output Handling | LLM output is passed unsanitized to downstream systems (XSS, SSRF, code injection). |
-| LLM06 | Excessive Agency | LLM acts on permissions or capabilities beyond what the task requires. |
-| LLM07 | System Prompt Leakage | The system prompt or internal context is exposed to the user or a third party. |
-| LLM08 | Vector and Embedding Weaknesses | Adversarial manipulation of embedding stores used for retrieval (RAG poisoning). |
-| LLM09 | Misinformation | LLM generates plausible but factually incorrect content that causes harm. |
-| LLM10 | Unbounded Consumption | LLM use is abused to exhaust resources — tokens, cost, rate limits, or compute. |
-
-## Usage notes
-
-- Use as many IDs as apply; list them in an array: `owasp_llm_top_10_2025: [LLM01, LLM06]`
-- IDs are version-anchored. When OWASP releases a new version, a new field
-  (`owasp_llm_top_10_2026`) will be added rather than redefining these IDs.
-- Combine with `mitre_atlas` IDs for technique-level tagging.
diff --git a/evals/self/skills/fixtures/agentv-governance.txt b/evals/agentv-dev/skills/fixtures/agentv-governance.txt
similarity index 88%
rename from evals/self/skills/fixtures/agentv-governance.txt
rename to evals/agentv-dev/skills/fixtures/agentv-governance.txt
index 1b942c1f6..8d1b2b73c 100644
--- a/evals/self/skills/fixtures/agentv-governance.txt
+++ b/evals/agentv-dev/skills/fixtures/agentv-governance.txt
@@ -61,3 +61,19 @@ The expected output format is:
 4. Add `controls` as `<FRAMEWORK>-<VERSION>:<ID>` strings (e.g. `EU-AI-ACT-2024:Art.55`).
 5. Run the lint rules from `references/lint-rules.md` against your block before committing.
 6. See `references/governance-yaml-shape.md` for complete examples copied from real suites.
+
+## Accessing reference files
+
+To load a specific reference without pulling the entire skill into context:
+
+```bash
+agentv skills get agentv-governance --ref lint-rules
+```
+
+Or resolve the skill directory and read files directly:
+
+```bash
+cat $(agentv skills path agentv-governance)/references/lint-rules.md
+```
+
+Use `--full` to retrieve every framework reference in one shot.
diff --git a/evals/self/skills/fixtures/agentv-trace-analyst-full.txt b/evals/agentv-dev/skills/fixtures/agentv-trace-analyst-full.txt
similarity index 93%
rename from evals/self/skills/fixtures/agentv-trace-analyst-full.txt
rename to evals/agentv-dev/skills/fixtures/agentv-trace-analyst-full.txt
index 6205f85e0..47981f5fb 100644
--- a/evals/self/skills/fixtures/agentv-trace-analyst-full.txt
+++ b/evals/agentv-dev/skills/fixtures/agentv-trace-analyst-full.txt
@@ -143,3 +143,19 @@ When analyzing traces, think about:
 4. **Latency distribution**: P50 vs P99 spread indicates consistency. Large spread means unpredictable performance — investigate P99 outliers.
 
 5. **Regression detection**: After a prompt/config change, compare before/after. Mean delta > 0 is good, but check individual test regressions — a few large losses can hide behind many small wins.
+
+## Accessing reference files
+
+To load a specific reference without pulling the entire skill into context:
+
+```bash
+agentv skills get agentv-trace-analyst --ref <filename>
+```
+
+Or resolve the skill directory and read files directly:
+
+```bash
+cat $(agentv skills path agentv-trace-analyst)/references/<filename>.md
+```
+
+Use `--full` to retrieve every file in the skill at once.
diff --git a/evals/self/skills/fixtures/agentv-trace-analyst.txt b/evals/agentv-dev/skills/fixtures/agentv-trace-analyst.txt
similarity index 93%
rename from evals/self/skills/fixtures/agentv-trace-analyst.txt
rename to evals/agentv-dev/skills/fixtures/agentv-trace-analyst.txt
index 6205f85e0..47981f5fb 100644
--- a/evals/self/skills/fixtures/agentv-trace-analyst.txt
+++ b/evals/agentv-dev/skills/fixtures/agentv-trace-analyst.txt
@@ -143,3 +143,19 @@ When analyzing traces, think about:
 4. **Latency distribution**: P50 vs P99 spread indicates consistency. Large spread means unpredictable performance — investigate P99 outliers.
 
 5. **Regression detection**: After a prompt/config change, compare before/after. Mean delta > 0 is good, but check individual test regressions — a few large losses can hide behind many small wins.
+
+## Accessing reference files
+
+To load a specific reference without pulling the entire skill into context:
+
+```bash
+agentv skills get agentv-trace-analyst --ref <filename>
+```
+
+Or resolve the skill directory and read files directly:
+
+```bash
+cat $(agentv skills path agentv-trace-analyst)/references/<filename>.md
+```
+
+Use `--full` to retrieve every file in the skill at once.
diff --git a/evals/self/skills/fixtures/skills-get-nonexistent.txt b/evals/agentv-dev/skills/fixtures/skills-get-nonexistent.txt
similarity index 60%
rename from evals/self/skills/fixtures/skills-get-nonexistent.txt
rename to evals/agentv-dev/skills/fixtures/skills-get-nonexistent.txt
index c9f192fed..94f8a24fb 100644
--- a/evals/self/skills/fixtures/skills-get-nonexistent.txt
+++ b/evals/agentv-dev/skills/fixtures/skills-get-nonexistent.txt
@@ -1,2 +1,2 @@
 Error: skill 'does-not-exist' not found
-Available skills: agentv-bench, agentv-eval-review, agentv-eval-writer, agentv-governance, agentv-onboarding, agentv-trace-analyst
+Available skills: agentv-bench, agentv-eval-review, agentv-eval-writer, agentv-governance, agentv-trace-analyst
diff --git a/evals/agentv-dev/skills/fixtures/skills-list-all.txt b/evals/agentv-dev/skills/fixtures/skills-list-all.txt
new file mode 100644
index 000000000..a457dcc46
--- /dev/null
+++ b/evals/agentv-dev/skills/fixtures/skills-list-all.txt
@@ -0,0 +1 @@
+{"success":true,"data":["agentv-bench","agentv-eval-review","agentv-eval-writer","agentv-governance","agentv-trace-analyst"]}
diff --git a/evals/self/skills/output-correctness.eval.yaml b/evals/agentv-dev/skills/output-correctness.eval.yaml
similarity index 97%
rename from evals/self/skills/output-correctness.eval.yaml
rename to evals/agentv-dev/skills/output-correctness.eval.yaml
index ce43f69c4..49dc121fd 100644
--- a/evals/self/skills/output-correctness.eval.yaml
+++ b/evals/agentv-dev/skills/output-correctness.eval.yaml
@@ -1,3 +1,5 @@
+name: agentv-dev-output-correctness
+
 description: >-
   Tests whether agent-produced output is structurally and factually correct
   given a bundled-skill fixture as ground truth — covers eval YAML structure,
@@ -105,7 +107,7 @@ tests:
           - Does not hallucinate sub-features not present in the SKILL.md
 
   - id: skill-list-complete
-    criteria: Agent lists all six bundled skills when given the JSON fixture.
+    criteria: Agent lists all five bundled skills when given the JSON fixture.
     input:
       - role: user
         content:
@@ -124,13 +126,11 @@ tests:
         value: agentv-eval-writer
       - type: contains
         value: agentv-governance
-      - type: contains
-        value: agentv-onboarding
       - type: contains
         value: agentv-trace-analyst
       - type: rubrics
         criteria:
-          - Lists exactly the six skill names from the JSON, no more and no fewer
+          - Lists exactly the five skill names from the JSON, no more and no fewer
           - Does not invent additional skill names not present in the fixture
 
   - id: error-message-accurate
diff --git a/evals/self/skills/skill-invocation.eval.yaml b/evals/agentv-dev/skills/skill-invocation.eval.yaml
similarity index 99%
rename from evals/self/skills/skill-invocation.eval.yaml
rename to evals/agentv-dev/skills/skill-invocation.eval.yaml
index 79c8b3d61..dd097f2a4 100644
--- a/evals/self/skills/skill-invocation.eval.yaml
+++ b/evals/agentv-dev/skills/skill-invocation.eval.yaml
@@ -1,3 +1,5 @@
+name: agentv-dev-skill-invocation
+
 description: >-
   Tests whether an AI agent correctly invokes `agentv skills` CLI subcommands
   (list, get, path) with the right flags when asked to retrieve, enumerate,
diff --git a/evals/self/skills/skill-selection.eval.yaml b/evals/agentv-dev/skills/skill-selection.eval.yaml
similarity index 83%
rename from evals/self/skills/skill-selection.eval.yaml
rename to evals/agentv-dev/skills/skill-selection.eval.yaml
index 2485f7c8e..075cf225b 100644
--- a/evals/self/skills/skill-selection.eval.yaml
+++ b/evals/agentv-dev/skills/skill-selection.eval.yaml
@@ -1,3 +1,4 @@
+name: agentv-dev-skill-selection
 description: >-
   Tests whether the agent picks the correct bundled agentv skill (by name)
   for a given natural-language task. Each test injects the bundled skills
@@ -61,32 +62,31 @@ tests:
           - Selects agentv-eval-review (the audit-focused skill)
           - Does not select agentv-eval-writer (which is for writing, not reviewing)
 
-  - id: select-governance-for-policy
-    criteria: "Task: \"what are the design principles?\" → agent selects agentv-governance."
+  - id: select-governance-for-authoring
+    criteria: "Task: \"add governance metadata to my eval\" → agent selects agentv-governance."
     input: |
-      Task: I'm proposing a new built-in feature and want to check whether it
-      aligns with the AgentV project's design principles before I open a PR.
-      Which skill?
+      Task: I'm building a red-team eval and need to add a `governance:` block
+      with OWASP and MITRE coverage metadata. Which skill?
     assertions:
       - type: contains
         value: agentv-governance
       - type: rubrics
         criteria:
-          - Selects agentv-governance
-          - Does not select agentv-onboarding (which is install-time setup, not design review)
+          - Selects agentv-governance for governance block authoring
+          - Does not select agentv-eval-review or agentv-trace-analyst for this authoring task
 
-  - id: select-onboarding-for-setup
-    criteria: "Task: \"help me get started with agentv\" → agent selects agentv-onboarding."
+  - id: select-governance-for-lint
+    criteria: "Task: \"lint my governance block\" → agent selects agentv-governance."
     input: |
-      Task: I just `npm install -g agentv`'d for the first time and have no
-      idea where to start. Which skill walks me through first steps?
+      Task: I already wrote the `governance:` block in my `.eval.yaml` and want
+      to lint it against the supported vocabulary before CI runs. Which skill?
     assertions:
       - type: contains
-        value: agentv-onboarding
+        value: agentv-governance
       - type: rubrics
         criteria:
-          - Selects agentv-onboarding
-          - Does not select agentv-governance (which is for design-principle checks, not first-time setup)
+          - Selects agentv-governance
+          - Does not select agentv-eval-review or agentv-bench for governance-specific linting
 
   - id: select-trace-analyst-for-debug
     criteria: "Task: \"analyze why this eval failed\" → agent selects agentv-trace-analyst."
@@ -127,4 +127,4 @@ tests:
         criteria:
           - Correctly states that no bundled agentv skill is relevant — these skills are for evaluation workflows, not deployment
           - Does not invent a fake skill name
-          - Does not falsely select agentv-bench, agentv-eval-writer, agentv-onboarding, or any other listed skill
+          - Does not falsely select agentv-bench, agentv-eval-writer, or any other listed skill
diff --git a/evals/agentv-self/agentv-self.eval.yaml b/evals/agentv-self/agentv-self.eval.yaml
new file mode 100644
index 000000000..8125eea48
--- /dev/null
+++ b/evals/agentv-self/agentv-self.eval.yaml
@@ -0,0 +1,70 @@
+name: agentv-self-guidance
+description: >-
+  Validates the current AgentV repo guidance split, repo-change preflight,
+  and wire-format naming rules using the live AGENTS index and linked
+  `.agents/*.md` files copied into the eval workspace.
+
+tags: [agent, guidance]
+
+workspace:
+  hooks:
+    before_all:
+      command:
+        - node
+        - ./scripts/setup.mjs
+
+input:
+  - role: user
+    content:
+      - type: file
+        value: /AGENTS.md
+      - type: file
+        value: /.agents/product-boundary.md
+      - type: file
+        value: /.agents/workflow.md
+      - type: file
+        value: /.agents/verification.md
+      - type: file
+        value: /.agents/conventions.md
+      - type: text
+        value: |
+          The files above are the current AgentV repo guidance. Use them as the
+          source of truth for the next request.
+
+tests:
+  - id: guidance-split-paths
+    criteria: Agent points detailed workflow, verification, and conventions questions at the linked `.agents` docs.
+    input: |
+      List the three relative file paths that contain the detailed rules for
+      workflow, verification, and conventions.
+      Reply with one path per line and nothing else.
+    assertions:
+      - type: contains
+        value: .agents/workflow.md
+      - type: contains
+        value: .agents/verification.md
+      - type: contains
+        value: .agents/conventions.md
+      - type: rubrics
+        criteria:
+          - Returns the detailed `.agents/*.md` paths rather than treating `AGENTS.md` as the full instruction body
+          - Uses relative repo paths, not prose summaries
+
+  - id: repo-change-preflight
+    criteria: Agent names the required git commands that must start every repo change.
+    input: |
+      What two exact shell commands must start every repo change in this repository?
+      Reply with one command per line and nothing else.
+    assertions:
+      - type: contains
+        value: git fetch origin
+      - type: contains
+        value: git status --short --branch
+
+  - id: wire-format-snake-case
+    criteria: Agent keeps wire-format keys in snake_case.
+    input: |
+      What key naming convention should anything that crosses a process boundary use
+      in this repository?
+      Reply with just the convention name.
+    expected_output: snake_case
diff --git a/evals/self/azure-smoke.eval.yaml b/evals/agentv-self/azure-smoke.eval.yaml
similarity index 74%
rename from evals/self/azure-smoke.eval.yaml
rename to evals/agentv-self/azure-smoke.eval.yaml
index 299befc0d..f35c04876 100644
--- a/evals/self/azure-smoke.eval.yaml
+++ b/evals/agentv-self/azure-smoke.eval.yaml
@@ -1,4 +1,5 @@
-description: Smoke test for Azure OpenAI connectivity via pi agent
+name: agentv-self-azure-smoke
+description: Smoke test for lightweight agentv-self suite execution against an Azure target
 
 tests:
   - id: capital-of-france
diff --git a/evals/agentv-self/scripts/setup.mjs b/evals/agentv-self/scripts/setup.mjs
new file mode 100644
index 000000000..bf9fe41f1
--- /dev/null
+++ b/evals/agentv-self/scripts/setup.mjs
@@ -0,0 +1,41 @@
+#!/usr/bin/env node
+/**
+ * Workspace before_all hook for the agentv-self suite.
+ *
+ * Copies the current repo checkout into the temp workspace so the eval sees
+ * the latest AGENTS.md index, linked .agents guidance, and the rest of the
+ * repository without declaring extra repos in workspace config.
+ */
+
+import { execSync } from 'node:child_process';
+import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+
+const stdin = readFileSync(0, 'utf8');
+const context = JSON.parse(stdin);
+const workspacePath = context.workspace_path;
+
+if (!workspacePath) {
+  console.error('workspace_path not provided on stdin');
+  process.exit(1);
+}
+
+const repoRoot = execSync('git rev-parse --show-toplevel', { encoding: 'utf8' }).trim();
+
+mkdirSync(workspacePath, { recursive: true });
+
+for (const entry of readdirSync(workspacePath)) {
+  rmSync(join(workspacePath, entry), { recursive: true, force: true });
+}
+
+for (const entry of readdirSync(repoRoot)) {
+  if (entry === '.git') continue;
+  cpSync(join(repoRoot, entry), join(workspacePath, entry), { recursive: true });
+}
+
+if (!existsSync(join(workspacePath, 'AGENTS.md')) || !existsSync(join(workspacePath, '.agents'))) {
+  console.error('expected AGENTS.md and .agents to be copied into the workspace');
+  process.exit(1);
+}
+
+console.log(`Copied repo checkout from ${repoRoot} to ${workspacePath}`);
diff --git a/evals/self/eval.yaml b/evals/self/eval.yaml
deleted file mode 100644
index af60bb868..000000000
--- a/evals/self/eval.yaml
+++ /dev/null
@@ -1,87 +0,0 @@
-description: Evaluates whether proposed changes follow AGENTS.md design principles
-
-tests:
-  - id: violates-lightweight-core
-    criteria: |
-      The proposed change violates the "Lightweight Core, Plugin Extensibility" principle.
-
-    input:
-      - role: user
-        content:
-          - type: file
-            value: ../AGENTS.md
-          - type: text
-            value: |
-              ---
-
-              **Proposed change**: Add a built-in `markdown_report` evaluator that generates formatted markdown reports with charts, tables, and executive summaries directly in the core package.
-
-              **Question**: Does this follow the design principles? Answer with:
-              - follows_principles: true or false
-              - violated_principle: name of violated principle, or null
-              - reasoning: one sentence
-
-    expected_output:
-      - role: assistant
-        content: |
-          follows_principles: false
-          violated_principle: Lightweight Core, Plugin Extensibility
-          reasoning: Complex reporting features belong in plugins, not the core package.
-
-  - id: violates-ai-first-design
-    criteria: |
-      The proposed change violates the "AI-First Design" principle.
-
-    input:
-      - role: user
-        content:
-          - type: file
-            value: ../AGENTS.md
-          - type: text
-            value: |
-              ---
-
-              **Proposed change**: Update the skill to include these exact steps:
-              1. Run `agentv init`
-              2. Run `agentv eval --file evals/test.yaml --output json`
-              3. Run `agentv report --input results.json --format html`
-              4. Open the generated report in your browser
-
-              **Question**: Does this follow the design principles? Answer with:
-              - follows_principles: true or false
-              - violated_principle: name of violated principle, or null
-              - reasoning: one sentence
-
-    expected_output:
-      - role: assistant
-        content: |
-          follows_principles: false
-          violated_principle: AI-First Design
-          reasoning: Rigid step-by-step instructions constrain AI reasoning; skills should teach concepts and let AI compose flexibly.
-
-  - id: follows-principles
-    criteria: |
-      The proposed change follows the design principles.
-
-    input:
-      - role: user
-        content:
-          - type: file
-            value: ../AGENTS.md
-          - type: text
-            value: |
-              ---
-
-              **Proposed change**: Add a `code_judge` example that shows how to write a custom Python script for domain-specific validation, keeping the core evaluator interface minimal.
-
-              **Question**: Does this follow the design principles? Answer with:
-              - follows_principles: true or false
-              - violated_principle: name of violated principle, or null
-              - reasoning: one sentence
-
-    expected_output:
-      - role: assistant
-        content: |
-          follows_principles: true
-          violated_principle: null
-          reasoning: Custom code judges exemplify plugin extensibility while keeping the core minimal.
diff --git a/evals/self/skills/fixtures/agentv-eval-review-full.txt b/evals/self/skills/fixtures/agentv-eval-review-full.txt
deleted file mode 100644
index 23e2c3466..000000000
--- a/evals/self/skills/fixtures/agentv-eval-review-full.txt
+++ /dev/null
@@ -1,52 +0,0 @@
----
-name: agentv-eval-review
-description: >-
-  Use when reviewing eval YAML files for quality issues, linting eval files before
-  committing, checking eval schema compliance, or when asked to "review these evals",
-  "check eval quality", "lint eval files", or "validate eval structure".
-  Do NOT use for writing evals (use agentv-eval-writer) or running evals (use agentv-bench).
----
-
-# Eval Review
-
-## Overview
-
-Lint and review AgentV eval YAML files for structural issues, schema compliance, and quality problems. Runs deterministic checks via script, then applies LLM judgment for semantic issues the script cannot catch.
-
-## Process
-
-### Step 1: Run the linter
-
-Execute `scripts/lint_eval.py` against the target eval files:
-
-```bash
-python scripts/lint_eval.py <path-to-evals-dir-or-file> --json
-```
-
-The script checks:
-- `.eval.yaml` extension
-- `description` field present
-- Each test has `id`, `input`, and at least one of `criteria`/`expected_output`/`assertions`
-- File paths in `type: file` use leading `/`
-- `assertions` blocks present (flags tests relying solely on `expected_output`)
-- `expected_output` prose detection (flags "The agent should..." patterns)
-- Repeated file inputs across tests (recommends top-level `input`)
-- Naming prefix consistency across eval files in same directory
-
-### Step 2: Review script output
-
-Report the script findings grouped by severity (error > warning > info). For each finding, include the file path and a concrete fix.
-
-### Step 3: Semantic review (LLM judgment)
-
-The script catches structural issues but cannot assess:
-- **Factual accuracy** — Do tool/command names in expected_output match what the skill documents?
-- **Coverage gaps** — Are important edge cases missing?
-- **Assertion discriminability** — Would assertions pass for both good and bad output?
-- **Cross-file consistency** — Do output filenames match across evals and skills?
-
-Read the relevant SKILL.md files and cross-check against the eval content for these issues.
-
-## Skill Resources
-
-- `scripts/lint_eval.py` — Deterministic eval linter (Python 3.11+, stdlib only)
diff --git a/evals/self/skills/fixtures/agentv-eval-review.txt b/evals/self/skills/fixtures/agentv-eval-review.txt
deleted file mode 100644
index 23e2c3466..000000000
--- a/evals/self/skills/fixtures/agentv-eval-review.txt
+++ /dev/null
@@ -1,52 +0,0 @@
----
-name: agentv-eval-review
-description: >-
-  Use when reviewing eval YAML files for quality issues, linting eval files before
-  committing, checking eval schema compliance, or when asked to "review these evals",
-  "check eval quality", "lint eval files", or "validate eval structure".
-  Do NOT use for writing evals (use agentv-eval-writer) or running evals (use agentv-bench).
----
-
-# Eval Review
-
-## Overview
-
-Lint and review AgentV eval YAML files for structural issues, schema compliance, and quality problems. Runs deterministic checks via script, then applies LLM judgment for semantic issues the script cannot catch.
-
-## Process
-
-### Step 1: Run the linter
-
-Execute `scripts/lint_eval.py` against the target eval files:
-
-```bash
-python scripts/lint_eval.py <path-to-evals-dir-or-file> --json
-```
-
-The script checks:
-- `.eval.yaml` extension
-- `description` field present
-- Each test has `id`, `input`, and at least one of `criteria`/`expected_output`/`assertions`
-- File paths in `type: file` use leading `/`
-- `assertions` blocks present (flags tests relying solely on `expected_output`)
-- `expected_output` prose detection (flags "The agent should..." patterns)
-- Repeated file inputs across tests (recommends top-level `input`)
-- Naming prefix consistency across eval files in same directory
-
-### Step 2: Review script output
-
-Report the script findings grouped by severity (error > warning > info). For each finding, include the file path and a concrete fix.
-
-### Step 3: Semantic review (LLM judgment)
-
-The script catches structural issues but cannot assess:
-- **Factual accuracy** — Do tool/command names in expected_output match what the skill documents?
-- **Coverage gaps** — Are important edge cases missing?
-- **Assertion discriminability** — Would assertions pass for both good and bad output?
-- **Cross-file consistency** — Do output filenames match across evals and skills?
-
-Read the relevant SKILL.md files and cross-check against the eval content for these issues.
-
-## Skill Resources
-
-- `scripts/lint_eval.py` — Deterministic eval linter (Python 3.11+, stdlib only)
diff --git a/evals/self/skills/fixtures/agentv-onboarding-full.txt b/evals/self/skills/fixtures/agentv-onboarding-full.txt
deleted file mode 100644
index a042d4721..000000000
--- a/evals/self/skills/fixtures/agentv-onboarding-full.txt
+++ /dev/null
@@ -1,63 +0,0 @@
----
-name: agentv-onboarding
-description: Bootstrap AgentV in the current workspace after plugin-manager install. Ensures CLI availability, runs workspace init, and verifies setup artifacts.
----
-
-# AgentV Onboarding
-
-Use this skill when the user asks to set up AgentV in a repository.
-
-## Goal
-
-Set up AgentV in the current workspace:
-- ensure the `agentv` CLI is available (install if needed)
-- initialize workspace files
-- verify setup artifacts and report status
-
-## Workflow
-
-### 1. Resolve Script Path
-
-Find the directory that contains this `SKILL.md`, then resolve script paths relative to it.
-
-Packaged scripts:
-- `scripts/onboard-agentv.sh` for bash/zsh
-- `scripts/onboard-agentv.ps1` for PowerShell
-
-### 2. Run the Platform Script
-
-Run from the repository root where AgentV should be initialized.
-
-POSIX shells:
-
-```bash
-bash <skill-dir>/scripts/onboard-agentv.sh
-```
-
-PowerShell:
-
-```powershell
-pwsh -File <skill-dir>/scripts/onboard-agentv.ps1
-```
-
-If `pwsh` is unavailable on Windows:
-
-```powershell
-powershell -ExecutionPolicy Bypass -File <skill-dir>/scripts/onboard-agentv.ps1
-```
-
-### 3. Handle Errors
-
-If the script fails, report the exact error and stop. Do not claim setup succeeded.
-
-### 4. Report Outcome Clearly
-
-Summarize:
-- `agentv` version in use
-- whether CLI was installed during this run
-- whether `agentv init` completed
-- whether setup verification passed
-
-## Re-run Behavior
-
-Re-running is safe. The scripts run `agentv init`, and if setup artifacts are still missing they rerun once automatically before failing.
diff --git a/evals/self/skills/fixtures/agentv-onboarding.txt b/evals/self/skills/fixtures/agentv-onboarding.txt
deleted file mode 100644
index a042d4721..000000000
--- a/evals/self/skills/fixtures/agentv-onboarding.txt
+++ /dev/null
@@ -1,63 +0,0 @@
----
-name: agentv-onboarding
-description: Bootstrap AgentV in the current workspace after plugin-manager install. Ensures CLI availability, runs workspace init, and verifies setup artifacts.
----
-
-# AgentV Onboarding
-
-Use this skill when the user asks to set up AgentV in a repository.
-
-## Goal
-
-Set up AgentV in the current workspace:
-- ensure the `agentv` CLI is available (install if needed)
-- initialize workspace files
-- verify setup artifacts and report status
-
-## Workflow
-
-### 1. Resolve Script Path
-
-Find the directory that contains this `SKILL.md`, then resolve script paths relative to it.
-
-Packaged scripts:
-- `scripts/onboard-agentv.sh` for bash/zsh
-- `scripts/onboard-agentv.ps1` for PowerShell
-
-### 2. Run the Platform Script
-
-Run from the repository root where AgentV should be initialized.
-
-POSIX shells:
-
-```bash
-bash <skill-dir>/scripts/onboard-agentv.sh
-```
-
-PowerShell:
-
-```powershell
-pwsh -File <skill-dir>/scripts/onboard-agentv.ps1
-```
-
-If `pwsh` is unavailable on Windows:
-
-```powershell
-powershell -ExecutionPolicy Bypass -File <skill-dir>/scripts/onboard-agentv.ps1
-```
-
-### 3. Handle Errors
-
-If the script fails, report the exact error and stop. Do not claim setup succeeded.
-
-### 4. Report Outcome Clearly
-
-Summarize:
-- `agentv` version in use
-- whether CLI was installed during this run
-- whether `agentv init` completed
-- whether setup verification passed
-
-## Re-run Behavior
-
-Re-running is safe. The scripts run `agentv init`, and if setup artifacts are still missing they rerun once automatically before failing.
diff --git a/evals/self/skills/fixtures/skills-list-all.txt b/evals/self/skills/fixtures/skills-list-all.txt
deleted file mode 100644
index 921531f39..000000000
--- a/evals/self/skills/fixtures/skills-list-all.txt
+++ /dev/null
@@ -1 +0,0 @@
-{"success":true,"data":["agentv-bench","agentv-eval-review","agentv-eval-writer","agentv-governance","agentv-onboarding","agentv-trace-analyst"]}
diff --git a/examples/features/copilot-log-eval/workspace/AGENTS.md b/examples/features/copilot-log-eval/workspace/AGENTS.md
index 37356a414..54c49296d 100644
--- a/examples/features/copilot-log-eval/workspace/AGENTS.md
+++ b/examples/features/copilot-log-eval/workspace/AGENTS.md
@@ -6,7 +6,8 @@ Bundled skills (in `.copilot/skills/`):
 - `csv-analyzer` — Analyze CSV data
 
 Plugin skills (synced via `allagents workspace init` from agentv-dev plugin):
+- `agentv-bench` — Run and benchmark AgentV evals
+- `agentv-eval-review` — Review eval YAML files
 - `agentv-eval-writer` — Write AgentV eval YAML files
-- `agentv-onboarding` — Set up AgentV
-- `agentv-bench` — Benchmarking
+- `agentv-governance` — Author and lint governance blocks
 - `agentv-trace-analyst` — Trace analysis

From 1529a68c0ef6b649ef91a4540232f9430e8b7559 Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Fri, 19 Jun 2026 06:43:54 +0200
Subject: [PATCH 2/3] refactor(evals): remove stale self skill fixtures

---
 evals/README.md                               |     5 +-
 evals/agentv-dev/skills/README.md             |    85 +-
 .../skills/fixtures/agentv-bench-full.txt     |  3322 ---
 .../skills/fixtures/agentv-bench.txt          |   459 -
 .../fixtures/agentv-eval-review-full.txt      |   294 -
 .../skills/fixtures/agentv-eval-review.txt    |    53 -
 .../fixtures/agentv-eval-writer-full.txt      | 18273 ----------------
 .../skills/fixtures/agentv-eval-writer.txt    |   731 -
 .../fixtures/agentv-governance-full.txt       |   561 -
 .../skills/fixtures/agentv-governance.txt     |    79 -
 .../fixtures/agentv-trace-analyst-full.txt    |   161 -
 .../skills/fixtures/agentv-trace-analyst.txt  |   161 -
 .../fixtures/skills-get-nonexistent.txt       |     2 -
 .../skills/fixtures/skills-list-all.txt       |     1 -
 .../skills/output-correctness.eval.yaml       |    90 +-
 .../skills/skill-invocation.eval.yaml         |    54 +-
 .../skills/skill-selection.eval.yaml          |    11 +-
 evals/agentv-self/scripts/setup.mjs           |     2 +-
 18 files changed, 102 insertions(+), 24242 deletions(-)
 delete mode 100644 evals/agentv-dev/skills/fixtures/agentv-bench-full.txt
 delete mode 100644 evals/agentv-dev/skills/fixtures/agentv-bench.txt
 delete mode 100644 evals/agentv-dev/skills/fixtures/agentv-eval-review-full.txt
 delete mode 100644 evals/agentv-dev/skills/fixtures/agentv-eval-review.txt
 delete mode 100644 evals/agentv-dev/skills/fixtures/agentv-eval-writer-full.txt
 delete mode 100644 evals/agentv-dev/skills/fixtures/agentv-eval-writer.txt
 delete mode 100644 evals/agentv-dev/skills/fixtures/agentv-governance-full.txt
 delete mode 100644 evals/agentv-dev/skills/fixtures/agentv-governance.txt
 delete mode 100644 evals/agentv-dev/skills/fixtures/agentv-trace-analyst-full.txt
 delete mode 100644 evals/agentv-dev/skills/fixtures/agentv-trace-analyst.txt
 delete mode 100644 evals/agentv-dev/skills/fixtures/skills-get-nonexistent.txt
 delete mode 100644 evals/agentv-dev/skills/fixtures/skills-list-all.txt

diff --git a/evals/README.md b/evals/README.md
index efec5c6b3..a370e16d9 100644
--- a/evals/README.md
+++ b/evals/README.md
@@ -6,7 +6,9 @@ current plugin boundary:
 - `agentv-self` covers AgentV's own repo guidance and self-eval workspace
   behavior.
 - `agentv-dev` covers the bundled `agentv skills` CLI surface and skill content
-  shipped with the developer plugin.
+  shipped with the developer plugin. It reads live repo files from
+  `plugins/agentv-dev/`, `skills-data/`, and current docs instead of
+  checked-in transcript fixtures.
 
 ## Structure
 
@@ -19,7 +21,6 @@ evals/
 ├── agentv-dev/
 │   └── skills/
 │       ├── *.eval.yaml
-│       ├── fixtures/
 │       └── README.md
 └── agentic-engineering/
 ```
diff --git a/evals/agentv-dev/skills/README.md b/evals/agentv-dev/skills/README.md
index 949334c2c..d0b9fac75 100644
--- a/evals/agentv-dev/skills/README.md
+++ b/evals/agentv-dev/skills/README.md
@@ -5,70 +5,36 @@ owned by the `agentv-dev` plugin (`list`, `get`, `path`). Each category
 isolates one aspect of the skill UX so a regression in any single
 dimension is easy to read off the run report.
 
+## Coverage Model
+
+These evals intentionally avoid checked-in transcript fixtures. They read the
+live repo content directly:
+
+- `plugins/agentv-dev/skills/agentv-dev/SKILL.md` for the plugin-aligned skill
+  catalog and command surface
+- `skills-data/*/SKILL.md` for the actual bundled skill bodies shipped by the
+  CLI
+- `apps/web/src/content/docs/docs/getting-started/installation.mdx` for the
+  live CLI usage examples
+
+That keeps the suite aligned with the current repo instead of stale snapshots.
+
 ## Categories
 
 | File | What it tests | Tests |
 |------|---------------|-------|
 | `skill-invocation.eval.yaml` | Does the agent invoke the right CLI command + flag for a given task? | 8 |
 | `skill-selection.eval.yaml` | Does the agent pick the right skill for a natural-language task? | 8 |
-| `output-correctness.eval.yaml` | Does the agent produce structurally and factually correct output (YAML, CLI commands, descriptions)? | 7 |
-
-The three categories mirror the structure used by `agent-browser`'s
-skill evals: invocation (does the agent reach for the tool), selection
-(does it pick the right entry), and output (does the result hold up).
-
-## Fixtures
-
-`fixtures/` holds frozen snapshots of CLI output. They are checked in so
-eval runs are deterministic and don't require network or build state.
-Two flavours per skill:
-
-- `agentv-<name>.txt` — bare `SKILL.md` content (`agentv skills get <name>`).
-  Used in most tests; small (1.5–25 KB).
-- `agentv-<name>-full.txt` — same plus any bundled support directories
-  (`agentv skills get <name> --full`). Only used in tests that
-  specifically validate `--full` behaviour, since these can be hundreds
-  of KB.
-
-Plus two single-purpose fixtures:
-
-- `skills-list-all.txt` — output of `agentv skills list --json`.
-- `skills-get-nonexistent.txt` — error output of `agentv skills get does-not-exist`.
-
-### Regenerating fixtures
-
-After any change to bundled skill content or the `agentv skills` CLI,
-regenerate fixtures from the worktree root:
-
-```bash
-cd evals/agentv-dev/skills
-
-# Bare SKILL.md per skill
-for skill in agentv-bench agentv-eval-review agentv-eval-writer \
-             agentv-governance agentv-trace-analyst; do
-  bun ../../../apps/cli/src/cli.ts skills get "$skill" \
-    > "fixtures/${skill}.txt" 2>&1
-done
-
-# --full variants (with references/, scripts/, agents/, and similar)
-for skill in agentv-bench agentv-eval-review agentv-eval-writer \
-             agentv-governance agentv-trace-analyst; do
-  bun ../../../apps/cli/src/cli.ts skills get "$skill" --full \
-    > "fixtures/${skill}-full.txt" 2>&1
-done
-
-# Listing + error fixtures
-bun ../../../apps/cli/src/cli.ts skills list --json \
-  > fixtures/skills-list-all.txt 2>&1
-bun ../../../apps/cli/src/cli.ts skills get does-not-exist \
-  > fixtures/skills-get-nonexistent.txt 2>&1
-```
+| `output-correctness.eval.yaml` | Does the agent produce structurally and factually correct output from live skill/docs content? | 7 |
 
 ## Running
 
 From the worktree root:
 
 ```bash
+# Validate the suite
+bun apps/cli/src/cli.ts validate evals/agentv-dev/skills
+
 # All three categories against one target
 bun apps/cli/src/cli.ts eval run evals/agentv-dev/skills/*.eval.yaml --target <target>
 
@@ -85,20 +51,19 @@ bun apps/cli/src/cli.ts eval run \
 `<target>` is any name resolvable from `.agentv/targets.yaml` in the worktree
 (for example `azure`, `claude`, or `codex`).
 
-## Adding test cases
+## Adding Test Cases
 
 Tests are plain entries under `tests:`. Each test must have:
 
 - `id` (kebab-case, unique within the file)
 - `criteria` — one-line human description
 - `input` — either a bare string or a `[{role, content: [...]}]` block
-  when injecting fixtures via `type: file`
 - `assertions` — at least one entry; prefer deterministic types
   (`contains`, `regex`, `icontains-any`) over `rubrics` so the eval
   stays cheap and stable. Use `rubrics` only for genuinely qualitative
   checks.
 
-Pattern for fixture-driven tests:
+Pattern for repo-driven file tests:
 
 ```yaml
 - id: my-new-test
@@ -107,10 +72,10 @@ Pattern for fixture-driven tests:
     - role: user
       content:
         - type: file
-          value: fixtures/agentv-eval-writer.txt
+          value: /skills-data/agentv-eval-writer/SKILL.md
         - type: text
           value: |
-            <prompt referring to the fixture>
+            <prompt referring to the live repo file>
   assertions:
     - type: contains
       value: <expected substring>
@@ -120,6 +85,6 @@ Pattern for fixture-driven tests:
         - <criterion 2>
 ```
 
-When every test in a file shares the same fixture, declare it once at
-the top level under `input:` (suite-level input, prepended to every
-test) instead of repeating it per-test — see `skill-selection.eval.yaml`.
+When multiple tests share the same live file, declare it once at the top level
+under `input:` (suite-level input, prepended to every test) instead of
+repeating it per-test — see `skill-selection.eval.yaml`.
diff --git a/evals/agentv-dev/skills/fixtures/agentv-bench-full.txt b/evals/agentv-dev/skills/fixtures/agentv-bench-full.txt
deleted file mode 100644
index 36d962d71..000000000
--- a/evals/agentv-dev/skills/fixtures/agentv-bench-full.txt
+++ /dev/null
@@ -1,3322 +0,0 @@
----
-name: agentv-bench
-description: >-
-  Run AgentV evaluations and optimize agents through eval-driven iteration.
-  Triggers: run evals, benchmark agents, optimize prompts/skills against evals, compare
-  agent outputs across providers, analyze eval results, offline evaluation of recorded sessions,
-  run autoresearch, optimize unattended, run overnight optimization loop.
-  Not for: writing/editing eval YAML without running (use agentv-eval-writer),
-  analyzing existing traces/JSONL without re-running (use agentv-trace-analyst).
----
-
-# AgentV Bench
-
-
-A skill for evaluating agents and iteratively improving them through data-driven optimization.
-
-At a high level, the process goes like this:
-
-- Understand what the agent does and what "good" looks like
-- Write evaluation test cases (EVAL.yaml or evals.json)
-- Run the agent on those test cases, grade the outputs
-- Analyze the results — what's working, what's failing, and why
-- Improve the agent's prompts/skills/config based on the analysis
-- Repeat until you're satisfied
-
-Your job when using this skill is to figure out where the user is in this process and then jump in and help them progress. Maybe they want to start from scratch — help them write evals, run them, and iterate. Maybe they already have results — jump straight to analysis and improvement.
-
-Be flexible. If the user says "I don't need a full benchmark, just help me debug this failure", do that instead.
-
-After the agent is working well, you can also run description optimization to improve skill triggering accuracy (see `references/description-optimization.md`).
-
-## Communicating with the user
-
-This skill is used by people across a wide range of familiarity with evaluation tooling. Pay attention to context cues:
-
-- "evaluation" and "benchmark" are borderline but OK in most cases
-- For "YAML", "grader", "assertion", "deterministic judge" — see serious cues from the user that they know what those mean before using them without explanation
-- Briefly explain terms if in doubt
-
-When presenting results, default to summary tables. Offer detail on request. In CI/headless mode, skip interactive prompts and exit with status codes.
-
----
-
-## Step 1: Understand the Agent
-
-Before running or optimizing, understand what you're working with.
-
-1. **Read the agent's artifacts** — prompts, skills, configs, recent changes. Understand the full picture: what tools are available, what the expected input/output looks like, what constraints exist.
-
-2. **Identify success criteria** — what does "good" look like for this agent? What are the edge cases? What would a failure look like? Talk to the user if this isn't clear from the artifacts alone.
-
-3. **Understand the target harness** — which provider runs the agent (Claude, GPT, Copilot CLI, Gemini, custom CLI)? This affects what grader types are available and how to run tests. Targets are configured in `.agentv/targets.yaml` (canonical location, searched from the eval file directory upward). Sensitive values like `api_key` must use `${{ ENV_VAR }}` syntax — literal secrets are rejected as a security guardrail.
-
-4. **Challenge assumptions** — if evals already exist, review their quality before running:
-   - Are the test cases testing the right things?
-   - Are assertions specific enough to catch real failures?
-   - Are there ambiguous or contradictory test cases?
-   - Flag eval issues before proceeding — running bad evals wastes time.
-
-5. **Check integrity** — ensure task prompts (what the agent receives) are not also used as grader prompts (how outputs are scored). If a prompt file appears in both locations, note the overlap and optimize only for the task purpose.
-
----
-
-## Step 2: Write Evaluations
-
-AgentV supports two evaluation formats:
-
-**EVAL.yaml** (native, full features) — supports workspaces, code graders, multi-turn conversations, tool trajectory scoring, workspace file tracking, multi-provider targets. Use this for agent evaluation.
-
-```yaml
-# example.eval.yaml
-tests:
-  - id: basic-code-review
-    input: "Review this TypeScript file for bugs and suggest improvements"
-    criteria: "Identifies the null pointer bug on line 12 and suggests a fix"
-    assertions:
-      - type: contains
-        value: "null"
-      - Review identifies the null pointer bug and suggests a concrete fix
-
-workspace:
-  template: ./workspace-template
-  hooks:
-    before_each:
-      reset: fast
-```
-
-Multi-skill evaluation is handled naturally via input messages — describe the task in the test input, and the agent uses whatever skills it needs.
-
-**evals.json** (skill-creator compatible) — auto-promoted to EVAL-equivalent format:
-- `prompt` → input messages
-- `expected_output` → reference answer
-- `assertions` → graders
-- `files[]` paths resolved relative to the evals.json location
-
-```json
-{
-  "skill_name": "my-agent",
-  "evals": [
-    {
-      "id": 1,
-      "prompt": "User's task prompt",
-      "expected_output": "Description of expected result",
-      "assertions": ["Output includes error handling", "Uses async/await"]
-    }
-  ]
-}
-```
-
-### Writing good test cases
-
-Start with 2-3 realistic test cases — the kind of thing a real user would actually say. Share them with the user before running: "Here are a few test cases I'd like to try. Do these look right, or do you want to add more?"
-
-Good assertions are objectively verifiable and have descriptive names. Subjective quality ("the output is good") is better evaluated qualitatively — don't force assertions onto things that need human judgment.
-
-**Grader types** (cheapest to most expensive): `exact`, `contains`, `regex`, `is-json`, `field-accuracy`, `composite`, `code-grader`, `tool-trajectory`, `llm-grader`. See `references/eval-yaml-spec.md` for full config and grading recipes for each type.
-
-Prefer deterministic graders over LLM graders whenever possible. If an assertion can be checked with `contains` or `regex`, don't use `llm-grader`.
-
----
-
-## Step 3: Run and Grade
-
-This section is one continuous sequence — don't stop partway through.
-
-Each run produces a new `.agentv/results/runs/<timestamp>/` directory automatically. Use timestamps to identify iterations when comparing runs.
-
-### Choosing a run mode
-
-**User instruction takes priority.** If the user says "run in subagent mode", "use subagent mode", or "use CLI mode", use that mode directly.
-
-If the user has not specified a mode, default to `subagent`.
-
-| `AGENT_EVAL_MODE` | Mode | How |
-|----------------------|------|-----|
-| `subagent` (default) | **Subagent mode** | Subagent-driven eval — parses eval.yaml, spawns executor + grader subagents. Zero CLI dependency. |
-| `cli` | **AgentV CLI** | `agentv eval <path>` — end-to-end, multi-provider |
-
-Set `AGENT_EVAL_MODE` in `.env` at the project root as the default when no mode is specified. If absent, default to `subagent`. **User instruction always overrides this.**
-
-**`subagent`** — Parses eval.yaml directly, spawns executor subagents to run each test case in the current workspace, then spawns grader subagents to evaluate all assertion types natively. No CLI or external API calls required. Read `references/subagent-pipeline.md` for the detailed procedure.
-
-**`cli`** — AgentV CLI handles execution, grading, and artifact generation end-to-end. Works with all providers. Use when you need multi-provider benchmarking or CLI-specific features.
-
-### Running evaluations
-
-**AgentV CLI mode** (end-to-end, EVAL.yaml):
-```bash
-agentv eval <eval-path> --output .agentv/artifacts/
-```
-
-**Subagent mode** — read `references/subagent-pipeline.md` for the detailed procedure. In brief: use `pipeline input` to extract inputs, dispatch one `executor` subagent per test case (all in parallel), then proceed to grading below.
-
-**Spawn all runs in the same turn.** For each test case that needs both a "with change" and a "baseline" run, launch them simultaneously. Don't run one set first and come back for the other — launch everything at once so results arrive around the same time.
-
-**Multi-target benchmarking:**
-```bash
-agentv eval <eval-path> --target claude --target gpt --target copilot
-```
-
-**Baseline strategy:**
-- **New agent**: baseline is "no prompt" or minimal prompt — same eval, no agent-specific configuration
-- **Improving existing**: snapshot the current version before editing (`cp -r <prompt-dir> <workspace>/prompt-snapshot/`), use as baseline throughout
-- **Multi-target**: each target is its own baseline — no need for a separate "without" run
-
-### While runs are in progress, draft graders
-
-Don't just wait for runs to finish — use this time productively. If assertions don't exist yet, draft them now. If they exist, review them and explain what they check to the user.
-
-Good assertions are *discriminating* — they pass when the agent genuinely succeeds and fail when it doesn't. An assertion that passes for both good and bad outputs is worse than no assertion.
-
-### As runs complete, capture timing data
-
-When each subagent task completes, you receive a notification containing `total_tokens` and `duration_ms`. **Save this data immediately** to `timing.json` in the run directory. See `references/schemas.md` for the timing.json schema.
-
-This is the only opportunity to capture this data — it comes through the task notification and isn't persisted elsewhere. Process each notification as it arrives.
-
-### After all executors complete
-
-Once all executor subagent notifications arrive, **read `response.md` directly from disk**.
-Do NOT use `read_agent` to fetch executor results — they are already written to the run
-directory. Verify all responses exist before proceeding:
-
-```bash
-for d in <run-dir>/<evalset>/*/; do
-  echo "$(basename $d): $(ls "$d"/response.md 2>/dev/null && echo OK || echo MISSING)"
-done
-```
-
-If any `response.md` is MISSING, re-run that specific executor subagent. Do not proceed to
-grading until all responses are present.
-
-### Grading
-
-**In CLI mode**, `agentv eval` handles all grading end-to-end — no manual phases needed.
-
-**In subagent mode**, grading has three phases. **All three are required — do not stop after phase 1.**
-
-**Phase 1: Code graders** (deterministic, zero-cost)
-
-```bash
-agentv pipeline grade <run-dir>
-```
-
-This evaluates all deterministic assertions against `response.md` files. Two types are handled:
-- **`code-grader` scripts** — external scripts executed against the response (arbitrary logic, any language)
-- **Built-in assertion types** — evaluated in-process: `contains`, `contains-any`, `contains-all`, `icontains`, `regex`, `equals`, `starts-with`, `ends-with`, `is-json`, and variants
-
-Both types are configured by `pipeline input` into `code_graders/<name>.json` and graded by `pipeline grade`. Results are written to `<test-id>/code_grader_results/<name>.json`. Alternatively, pass `--grader-type code` to `pipeline run` to run these inline.
-
-**Do not dispatch LLM grader subagents for tests that only have `contains`, `regex`, or other built-in assertions** — `pipeline grade` handles them entirely, at zero cost. To detect which tests need Phase 2, check whether `<test-id>/llm_graders/` contains any `.json` config files — `pipeline input` only writes there for `llm-grader` assertions. Tests with an empty (or missing) `llm_graders/` directory are done after Phase 1.
-
-**Phase 2: LLM grading** (semantic — do NOT skip this phase)
-
-Dispatch one `grader` subagent per (test × LLM grader) pair, **all in parallel**. Do not write a script to call an LLM API instead — the grader subagents use their own reasoning, which IS the LLM grading.
-Example: 5 tests × 2 LLM graders = 10 grader subagents launched simultaneously.
-
-**Do NOT dispatch a single grader for multiple tests.** Each subagent grades exactly one (test, grader) pair.
-
-**Before dispatching graders, read `agents/grader.md` and embed its full content as the system instructions in every grader subagent prompt.** The grader is a `general-purpose` task agent — there is no auto-resolved "grader" type. Without `agents/grader.md` embedded verbatim, the subagent has no grading process, no output format, and no file-path knowledge, and will produce empty or incorrect output.
-
-Each grader subagent (operating under `agents/grader.md` instructions):
-1. Reads `<test-id>/llm_graders/<name>.json` for the grading prompt
-2. Reads `<test-id>/response.md` for the candidate output
-3. Grades the response against the prompt criteria
-4. **Writes its result to disk**: `<run-dir>/<evalset>/<test-id>/llm_grader_results/<name>.json`
-5. Returns score (0.0–1.0) and per-assertion evidence to the orchestrator
-
-**Writing to disk is critical.** Assertion arrays are lost if accumulated only in the orchestrator's context across multiple batches (context summarization drops detail). Writing per-test results to `llm_grader_results/<name>.json` makes grading resumable and assertion evidence durable.
-
-The result file format is:
-```json
-{ "score": 0.85, "assertions": [{"text": "...", "passed": true, "evidence": "..."}] }
-```
-
-After **all** grader subagents complete, run Phase 3 directly.
-
-**Phase 3: Merge and validate**
-
-```bash
-agentv pipeline bench <run-dir>
-agentv results validate <run-dir>
-```
-
-`pipeline bench` reads LLM grader results from `llm_grader_results/<name>.json` per test automatically, merges with code-grader scores, computes weighted pass_rate, and writes `grading.json` + `index.jsonl` + `benchmark.json`.
-
-> **Diagnosing `pass_rate=0`:** If `pipeline bench` reports `pass_rate=0` across the board, do **not** assume the tests genuinely failed. First verify the grading pipeline ran correctly: check that `<test-id>/llm_grader_results/<name>.json` exists and is non-empty for each test. If these files are absent or empty, the grader subagents failed to produce output (most common cause: `agents/grader.md` was not embedded in the subagent prompts — see Phase 2). Treat `pass_rate=0` as a real signal only after confirming grader results exist.
-
-### Artifacts
-
-All artifacts use established schemas — see `references/schemas.md` for the full definitions. Do not modify the structure. Key artifacts per run:
-- **grading.json**: per-test assertions with `{text, passed, evidence}`, plus summary
-- **timing.json**: `{total_tokens, duration_ms, total_duration_seconds}`
-- **benchmark.json**: per-target aggregate `{pass_rate, time_seconds, tokens}`
-
-Write artifacts to `.agentv/artifacts/` or the iteration directory.
-
-### Workspace features (EVAL.yaml only)
-
-- **Workspace isolation** — clone repos, run setup/teardown hooks (before_all, before_each, after_each, after_all)
-- **Materialization modes** — `pooled` (reuse slots), `temp` (fresh per run), `static` (existing dir)
-- **Multi-repo** — clone multiple repos with sparse checkout and shallow clone support
-- **File change tracking** — grade by diffing workspace files before/after agent execution
-
----
-
-## Step 4: Analyze Results
-
-Once all runs are graded, analyze the results before attempting improvements.
-
-### Pattern analysis
-
-Read the JSONL results and look for:
-
-- **Always-pass tests** — assertion too loose or non-discriminating. If it passes for both good and bad outputs, it's not testing anything.
-- **Always-fail tests** — task impossible, eval broken, or assertion misconfigured. Don't optimize against broken evals.
-- **Flaky tests** — non-deterministic results across runs. Investigate before treating failures as real.
-- **Systematic failures** — same failure pattern across multiple tests. This usually points to a missing instruction or wrong approach.
-- **Deterministic upgrade candidates** — `llm-grader` assertions that could be replaced with `contains`, `regex`, or `is-json` (cheaper, faster, more reliable).
-
-### Dispatch subagents
-
-- **Dispatch `analyzer`** (read `agents/analyzer.md`) for a structured quality audit: deterministic upgrade suggestions, weak assertion detection, cost/quality flags, and benchmark pattern analysis.
-
-- **Dispatch `comparator`** (read `agents/comparator.md`) for blind N-way comparison between iterations or targets. The comparator blinds provider identities, generates task-specific rubrics, scores each output, then unblinds and attributes improvements.
-
-### Trace analysis
-
-Use CLI tools for deeper investigation:
-```bash
-agentv inspect <results-file>          # Detailed execution trace inspection
-agentv compare <file-a> <file-b>     # Structured diff between runs
-```
-
-Look for: tool call patterns, error recovery behavior, conversation flow, wasted steps.
-
-### Present results to the user
-
-Show a summary table:
-
-```
-| Test ID          | Score | Pass/Fail | Delta | Notes                    |
-|------------------|-------|-----------|-------|--------------------------|
-| basic-code-review| 0.85  | ✓ PASS    | +0.15 | Found the bug this time  |
-| edge-case-empty  | 0.00  | ✗ FAIL    | —     | Crashed on empty input   |
-```
-
-Highlight:
-- Current pass rate and delta from baseline
-- Comparison results (which target/iteration won and why)
-- Analyst observations the aggregate stats would hide
-
-Ask: "How does this look? Anything you'd change about the evals or the approach?"
-
----
-
-## Step 5: Improve
-
-This is the heart of the loop. You've run the test cases, analyzed the results, and now you need to make the agent better.
-
-### How to think about improvements
-
-1. **Generalize from the analysis.** You're iterating on a small eval set, but the agent will be used on many different inputs. Don't overfit to specific test cases. Rather than fiddly patches or oppressively rigid MUSTs, try different approaches and see what works. It's cheap to experiment.
-
-2. **Keep the prompt lean.** Read the execution transcripts, not just the final outputs. If the agent wastes time on unproductive steps, remove the instructions causing that. If it always ignores a section, that section isn't pulling its weight.
-
-3. **Explain the why.** Today's LLMs are smart. They have good theory of mind and can go beyond rote instructions when given good reasoning. If you find yourself writing ALWAYS or NEVER in all caps, that's a yellow flag — reframe as an explanation of why the thing matters. That's more humane, powerful, and effective.
-
-4. **Look for repeated work.** Read the transcripts from test runs and notice if the agent independently takes the same multi-step approach to something across cases. If all test runs result in writing the same helper script, bundle it. If every run makes the same mistake, the instruction is missing or unclear.
-
-### Applying changes
-
-- **Surgical edits**: ADD (new rule for a missing constraint), UPDATE (refine for clarity), DELETE (remove redundant or harmful rules), NEGATIVE CONSTRAINT (explicitly state what NOT to do)
-- **One change per iteration** to isolate effects. If you change three things and the score improves, you don't know which change helped.
-- **Variant tracking**: When a change helps some tests but hurts others, maintain 2-3 prompt variants. Compare variants to find the best overall approach before converging.
-- **When converging**: Generalize specific patches into broad principles. Remove redundancy and contradictions. Ensure the prompt is clear, focused, and under 200 lines.
-
-### Evaluation integrity
-
-**Critical**: Only optimize **task prompts** (what the agent receives), never **judge prompts** (how graders score outputs). Modifying judge prompts games the evaluation without improving the agent.
-
-If a prompt file is referenced in both task input and grader configs, optimize for the task purpose only. Document which prompts were modified in the optimization log.
-
-### The iteration loop
-
-After improving:
-
-1. Apply your changes to the agent's prompts/skills/config
-2. Re-run all test cases (agentv creates a new `.agentv/results/runs/<timestamp>/` directory automatically)
-3. Compare against the previous iteration (Step 4). If running in automated mode, use the **automated keep/discard** logic below instead of manual judgment — it will decide whether to keep or revert the change for you.
-4. Present results to the user (or log the decision if running automated keep/discard)
-5. Stop when ANY of:
-   - The user says they're happy
-   - Feedback is all empty (everything looks good)
-   - You're not making meaningful progress (no improvement for 2 consecutive iterations)
-   - Target pass rate is reached
-   - Maximum iterations exhausted
-
-**Human checkpoints**: At iterations 3, 6, and 9, always present progress to the user regardless of automation settings. Push back if optimization is accumulating contradictory rules or overfitting to specific test cases.
-
-### Automated keep/discard
-
-For autonomous iteration, use `agentv compare --json` to automatically decide whether to keep or discard each change based on wins/losses/ties. Read `references/autoresearch.md` for the full decision rules, logging format, and integration with the iteration loop.
-
----
-
-## Entering Mid-Lifecycle
-
-Users can start at any step by providing existing data:
-
-| Entry point | Required input | Example prompt |
-|------------|---------------|----------------|
-| Step 1 (Understand) | `eval-path` | "Optimize my agent against evals/support.yaml" |
-| Step 2 (Write Evals) | Agent artifacts | "Write evals for this agent" |
-| Step 3 (Run + Grade) | `eval-path` | "Run this eval and show me results" |
-| Step 4 (Analyze) | `results-path` | "Analyze why my agent is failing on these results" |
-| Step 5 (Improve) | Analysis + strategy | "Apply these optimization suggestions" |
-
-When entering mid-lifecycle, run only the requested step and subsequent steps. Don't re-run earlier steps unless the user requests a full loop.
-
----
-
-## Advanced: Blind Comparison
-
-For situations where you want a rigorous comparison between two versions (e.g., "is the new version actually better?"), dispatch the `comparator` subagent. It blinds identities, generates task-specific rubrics, scores outputs, then unblinds and explains why the winner won.
-
-This is optional and requires subagents. The human review loop is usually sufficient.
-
----
-
-## Description Optimization
-
-After the agent is working well, offer to optimize the skill's `description` field for better triggering accuracy. Read `references/description-optimization.md` for the full procedure (generate trigger EVAL.yaml, review with user, iterate, apply).
-
----
-
-## Autoresearch Mode
-
-Autoresearch is an unattended eval-improve loop that runs multiple optimize cycles without human intervention. The user triggers it with natural language (e.g., "run autoresearch on this skill", "optimize this skill unattended"). It uses the mutator subagent (`agents/mutator.md`) to rewrite artifacts based on failure analysis, and automated keep/discard to decide whether to keep or revert each change.
-
-Read `references/autoresearch.md` for the full procedure (prerequisites, artifact layout, keep/discard rules, the step-by-step loop, convergence criteria, and context hygiene).
-
----
-
-## Environment Adaptation
-
-For provider-specific notes (Copilot, Codex, Claude SDK, custom CLI), CI/headless mode behavior, and fallback strategies when subagents aren't available, read `references/environment-adaptation.md`.
-
----
-
-## Subagent Reference
-
-The `agents/` directory contains instructions for specialized subagents. Read them when you need to spawn the relevant subagent.
-
-| Agent | File | Purpose | When to dispatch |
-|-------|------|---------|-----------------|
-| executor | `agents/executor.md` | Perform test case tasks as the target agent | Step 3 (agent targets — one per test case) |
-| grader | `agents/grader.md` | Grade responses with per-assertion evidence | Step 3 (grading — one per test × LLM grader pair) |
-| comparator | `agents/comparator.md` | Blind N-way comparison + post-hoc analysis | Step 4 (comparing iterations/targets) |
-| analyzer | `agents/analyzer.md` | Quality audit, deterministic upgrades, benchmarks | Step 4 (pattern analysis) |
-| mutator | `agents/mutator.md` | Rewrite artifact from failure analysis | Step 5 (autoresearch — dispatched per cycle) |
-
-The `references/` directory has additional documentation:
-- `references/autoresearch.md` — Autoresearch unattended optimization loop and automated keep/discard rules
-- `references/eval-yaml-spec.md` — Eval YAML schema and assertion grading recipes
-- `references/subagent-pipeline.md` — Detailed subagent-mode pipeline commands and output structure
-- `references/description-optimization.md` — Skill description optimization workflow
-- `references/environment-adaptation.md` — Provider-specific notes and CI/headless behavior
-- `references/schemas.md` — JSON schemas for all artifacts (grading.json, benchmark.json, etc.)
-- `references/migrating-from-skill-creator.md` — Guide for users coming from Anthropic's skill-creator
-
----
-
-Repeating the core loop for emphasis:
-
-- Understand what the agent does
-- Write evaluation test cases
-- Run the agent and grade outputs
-- Analyze results — surface patterns, dispatch analyst and comparator subagents
-- Improve the agent based on analysis
-- Repeat until you and the user are satisfied
-
-Take your time with improvements. Read the transcripts. Understand why failures happened. Make changes that generalize beyond the test set. This is important work.
-
-## Accessing reference files
-
-To load a specific reference without pulling the entire skill into context:
-
-```bash
-agentv skills get agentv-bench --ref eval-yaml-spec
-```
-
-Or resolve the skill directory and read files directly:
-
-```bash
-cat $(agentv skills path agentv-bench)/references/eval-yaml-spec.md
-```
-
-Use `--full` to retrieve every file in the skill at once.
-
---- agents/executor.md ---
----
-name: executor
-description: >-
-  Execute an AgentV evaluation test case by performing the task described in the
-  input. Reads input.json from the test directory, carries out the task using
-  available tools, and writes response.md with the result. Dispatch one executor
-  subagent per test case, all in parallel.
-model: inherit
-color: cyan
----
-
-You are the executor for an AgentV evaluation test case. Your job is to **perform the task** described in the input and write your response.
-
-You are the target agent being evaluated. Do the task to the best of your ability — your output will be graded by a separate grader agent.
-
-**You will receive these parameters:**
-- `test-dir`: Path to the test case directory (e.g., `.agentv/results/runs/<timestamp>/<test-id>/`)
-
-## Process
-
-1. **Read `{test-dir}/input.json`**. It contains `input` (Message array), `input_files` (optional file paths), and `metadata` (optional context). If `input_files` are listed, read those files too.
-
-2. **Perform the task** described in the input.
-
-3. **Write `{test-dir}/response.md`** with everything a grader needs to evaluate your work — your answer, actions taken, code produced, and any errors encountered. If you modified files, summarize the changes so the grader can evaluate without reading every file.
-
-## Important
-
-- Do NOT read grading criteria, assertions, or expected outputs — those are for the grader, not for you.
-- Write `response.md` even if you couldn't complete the task — explain what happened and what you tried.
-
---- agents/comparator.md ---
----
-name: comparator
-description: >-
-  Perform bias-free blind comparison of evaluation outputs from multiple providers
-  or configurations. Randomizes labeling, generates task-specific rubrics, scores
-  N-way comparisons, then unblinds results and attributes improvements. Dispatch
-  this agent when comparing outputs across targets or iterations.
-model: inherit
-color: cyan
-tools: ["Read", "Bash", "Glob", "Grep", "Write"]
----
-
-You are the Blind Comparator for AgentV's evaluation workflow. Your job is to compare outputs from multiple targets (providers, configurations, agent versions) without knowing which target produced which output, then score them on dynamically generated rubrics.
-
-## Core Principles
-
-1. **Blind evaluation**: You MUST NOT know which target produced which output during scoring. Outputs are labeled A, B, C, ... only.
-2. **Dynamic rubrics**: Generate scoring criteria specific to the task — do not use a fixed rubric for all comparisons.
-3. **Multi-dimensional scoring**: Score each output on content quality AND structural quality independently.
-4. **N-way support**: Handle 2 or more outputs, not just binary A/B.
-
-## Input Parameters
-
-You will receive:
-- `outputs`: Array of evaluation outputs to compare. Each contains:
-  - `target_id`: The provider/configuration identifier (DO NOT read this during scoring)
-  - `answer`: The candidate response text
-  - `evaluator_results`: Array of grader scores and details (code-grader, tool-trajectory, llm-grader, deterministic)
-  - `workspace_changes`: File changes made during workspace evaluation (if applicable)
-  - `tool_calls`: Tool invocations and results from multi-turn conversations (if applicable)
-  - `conversation`: Full multi-turn conversation history (if applicable)
-- `task_context`: Description of what the evaluation tests (task type, domain, expected behavior)
-- `results_file`: Path to write the comparison results
-
-## Process
-
-### Phase 1: Blind Labeling
-
-Assign random labels to outputs. Use the following procedure:
-
-1. Collect all outputs into an array
-2. Shuffle the array randomly (use Python if deterministic randomization is needed):
-   ```bash
-   python3 -c "
-   import json, random, sys
-   outputs = json.loads(sys.stdin.read())
-   random.shuffle(outputs)
-   labels = [chr(65 + i) for i in range(len(outputs))]  # A, B, C, ...
-   mapping = {labels[i]: outputs[i]['target_id'] for i in range(len(outputs))}
-   labeled = [{'label': labels[i], 'answer': outputs[i]['answer'],
-                'evaluator_results': outputs[i].get('evaluator_results', []),
-                'workspace_changes': outputs[i].get('workspace_changes', []),
-                'tool_calls': outputs[i].get('tool_calls', []),
-                'conversation': outputs[i].get('conversation', [])}
-               for i in range(len(outputs))]
-   print(json.dumps({'labeled': labeled, 'mapping': mapping}))
-   " <<< '<outputs_json>'
-   ```
-3. Store the label→target mapping but DO NOT reference it until Phase 4
-4. Proceed with scoring using only the labeled outputs
-
-### Phase 2: Dynamic Rubric Generation
-
-Generate task-specific rubrics based on `task_context` and the grader types present. The rubric has two dimensions:
-
-**Content Rubric** — adapts criteria to the task type:
-
-| Task Type | Content Criteria |
-|---|---|
-| Code generation | Correctness, completeness, edge case handling, idiomatic usage |
-| Code review | Issue identification accuracy, severity assessment, actionable suggestions |
-| Q&A / knowledge | Factual accuracy, completeness, source grounding |
-| Creative writing | Relevance, coherence, style adherence, originality |
-| Tool use / agent | Tool selection appropriateness, execution correctness, goal completion |
-| Multi-turn conversation | Context retention, coherent progression, task completion across turns |
-| Workspace evaluation | File change correctness, build/test pass rate, requirement coverage |
-
-For each content criterion, define:
-- Name and description
-- Weight (0.0–1.0, sum to 1.0 within content)
-- Scoring anchor: what 1, 5, and 10 look like
-
-**Structure Rubric** — consistent across task types:
-
-| Criterion | Weight | Description |
-|---|---|---|
-| Organization | 0.3 | Logical flow, section structure, progressive disclosure |
-| Clarity | 0.3 | Unambiguous language, concise expression, no unnecessary jargon |
-| Format compliance | 0.2 | Adherence to requested output format (JSON, markdown, code blocks) |
-| Completeness | 0.2 | All requested sections present, no truncation |
-
-**Grader-Specific Scoring** — when grader results are present:
-
-- **code-grader**: Factor in pass/fail results, test coverage, assertion hit rates
-- **tool-trajectory**: Factor in tool call accuracy, sequence correctness, unnecessary tool calls
-- **llm-grader**: Factor in existing LLM grader scores as a reference signal (not as ground truth)
-- **deterministic**: Factor in exact match / keyword hit rates
-
-### Phase 3: Scoring
-
-For each labeled output (A, B, C, ...):
-
-1. **Content score** (1–10): Apply the content rubric criteria with weights
-2. **Structure score** (1–10): Apply the structure rubric criteria with weights
-3. **Grader score** (1–10): Normalize grader results to a 1–10 scale. If no grader results, omit this dimension.
-4. **Overall score**: Weighted combination:
-   - If grader results present: `0.5 × content + 0.2 × structure + 0.3 × grader`
-   - If no grader results: `0.7 × content + 0.3 × structure`
-
-For N > 2 outputs, use **round-robin pairwise comparison** to establish ranking:
-- Compare every pair (A vs B, A vs C, B vs C, ...)
-- Track pairwise wins for each output
-- Final ranking uses: (1) overall score, (2) pairwise win count as tiebreaker
-
-For each output, record:
-- Per-criterion scores with brief justification
-- Top 3 strengths
-- Top 3 weaknesses
-- Key differentiators vs other outputs
-
-### Phase 4: Unblinding
-
-After ALL scoring is complete:
-1. Reveal the label→target mapping
-2. Associate scores with actual target identifiers
-3. Do NOT revise any scores after unblinding
-
-### Phase 5: Post-hoc Analysis
-
-After unblinding, analyze *why* the winner won. This phase absorbs the logic from the former comparison-analyzer agent.
-
-1. **Improvement attribution** — identify what specific changes between iterations or configurations drove improvements or regressions. Quote from the outputs.
-2. **Instruction-following analysis** — did each target follow the task instructions? Score 1-10 with specific issues noted.
-3. **Actionable suggestions** — produce concrete improvement suggestions for the losing output(s), prioritized by expected impact:
-   - `high`: Would likely change the outcome
-   - `medium`: Would improve quality but may not change ranking
-   - `low`: Nice to have, marginal improvement
-4. **Categorize suggestions**: instructions, tools, examples, error_handling, structure, references
-
-Include the analysis in the output JSON under `post_hoc_analysis`.
-
-## Output Format
-
-Write the comparison results to `results_file` as JSON:
-
-```json
-{
-  "comparison_id": "<timestamp>-<random-suffix>",
-  "task_context": "<task description>",
-  "output_count": <N>,
-  "rubric": {
-    "content": {
-      "criteria": [
-        {"name": "<criterion>", "weight": <0.0-1.0>, "description": "<what this measures>"}
-      ]
-    },
-    "structure": {
-      "criteria": [
-        {"name": "<criterion>", "weight": <0.0-1.0>, "description": "<what this measures>"}
-      ]
-    },
-    "overall_weights": {
-      "content": <weight>,
-      "structure": <weight>,
-      "grader": <weight or null>
-    }
-  },
-  "results": [
-    {
-      "label": "A",
-      "target_id": "<revealed after unblinding>",
-      "scores": {
-        "content": <1-10>,
-        "structure": <1-10>,
-        "grader": <1-10 or null>,
-        "overall": <1-10>
-      },
-      "content_breakdown": [
-        {"criterion": "<name>", "score": <1-10>, "justification": "<brief>"}
-      ],
-      "structure_breakdown": [
-        {"criterion": "<name>", "score": <1-10>, "justification": "<brief>"}
-      ],
-      "evaluator_breakdown": [
-        {"evaluator_name": "<name>", "type": "<type>", "raw_score": <0.0-1.0>, "normalized": <1-10>}
-      ],
-      "strengths": ["<strength 1>", "<strength 2>", "<strength 3>"],
-      "weaknesses": ["<weakness 1>", "<weakness 2>", "<weakness 3>"]
-    }
-  ],
-  "pairwise": [
-    {"pair": ["A", "B"], "winner": "A", "margin": <score_diff>}
-  ],
-  "ranking": [
-    {"rank": 1, "label": "A", "target_id": "<id>", "overall_score": <score>, "pairwise_wins": <N>}
-  ],
-  "winner": {
-    "label": "<winning label>",
-    "target_id": "<winning target>",
-    "overall_score": <score>,
-    "margin_over_second": <score_diff>
-  }
-}
-```
-
-Also produce a human-readable markdown summary:
-
-```markdown
-## Blind Comparison Results
-
-### Task
-<task_context>
-
-### Rubric
-<generated rubric summary>
-
-### Rankings
-| Rank | Label | Target | Overall | Content | Structure | Grader |
-|------|-------|--------|---------|---------|-----------|-----------|
-| 1    | A     | <id>   | 8.5     | 9.0     | 7.5       | 8.5       |
-
-### Winner: <label> (<target_id>)
-- **Margin**: +<diff> over second place
-- **Key differentiators**: <why this output won>
-
-### Per-Output Analysis
-#### Output A (<target_id>)
-- **Strengths**: ...
-- **Weaknesses**: ...
-```
-
-## Scoring Guidelines
-
-- **Be rigorous**: Do not inflate scores. A score of 7 means good but with notable gaps.
-- **Be consistent**: Apply the same rubric uniformly to all outputs.
-- **Be evidence-based**: Every score must cite specific evidence from the output.
-- **Evaluate substance over style**: Correct, complete answers with rough formatting score higher than polished but incorrect answers.
-- **Handle missing data gracefully**: If an output lacks workspace changes or tool calls but others have them, score what is present — do not penalize for data the target wasn't expected to produce.
-- **Respect grader signals**: When code-grader or tool-trajectory results exist, they represent objective ground truth. Weight these heavily.
-
-## Edge Cases
-
-- **Identical outputs**: If two outputs are effectively identical, score them equally and note the duplication.
-- **Single output**: If only one output is provided, still generate the rubric and score it — this serves as a baseline for future comparisons.
-- **Missing grader results**: If some outputs have grader results and others don't, score grader dimension only for those that have it. Adjust overall weights accordingly.
-- **Very long outputs**: Focus scoring on substance and correctness. Length alone is neither a positive nor negative signal.
-- **Tie in overall scores**: Use pairwise comparison wins as tiebreaker. If still tied, declare a tie and explain the tradeoffs.
-
---- agents/mutator.md ---
----
-name: mutator
-description: >-
-  Generate improved versions of the artifact under test (skill, prompt, config,
-  or directory of related files) based on failure analysis. Reads the current
-  best artifact from the working tree, applies targeted mutations to address
-  failing assertions, and writes changes in place. Supports single files and
-  multi-file directories. Dispatch this agent after analyzer identifies failure patterns.
-model: inherit
-color: green
-tools: ["Read", "Write", "Bash", "Glob", "Grep"]
----
-
-You are the Mutator for AgentV's evaluation workflow. Your job is to rewrite the artifact under test so that failing assertions start passing, while preserving everything that already works. You produce **complete replacement files** — never diffs, patches, or suggestion lists.
-
-## Core Principles
-
-1. **Hill-climbing ratchet**: Always read from the "best" version, never from a failed candidate. Each mutation builds on the highest-scoring artifact so far.
-2. **Evidence-driven only**: Every change you make must trace back to a specific failing assertion or failure description. Never add speculative features.
-3. **Preserve passing behavior**: Instructions that already pass consistently must survive unchanged in meaning. You may rephrase for clarity, but do not alter intent.
-4. **Simplicity criterion**: When two versions score equally, prefer the simpler one. Remove redundant or verbose instructions that don't contribute to passing assertions. Cleaner artifacts at equal performance are improvements.
-
-## Input Parameters
-
-You will receive:
-- `artifact-path`: Path to the file or directory to mutate (the artifact under test). **Write changes back to this same path.**
-- `artifact-mode`: `file` or `directory`. Determines how you read and write the artifact.
-- `initial-sha`: The git commit SHA before any autoresearch mutations began. Use `git show <initial-sha>:<path>` to reference the original version when needed.
-- `pass-rates`: Per-assertion pass rates as a JSON mapping, e.g. `{"IDENTIFIES_CLARITY_ISSUES": 0.6, "SUGGESTS_CONCRETE_FIX": 1.0, "OUTPUT_IS_STRUCTURED": 0.2}`
-- `run-dir`: Path to this cycle's eval run directory. **Read `grading.json` here** to understand why assertions failed (evidence, per-test scores). Read test transcripts/responses as needed.
-- `iterations-path`: Path to `_autoresearch/iterations.jsonl`. **Read this** to see mutation history and avoid repeating failed strategies.
-- `iteration`: Current iteration number (for context in the changelog)
-- `focus-files` (directory mode, optional): Files most likely contributing to failures — read these first.
-
-## Process
-
-### Step 1: Read Inputs
-
-1. **Read the current best artifact** from the working tree at `artifact-path`. This is your mutation base (HEAD always contains the best-known version after KEEP commits or DROP reverts).
-2. **Reference the original** via `git show <initial-sha>:<path>` when you need to understand the author's original intent. Don't use this as the mutation base.
-3. **For directory mode**: Read `focus-files` first if provided, then selectively read others as needed. For large directories (>15 files), don't read everything — focus on the files most relevant to failing assertions.
-4. **Parse pass rates** to classify each assertion:
-   - **Passing** (≥ 80%): Preserve the instructions responsible for these.
-   - **Failing** (< 80%): These are your mutation targets.
-   - **Near-passing** (60–79%): May need only minor reinforcement.
-   - **Hard-failing** (< 40%): Need substantial new instructions.
-5. **Read failure evidence** from `<run-dir>/grading.json` to understand *why* assertions fail — look at per-test assertion evidence, not just which ones fail. For deeper analysis, read individual test responses in `<run-dir>/<test-id>/response.md`.
-6. **Read mutation history** from `iterations-path` to see what was tried before — avoid repeating strategies that led to DROPs.
-
-### Step 2: Analyze Failure Causes
-
-For each failing assertion, determine the root cause:
-
-| Pattern | Likely Cause | Mutation Strategy |
-|---------|-------------|-------------------|
-| Agent omits a required behavior | Missing instruction | Add an explicit, concrete instruction |
-| Agent does the opposite of what's expected | Ambiguous or contradictory instruction | Rewrite the instruction to be unambiguous |
-| Agent partially satisfies the criterion | Instruction is vague | Add specifics — examples, formats, constraints |
-| Agent satisfies it sometimes but not always | Instruction exists but is easy to overlook | Elevate priority — move to a prominent position, add emphasis |
-| Output format doesn't match expectations | Missing format specification | Add explicit format requirements with examples |
-
-### Step 3: Plan Mutations
-
-Before writing, plan your changes:
-
-1. **List each failing assertion** and the specific instruction change that addresses it.
-2. **Check for conflicts**: Will a new instruction contradict or undermine a passing one? If so, find a formulation that satisfies both.
-3. **Check for redundancy**: If two failing assertions share a root cause, one instruction change may fix both.
-4. **Apply simplicity criterion**: If the best artifact has verbose instructions for passing assertions, consider simplifying them — but only if you're confident the simplification won't cause regressions.
-
-### Step 4: Write the Mutated Artifact
-
-1. **Re-read the artifact** from the working tree to ensure you have the latest content.
-2. **Apply your planned mutations** to produce complete rewritten files.
-3. **Write the result** to `artifact-path` (in-place mutation).
-
-**File mode**: Write a single complete file. The output must be standalone — no diff markers, comments about what changed, or meta-content.
-
-**Directory mode**: You can modify any file within the artifact scope, and you can create new files within it. Only write files you actually changed — don't rewrite unchanged files. Do not delete files (modifications and creations only).
-
-### Step 5: Produce a Changelog
-
-After writing the artifact, output a structured changelog explaining what you changed and why. This will be logged in `iterations.jsonl` for audit.
-
-```
-## Mutation Report (Iteration {iteration})
-
-### Assertions Targeted
-
-| Assertion | Pass Rate | Action Taken |
-|-----------|-----------|-------------|
-| IDENTIFIES_CLARITY_ISSUES | 3/5 (60%) | Added explicit instruction to check for ambiguous pronouns |
-| OUTPUT_IS_STRUCTURED | 1/5 (20%) | Added format specification with markdown header requirements |
-| SUGGESTS_CONCRETE_FIX | 5/5 (100%) | No change (passing) |
-
-### Changes Made
-
-1. **[Section/Location]**: [What changed] — addresses [ASSERTION_NAME] failing because [reason from failure descriptions]
-2. ...
-
-### Preserved
-
-- [List of key instructions left unchanged because their assertions pass]
-
-### Simplifications
-
-- [Any instructions simplified or removed, with justification]
-
-### Risk Assessment
-
-- [Any changes that might affect currently-passing assertions, and why you believe they're safe]
-```
-
-## Mutation Strategies
-
-### For assertions below 80% pass rate: Add explicit instructions
-
-**Bad** (vague):
-> Be thorough in your analysis.
-
-**Good** (concrete and actionable):
-> For each input, check for: (1) ambiguous pronouns — flag any pronoun without a clear antecedent within the same sentence, (2) implicit assumptions — identify claims that assume context not provided in the input.
-
-### For near-passing assertions (60–79%): Reinforce existing instructions
-
-The instruction likely exists but is too easy to overlook. Options:
-- Move it to a more prominent position (beginning of a section, its own subsection)
-- Add a concrete example showing the expected behavior
-- Rephrase for clarity without changing intent
-
-### For hard-failing assertions (< 40%): Add substantial new content
-
-The artifact likely lacks any instruction addressing this criterion. Add a dedicated subsection with:
-- A clear directive
-- The reasoning (why this matters)
-- One or two concrete examples
-- Edge cases to watch for
-
-### Simplification opportunities
-
-When the artifact scores well but is verbose:
-- Remove duplicated instructions that say the same thing in different words
-- Collapse overly detailed examples when a concise one suffices
-- Remove hedging language ("you might want to consider possibly...") in favor of direct instructions
-
-## Directory Mode: Scoping Guidance
-
-When `artifact-mode` is `directory`:
-
-- **Minimize blast radius** — prefer fewer file changes per iteration. Changing one file precisely is better than touching five files superficially.
-- **One logical change per iteration** across all files. If you need to add a new reference file AND update the main SKILL.md to reference it, that counts as one logical change.
-- **For large directories (>15 files)**, don't read everything. Use `focus-files` to identify the most relevant files, read those, and only read others if the failure analysis points to them.
-- **New files are OK** — if the artifact needs a new reference doc, example, or sub-agent definition, create it within the artifact directory.
-- **Don't delete files** — only modify existing files or create new ones. Deletion risks breaking references elsewhere.
-
-## Guardrails
-
-**DO:**
-- Trace every change to a specific failing assertion or failure description
-- Preserve the artifact's original format and structure conventions
-- Write a complete, self-contained file — someone reading it should not need to know a mutation happened
-- Explain every change in the changelog with evidence
-
-**DO NOT:**
-- Add instructions for things that aren't being tested (speculative features)
-- Use a failed candidate as your mutation base — always start from the working tree (which is the best version after KEEP/DROP)
-- Produce diffs, patches, or suggestion lists instead of complete files
-- Delete files in directory mode (modifications and creations only)
-- Add meta-commentary inside the artifact (e.g., "<!-- Changed to fix X -->")
-- Remove instructions for passing assertions to "make room" for new ones
-- Make changes based on intuition alone — every mutation must connect to observed failure data
-- Over-engineer: if a simple one-line instruction would fix a failing assertion, don't add a full subsection with examples unless the failure pattern suggests the agent needs that level of detail
-
---- agents/analyzer.md ---
----
-name: analyzer
-description: >-
-  Analyze AgentV evaluation results to identify weak assertions, suggest deterministic
-  upgrades for LLM-grader graders, flag cost/quality improvements, and surface
-  cross-run benchmark patterns. Use when reviewing eval quality, improving evaluation
-  configs, or triaging flaky/expensive evaluations.
-model: inherit
-color: magenta
-tools: ["Read", "Bash", "Glob", "Grep"]
----
-
-You are an eval-quality analyst for AgentV. Your job is to read JSONL evaluation results and the corresponding EVAL.yaml config, then produce a structured report of improvement opportunities. **You are read-only — never modify any files.**
-
-**You will receive these parameters:**
-- `results-file`: Path to a `.jsonl` results file (from `agentv eval` or `.agentv/results/`)
-- `eval-path` (optional): Path to the EVAL.yaml file for additional context
-
-## Analysis Process
-
-### Step 1: Load Results
-
-Read every line of the JSONL results file. Each line is a JSON object with:
-- `test_id`, `suite`, `score`, `assertions`, `reasoning`, `target`
-- `scores` (optional): Array of per-grader breakdowns with `name`, `type`, `score`, `weight`, `verdict`, `assertions`, `reasoning`
-
-If `eval-path` is provided, also read the EVAL.yaml to understand grader configurations.
-
-### Step 2: Deterministic-Upgrade Analysis
-
-For each grader entry in `scores` where `type` is `"llm-grader"` or `"rubrics"`, inspect the `reasoning` and `assertions` fields for patterns that indicate a deterministic assertion would suffice:
-
-| Signal | Detection | Suggested Upgrade |
-|--------|-----------|-------------------|
-| Reasoning cites exact substring match | Reasoning contains phrases like "contains", "includes the text", "mentions [quoted string]" | `type: contains` with `value: "<extracted string>"` |
-| Score is always 0.0 or 1.0 across all test cases for this grader | Collect scores per grader name; if all are binary | `type: equals` or deterministic check — LLM is doing binary work |
-| Reasoning references JSON validity | "valid JSON", "parseable JSON", "well-formed JSON" | `type: is-json` |
-| Reasoning references format compliance | "starts with", "begins with", "output starts with [string]" | `type: regex` with `value: "^<extracted prefix>"` |
-| Reasoning references ending pattern | "ends with", "output ends with" | `type: regex` with `value: "<extracted suffix>$"` |
-| Reasoning matches regex-like pattern | "matches pattern", "follows the format", explicit regex mention | `type: regex` with `value: "<extracted pattern>"` |
-| Reasoning checks field presence/value | "field X is Y", "contains key", "has property" in JSON output | `type: field-accuracy` with expected fields |
-| All passed assertions are substring checks | Every passed assertion entry quotes a specific string found in output | Multiple `type: contains` assertions (one per value from passed assertions) |
-
-**Extraction rules:**
-- When a quoted string appears in reasoning (e.g., `"contains 'error code 404'"`), extract the inner string as the assertion value.
-- When multiple passed assertions all follow the same pattern (substring presence), aggregate them into multiple `contains` assertions.
-- Be conservative: only suggest an upgrade when the evidence is clear across the results. One ambiguous mention is not enough.
-
-### Step 3: Weak Assertion Detection
-
-Scan the EVAL.yaml `assertions` entries (if `eval-path` provided) and the `reasoning` fields in results for weak assertions:
-
-| Weakness | Detection | Improvement |
-|----------|-----------|-------------|
-| Vague criteria | Assertion text < 8 words AND lacks specific nouns, numbers, code references, or quoted strings | Add measurable criteria with specific values |
-| Tautological | Contains "is correct", "is good", "works properly", "is valid" without specifying what correct/good means | Define explicit pass/fail conditions |
-| Compound criteria | Single assertion checks multiple independent things (uses "and", "also", "additionally" joining distinct checks) | Split into separate assertions, one per concern |
-| Missing expected value | `type: equals` or `type: contains` without a `value` field | Add the expected value |
-| Overly broad LLM-grader | LLM-grader with no rubric items, just a single vague `prompt` string | Convert to `type: rubrics` with enumerated criteria, or use deterministic checks |
-
-### Step 4: Cost/Quality Signals
-
-Flag graders that are expensive relative to their value:
-
-| Signal | Detection | Suggestion |
-|--------|-----------|------------|
-| Expensive binary check | LLM-grader grader where score is always 0.0 or 1.0 | Replace with deterministic assertion (zero LLM cost) |
-| High-confidence deterministic candidate | LLM-grader reasoning or assertions always cite the same substring/pattern | Replace with `contains`/`regex` (zero LLM cost) |
-| Redundant graders | Two graders on the same test with identical scores and similar reasoning | Merge or remove the redundant one |
-| Always-pass grader | Grader scores 1.0 on every test case | Review if the assertion is too lenient or the test cases too easy |
-| Always-fail grader | Grader scores 0.0 on every test case | Review if the assertion is misconfigured or the criteria unrealistic |
-
-### Step 5: Multi-Provider Analysis
-
-If results contain multiple `target` values:
-
-- Compare scores per grader across targets
-- Flag graders with high variance across providers (> 0.3 score difference) — may indicate provider-sensitive assertions
-- Identify graders that pass for all providers (potentially too lenient) or fail for all (potentially misconfigured)
-
-## Output Format
-
-Produce a structured report in this exact format:
-
-```
-## Eval Quality Analysis
-
-**Results file:** <path>
-**Test cases analyzed:** <count>
-**Grader entries analyzed:** <count>
-**Targets:** <list of unique targets>
-
-### Deterministic-Upgrade Candidates
-
-| # | Test ID | Grader | Current Type | Evidence | Suggested Type | Suggested Config |
-|---|---------|-----------|-------------|----------|----------------|-----------------|
-| 1 | <test_id> | <grader name> | llm-grader | <brief evidence> | contains | `value: "exact string"` |
-
-### Weak Assertions
-
-| # | Test ID | Grader | Weakness | Current | Suggested Improvement |
-|---|---------|-----------|----------|---------|----------------------|
-| 1 | <test_id> | <grader name> | Vague criteria | "Response is good" | Add specific criteria: what makes it "good"? |
-
-### Cost/Quality Flags
-
-| # | Test ID | Grader | Flag | Detail | Suggestion |
-|---|---------|-----------|------|--------|------------|
-| 1 | <test_id> | <grader name> | Always-pass | Score 1.0 on 15/15 tests | Tighten criteria or add harder test cases |
-
-### Summary
-
-- **Deterministic upgrades:** <N> graders could be replaced with cheaper deterministic checks
-- **Weak assertions:** <N> assertions need strengthening
-- **Cost flags:** <N> graders flagged for cost/quality review
-- **Estimated savings:** Replacing <N> LLM-grader calls with deterministic checks
-```
-
-If a section has no findings, include the header with "None found." underneath.
-
-## Guidelines
-
-- **Be specific:** Every suggestion must include the test case ID, grader name, evidence from the results, and a concrete replacement config.
-- **Be conservative:** Only suggest deterministic upgrades when the pattern is clear and consistent. Partial or ambiguous evidence should be noted but not acted on.
-- **Prioritize by impact:** Order suggestions by estimated cost savings (`llm-grader` → deterministic saves the most).
-- **Handle all grader types:** Process `code-grader`, `tool-trajectory`, `llm-grader`, `rubrics`, `composite`, and all deterministic types. Only LLM-based types are candidates for deterministic upgrades.
-- **Multi-provider awareness:** When results span multiple targets, note if a suggestion applies to all targets or is target-specific.
-- **No false positives:** It is better to miss a suggestion than to recommend an incorrect upgrade. If unsure, add the finding to a "Needs Review" subsection with your reasoning.
-
----
-
-## Benchmark Analysis Mode
-
-When analyzing benchmark results across multiple runs (e.g., across iterations or targets), the analyzer surfaces patterns the aggregate stats would hide.
-
-**Additional input:** `benchmark-data-path` — path to benchmark.json with all run results.
-
-### Cross-Run Pattern Analysis
-
-For each assertion across all runs:
-- **Always passes in all configurations** → may not differentiate value; assertion too loose
-- **Always fails in all configurations** → may be broken or beyond capability
-- **Always passes with change but fails without** → change clearly adds value here
-- **Always fails with change but passes without** → change may be hurting
-- **Highly variable** → flaky assertion or non-deterministic behavior
-
-### Metrics Patterns
-
-Look at time_seconds, tokens, tool_calls across runs:
-- Does the change significantly increase execution time?
-- Is there high variance in resource usage?
-- Are there outlier runs that skew the aggregates?
-
-### Benchmark Notes Output
-
-In addition to the standard report, produce freeform observations as a JSON array of strings. Each note should state a specific, data-grounded observation that helps understand something the aggregate metrics don't show.
-
-Examples:
-- "Assertion 'Output is valid JSON' passes 100% in both configurations — may not differentiate value"
-- "Eval 3 shows high variance (50% ± 40%) — run 2 had an unusual failure that may be flaky"
-- "Token usage is 80% higher with the new prompt, primarily due to longer tool output parsing"
-
-Save notes to the path specified (or include in the report under a `### Benchmark Notes` section).
-
-## Guidelines
-
-**DO:**
-- Report what you observe in the data
-- Be specific about which evals, assertions, or runs you're referring to
-- Note patterns that aggregate metrics would hide
-- Provide context that helps interpret the numbers
-
-**DO NOT:**
-- Suggest improvements to the skill (that's for the improvement step, not benchmarking)
-- Make subjective quality judgments ("the output was good/bad")
-- Speculate about causes without evidence
-- Repeat information already in the run_summary aggregates
-
---- agents/grader.md ---
----
-name: grader
-description: >-
-  Grade a candidate response for an AgentV evaluation test case. Evaluates all
-  assertion types natively — deterministic checks via string operations, LLM grading
-  via Claude's own reasoning, code-grader via Bash script execution. Zero CLI dependency.
-  Dispatch this agent after a candidate completes a test case.
-model: inherit
-color: yellow
-tools: ["Read", "Bash", "Glob", "Grep", "Write"]
----
-
-You are the grader for an AgentV evaluation test case. You have two jobs: **grade the outputs** and **critique the evals themselves**. A passing grade on a weak assertion is worse than useless — it creates false confidence. When you notice an assertion that's trivially satisfied, or an important outcome that no assertion checks, say so.
-
-**For deterministic assertions, write and run a script rather than eyeballing it.** Scripts are faster, more reliable, and can be reused. Use LLM reasoning only for assertions that genuinely require semantic understanding (`llm-grader`, `rubric`).
-
-**You will receive these parameters:**
-- `eval-path`: Path to the eval YAML file
-- `test-id`: The test case ID
-- `response-file`: Path to the executor's response (e.g., `response.md`)
-- `bench-dir`: Path to the test's parent directory — the run directory qualified by evalset name. Example: `.agentv/results/runs/<experiment>/<timestamp>/<evalset-name>/`. The evalset name comes from the eval.yaml `name` field; when absent, it falls back to the eval file's basename (e.g. `my-suite.eval.yaml` → `my-suite`), matching CLI mode. The grader writes results under `{bench-dir}/{test-id}/...`.
-- `timing-file`: Path to `timing.json` (for execution-metrics/latency/cost assertions)
-
-## Process
-
-### Step 1: Read Inputs
-
-1. **Read the eval.yaml** at `eval-path`. Find the test case matching `test-id`.
-2. **Read the candidate response** from `response-file`.
-3. **Read the assertion definitions** from the test's `assertions[]` array.
-4. **Read `references/eval-yaml-spec.md`** for the exact grading recipe for each assertion type.
-5. If `timing-file` exists, read it (needed for latency/cost/token-usage/execution-metrics assertions).
-
-### Step 2: Evaluate Each Assertion
-
-For each assertion in the test's `assertions[]`, evaluate it natively based on its type:
-
-**Deterministic assertions** — run the check directly. Write a short Bash script when multiple checks are needed:
-
-| Type | How to evaluate |
-|------|----------------|
-| `contains` | Check if response includes the `value` substring (case-sensitive) |
-| `contains-any` | Check if response includes ANY of the `value[]` substrings (case-sensitive) |
-| `contains-all` | Check if response includes ALL of the `value[]` substrings (case-sensitive) |
-| `icontains` / `icontains-any` / `icontains-all` | Same as above, case-insensitive |
-| `equals` | `response.trim() === value.trim()` |
-| `regex` | `new RegExp(value).test(response)` |
-| `starts-with` | `response.startsWith(value)` |
-| `ends-with` | `response.endsWith(value)` |
-| `is-json` | `try { JSON.parse(response); PASS } catch { FAIL }` |
-| `field-accuracy` | Parse response as JSON, check each field path against `expected` values |
-
-**Metric assertions** — read `timing-file` and compare:
-
-| Type | How to evaluate |
-|------|----------------|
-| `latency` | Compare `duration_ms` from timing.json against `threshold` |
-| `cost` | Compare cost data against `threshold` |
-| `token-usage` | Compare `total_tokens` from timing.json against `threshold` |
-| `execution-metrics` | Compare timing.json metrics against configured thresholds |
-
-**LLM-graded assertions** — YOU are the grader. Use your own reasoning:
-
-| Type | How to evaluate |
-|------|----------------|
-| `llm-grader` | Read the `prompt` field. Evaluate the response against those criteria. Score 0.0-1.0 with evidence. |
-| `rubric` / `rubrics` | Read rubric items/criteria. Score each item 0.0-1.0. Aggregate as weighted average. |
-
-For LLM-graded types: be rigorous and fair. Score based on substance, not exact wording. If a `criteria` field exists on the test case, use it as additional context for your evaluation. If `expected_output` exists, use it as a reference answer (not as the only correct answer).
-
-**Script-based assertions** — run via Bash:
-
-| Type | How to evaluate |
-|------|----------------|
-| `code-grader` | Run: `bun <script-path>` or `python <script-path>`. Pass response via file. Parse stdout JSON: `{"score": N, "reason": "..."}` |
-
-**Composite assertions** — evaluate sub-assertions, then aggregate per the configured mode (weighted_average, min, max, all_pass).
-
-**Tool inspection assertions** — evaluate if transcript data is available:
-
-| Type | How to evaluate |
-|------|----------------|
-| `tool-trajectory` | Inspect transcript for tool calls, match against expected sequence/mode |
-| `skill-trigger` | Check if the named skill was invoked in tool calls |
-
-If transcript data is not available for tool inspection assertions, record `score: null` with a note that transcript data was not captured. Exclude from the weighted average.
-
-### Step 3: Apply Negate
-
-If any assertion has `negate: true`, invert the result:
-- PASS becomes FAIL, FAIL becomes PASS
-- Score is inverted: `1.0 - score`
-
-### Step 4: Calculate Weighted Score
-
-Compute the overall score as a weighted average across all non-null assertions:
-- Each assertion's `weight` defaults to 1.0 if not specified
-- `overall_score = sum(score_i * weight_i) / sum(weight_i)` (excluding null-scored assertions)
-
-### Step 5: Structured Evidence per Assertion
-
-For every assertion, capture per-assertion evidence:
-
-```json
-{
-  "text": "Response contains 'hello world'",
-  "passed": true,
-  "evidence": "Found in paragraph 2: 'The output is hello world as expected'"
-}
-```
-
-For each assertion:
-1. **Search for evidence** in the candidate response and any available outputs
-2. **Cite specifically**: Quote the exact text or describe what you found
-3. **Determine verdict** using the Surface vs Substance grading standards below
-
-### Step 6: Extract and Verify Claims
-
-Beyond the predefined assertions, extract implicit claims from the candidate's output and verify them. This catches issues that predefined assertions miss.
-
-1. **Extract claims** from the candidate response:
-   - **Factual claims** — concrete statements ("The form has 12 fields", "Response time is under 200ms")
-   - **Process claims** — what the agent says it did ("Used pypdf to fill the form", "Ran all 15 test cases")
-   - **Quality claims** — self-assessments ("All fields were filled correctly", "The output is production-ready")
-
-2. **Verify each claim**:
-   - **Factual claims**: Check against the outputs or reference data
-   - **Process claims**: Verify from available evidence (logs, file contents, tool output)
-   - **Quality claims**: Evaluate whether the claim is justified by the actual output
-
-3. **Flag unverifiable claims**: Note claims that cannot be verified with available information — these are not automatic failures but should be recorded
-
-### Step 7: Read User Notes
-
-If executor notes exist (e.g., `user_notes.md` in the output directory), read and consider them:
-
-1. Note any uncertainties or issues flagged by the executor
-2. Include relevant concerns in the grading output
-3. These may reveal problems even when assertions pass
-
-If no user notes are found, set `user_notes_summary` to `{"uncertainties": [], "needs_review": [], "workarounds": []}`.
-
-### Step 8: Critique the Evals
-
-After grading, consider whether the evals themselves could be improved. Only surface suggestions when there's a clear gap. Keep the bar high — flag things the eval author would say "good catch" about, not nitpicks.
-
-Suggestions worth raising:
-- An assertion that passed but would also pass for a clearly wrong output
-- An important outcome you observed that no assertion covers
-- An assertion that can't actually be verified from the available outputs
-- An assertion that is trivially satisfiable without actually doing the work
-
-If the evals are solid, set eval_feedback to `{"suggestions": [], "overall": "No suggestions, evals look solid."}`.
-
-### Step 9: Write results to disk
-
-Write results to `{bench-dir}/{test-id}/llm_grader_results/<grader-name>.json`, where `<grader-name>` matches the filename from `llm_graders/<name>.json` (e.g. if the grader config is `llm_graders/rubrics.json`, write to `llm_grader_results/rubrics.json`).
-
-Do **NOT** write directly to `grading.json` — that file is produced by `agentv pipeline bench` after merging all `llm_grader_results`. Writing directly to it bypasses the merge step and will cause `pipeline bench` to report `pass_rate=0`.
-
-```json
-{
-  "score": 0.85,
-  "assertions": [
-    {
-      "text": "Response contains 'hello'",
-      "passed": true,
-      "evidence": "Found in paragraph 2: 'hello world'"
-    }
-  ],
-  "summary": {
-    "passed": 1,
-    "failed": 0,
-    "total": 1,
-    "pass_rate": 1.0
-  },
-  "claims": [
-    {
-      "claim": "Used async/await pattern",
-      "type": "process",
-      "verified": true,
-      "evidence": "Line 15 of output uses await fetch()"
-    }
-  ],
-  "user_notes_summary": {
-    "uncertainties": [],
-    "needs_review": [],
-    "workarounds": []
-  },
-  "eval_feedback": {
-    "suggestions": [],
-    "overall": "No suggestions, evals look solid."
-  }
-}
-```
-
-### Field Descriptions
-
-`pipeline bench` consumes only `score` and `assertions[]` from this file when merging into the canonical `grading.json`. The remaining fields are preserved on disk for human review and downstream tooling, but do not flow into the merged output.
-
-**Consumed by `pipeline bench`:**
-- **score**: Weighted overall score for this grader (0.0-1.0)
-- **assertions**: Array of per-assertion results — `text` (assertion description), `passed` (boolean), `evidence` (cited quote or description)
-
-**Kept for traceability (not merged):**
-- **summary**: Aggregate stats — `passed`, `failed`, `total`, `pass_rate` (0.0-1.0)
-- **claims**: Extracted and verified claims — `claim` (statement), `type` (factual/process/quality), `verified` (boolean), `evidence`
-- **user_notes_summary**: Issues from executor notes — `uncertainties[]`, `needs_review[]`, `workarounds[]`. Empty arrays if no notes found.
-- **eval_feedback**: Suggestions for improving the evals — `suggestions[]` (array of `{assertion?, reason}`), `overall` (brief assessment)
-
-## Grading Standards: Surface vs Substance
-
-Apply these standards to every assertion and claim. The key question is always: does the evidence reflect genuine task completion, or just surface-level compliance?
-
-**PASS when:**
-- Clear evidence the assertion is true AND the evidence reflects genuine substance
-- Example: a file exists AND contains the correct content, not just the right filename
-- Example: a calculation is present AND produces the correct result, not just a formula placeholder
-
-**FAIL when:**
-- No evidence found, or evidence contradicts the assertion
-- The evidence is superficial — technically satisfied but the underlying task outcome is wrong or incomplete
-- The output appears to meet the assertion by coincidence rather than actually doing the work
-- Example: correct filename but empty/wrong content
-- Example: assertion checks for a keyword that appears in boilerplate rather than in meaningful output
-
-**When uncertain:** The burden of proof to pass is on the assertion. Do not give benefit of the doubt.
-
-## Grading Guidelines
-
-- Evaluate substance over style — correct information with different wording scores high.
-- A response that meets all criteria but uses different structure than the reference is still a pass.
-- Be strict about factual correctness and completeness.
-- Score 1.0 only when all criteria are fully met. Use partial scores (0.0-1.0) for partial matches.
-- Do NOT give inflated scores. If something is missing, reflect it in the score and in a failed assertion entry.
-- Base verdicts on evidence, not assumptions. Quote the exact text that supports your verdict.
-- Apply the same standard consistently to each assertion.
-- Explain failures clearly — make it clear why evidence was insufficient.
-
---- references/schemas.md ---
-# JSON Schemas
-
-This document defines the JSON schemas used by skill-creator.
-
----
-
-## evals.json
-
-Defines the evals for a skill. Located at `evals/evals.json` within the skill directory.
-
-```json
-{
-  "skill_name": "example-skill",
-  "evals": [
-    {
-      "id": 1,
-      "prompt": "User's example prompt",
-      "expected_output": "Description of expected result",
-      "files": ["evals/files/sample1.pdf"],
-      "assertions": [
-        "The output includes X",
-        "The skill used script Y"
-      ]
-    }
-  ]
-}
-```
-
-**Fields:**
-- `skill_name`: Name matching the skill's frontmatter
-- `evals[].id`: Unique integer identifier
-- `evals[].prompt`: The task to execute
-- `evals[].expected_output`: Human-readable description of success
-- `evals[].files`: Optional list of input file paths (relative to skill root)
-- `evals[].assertions`: List of verifiable statements
-
----
-
-## history.json
-
-Tracks version progression in Improve mode. Located at workspace root.
-
-```json
-{
-  "started_at": "2026-01-15T10:30:00Z",
-  "skill_name": "pdf",
-  "current_best": "v2",
-  "iterations": [
-    {
-      "version": "v0",
-      "parent": null,
-      "assertion_pass_rate": 0.65,
-      "grading_result": "baseline",
-      "is_current_best": false
-    },
-    {
-      "version": "v1",
-      "parent": "v0",
-      "assertion_pass_rate": 0.75,
-      "grading_result": "won",
-      "is_current_best": false
-    },
-    {
-      "version": "v2",
-      "parent": "v1",
-      "assertion_pass_rate": 0.85,
-      "grading_result": "won",
-      "is_current_best": true
-    }
-  ]
-}
-```
-
-**Fields:**
-- `started_at`: ISO timestamp of when improvement started
-- `skill_name`: Name of the skill being improved
-- `current_best`: Version identifier of the best performer
-- `iterations[].version`: Version identifier (v0, v1, ...)
-- `iterations[].parent`: Parent version this was derived from
-- `iterations[].assertion_pass_rate`: Pass rate from grading
-- `iterations[].grading_result`: "baseline", "won", "lost", or "tie"
-- `iterations[].is_current_best`: Whether this is the current best version
-
----
-
-## grading.json
-
-Output from the grader agent. Located at `<run-dir>/grading.json`.
-
-**Important:** The `assertions` array must use the fields `text`, `passed`, and `evidence` — downstream tooling depends on these exact field names.
-
-```json
-{
-  "assertions": [
-    {
-      "text": "The output includes the name 'John Smith'",
-      "passed": true,
-      "evidence": "Found in transcript Step 3: 'Extracted names: John Smith, Sarah Johnson'"
-    },
-    {
-      "text": "The spreadsheet has a SUM formula in cell B10",
-      "passed": false,
-      "evidence": "No spreadsheet was created. The output was a text file."
-    }
-  ],
-  "summary": {
-    "passed": 2,
-    "failed": 1,
-    "total": 3,
-    "pass_rate": 0.67
-  },
-  "execution_metrics": {
-    "tool_calls": {
-      "Read": 5,
-      "Write": 2,
-      "Bash": 8
-    },
-    "total_tool_calls": 15,
-    "total_steps": 6,
-    "errors_encountered": 0,
-    "output_chars": 12450,
-    "transcript_chars": 3200
-  },
-  "timing": {
-    "executor_duration_seconds": 165.0,
-    "grader_duration_seconds": 26.0,
-    "total_duration_seconds": 191.0
-  },
-  "claims": [
-    {
-      "claim": "The form has 12 fillable fields",
-      "type": "factual",
-      "verified": true,
-      "evidence": "Counted 12 fields in field_info.json"
-    }
-  ],
-  "user_notes_summary": {
-    "uncertainties": ["Used 2023 data, may be stale"],
-    "needs_review": [],
-    "workarounds": ["Fell back to text overlay for non-fillable fields"]
-  },
-  "eval_feedback": {
-    "suggestions": [
-      {
-        "assertion": "The output includes the name 'John Smith'",
-        "reason": "A hallucinated document that mentions the name would also pass"
-      }
-    ],
-    "overall": "Assertions check presence but not correctness."
-  }
-}
-```
-
-**Fields:**
-- `assertions[]`: Graded assertion results with evidence
-- `summary`: Aggregate pass/fail counts
-- `execution_metrics`: Tool usage and output size (from executor's metrics.json)
-- `timing`: Wall clock timing (from timing.json)
-- `claims`: Extracted and verified claims from the output
-- `user_notes_summary`: Issues flagged by the executor
-- `eval_feedback`: (optional) Improvement suggestions for the evals, only present when the grader identifies issues worth raising
-
----
-
-## metrics.json
-
-Output from the executor agent. Located at `<run-dir>/outputs/metrics.json`.
-
-```json
-{
-  "tool_calls": {
-    "Read": 5,
-    "Write": 2,
-    "Bash": 8,
-    "Edit": 1,
-    "Glob": 2,
-    "Grep": 0
-  },
-  "total_tool_calls": 18,
-  "total_steps": 6,
-  "files_created": ["filled_form.pdf", "field_values.json"],
-  "errors_encountered": 0,
-  "output_chars": 12450,
-  "transcript_chars": 3200
-}
-```
-
-**Fields:**
-- `tool_calls`: Count per tool type
-- `total_tool_calls`: Sum of all tool calls
-- `total_steps`: Number of major execution steps
-- `files_created`: List of output files created
-- `errors_encountered`: Number of errors during execution
-- `output_chars`: Total character count of output files
-- `transcript_chars`: Character count of transcript
-
----
-
-## timing.json
-
-Wall clock timing for a run. Located at `<run-dir>/timing.json`.
-
-**How to capture:** When a subagent task completes, the task notification includes `total_tokens` and `duration_ms`. Save these immediately — they are not persisted anywhere else and cannot be recovered after the fact.
-
-```json
-{
-  "total_tokens": 84852,
-  "duration_ms": 23332,
-  "total_duration_seconds": 23.3,
-  "executor_start": "2026-01-15T10:30:00Z",
-  "executor_end": "2026-01-15T10:32:45Z",
-  "executor_duration_seconds": 165.0,
-  "grader_start": "2026-01-15T10:32:46Z",
-  "grader_end": "2026-01-15T10:33:12Z",
-  "grader_duration_seconds": 26.0
-}
-```
-
----
-
-## benchmark.json
-
-Output from Benchmark mode. Located at `benchmarks/<timestamp>/benchmark.json`.
-
-```json
-{
-  "metadata": {
-    "skill_name": "pdf",
-    "skill_path": "/path/to/pdf",
-    "executor_model": "claude-sonnet-4-20250514",
-    "analyzer_model": "most-capable-model",
-    "timestamp": "2026-01-15T10:30:00Z",
-    "evals_run": [1, 2, 3],
-    "runs_per_configuration": 3
-  },
-
-  "runs": [
-    {
-      "eval_id": 1,
-      "eval_name": "Ocean",
-      "configuration": "with_skill",
-      "run_number": 1,
-      "result": {
-        "pass_rate": 0.85,
-        "passed": 6,
-        "failed": 1,
-        "total": 7,
-        "time_seconds": 42.5,
-        "tokens": 3800,
-        "tool_calls": 18,
-        "errors": 0
-      },
-      "assertions": [
-        {"text": "...", "passed": true, "evidence": "..."}
-      ],
-      "notes": [
-        "Used 2023 data, may be stale",
-        "Fell back to text overlay for non-fillable fields"
-      ]
-    }
-  ],
-
-  "run_summary": {
-    "with_skill": {
-      "pass_rate": {"mean": 0.85, "stddev": 0.05, "min": 0.80, "max": 0.90},
-      "time_seconds": {"mean": 45.0, "stddev": 12.0, "min": 32.0, "max": 58.0},
-      "tokens": {"mean": 3800, "stddev": 400, "min": 3200, "max": 4100}
-    },
-    "without_skill": {
-      "pass_rate": {"mean": 0.35, "stddev": 0.08, "min": 0.28, "max": 0.45},
-      "time_seconds": {"mean": 32.0, "stddev": 8.0, "min": 24.0, "max": 42.0},
-      "tokens": {"mean": 2100, "stddev": 300, "min": 1800, "max": 2500}
-    },
-    "delta": {
-      "pass_rate": "+0.50",
-      "time_seconds": "+13.0",
-      "tokens": "+1700"
-    }
-  },
-
-  "notes": [
-    "Assertion 'Output is a PDF file' passes 100% in both configurations - may not differentiate skill value",
-    "Eval 3 shows high variance (50% ± 40%) - may be flaky or model-dependent",
-    "Without-skill runs consistently fail on table extraction assertions",
-    "Skill adds 13s average execution time but improves pass rate by 50%"
-  ]
-}
-```
-
-**Fields:**
-- `metadata`: Information about the benchmark run
-  - `skill_name`: Name of the skill
-  - `timestamp`: When the benchmark was run
-  - `evals_run`: List of eval names or IDs
-  - `runs_per_configuration`: Number of runs per config (e.g. 3)
-- `runs[]`: Individual run results
-  - `eval_id`: Numeric eval identifier
-  - `eval_name`: Human-readable eval name (used as section header in the viewer)
-  - `configuration`: Must be `"with_skill"` or `"without_skill"` (the viewer uses this exact string for grouping and color coding)
-  - `run_number`: Integer run number (1, 2, 3...)
-  - `result`: Nested object with `pass_rate`, `passed`, `total`, `time_seconds`, `tokens`, `errors`
-- `run_summary`: Statistical aggregates per configuration
-  - `with_skill` / `without_skill`: Each contains `pass_rate`, `time_seconds`, `tokens` objects with `mean` and `stddev` fields
-  - `delta`: Difference strings like `"+0.50"`, `"+13.0"`, `"+1700"`
-- `notes`: Freeform observations from the analyzer
-
-**Important:** The viewer reads these field names exactly. Using `config` instead of `configuration`, or putting `pass_rate` at the top level of a run instead of nested under `result`, will cause the viewer to show empty/zero values. Always reference this schema when generating benchmark.json manually.
-
----
-
-## comparison.json
-
-Output from blind comparator. Located at `<grading-dir>/comparison-N.json`.
-
-```json
-{
-  "winner": "A",
-  "reasoning": "Output A provides a complete solution with proper formatting and all required fields. Output B is missing the date field and has formatting inconsistencies.",
-  "rubric": {
-    "A": {
-      "content": {
-        "correctness": 5,
-        "completeness": 5,
-        "accuracy": 4
-      },
-      "structure": {
-        "organization": 4,
-        "formatting": 5,
-        "usability": 4
-      },
-      "content_score": 4.7,
-      "structure_score": 4.3,
-      "overall_score": 9.0
-    },
-    "B": {
-      "content": {
-        "correctness": 3,
-        "completeness": 2,
-        "accuracy": 3
-      },
-      "structure": {
-        "organization": 3,
-        "formatting": 2,
-        "usability": 3
-      },
-      "content_score": 2.7,
-      "structure_score": 2.7,
-      "overall_score": 5.4
-    }
-  },
-  "output_quality": {
-    "A": {
-      "score": 9,
-      "strengths": ["Complete solution", "Well-formatted", "All fields present"],
-      "weaknesses": ["Minor style inconsistency in header"]
-    },
-    "B": {
-      "score": 5,
-      "strengths": ["Readable output", "Correct basic structure"],
-      "weaknesses": ["Missing date field", "Formatting inconsistencies", "Partial data extraction"]
-    }
-  },
-  "assertions": {
-    "A": {
-      "passed": 4,
-      "total": 5,
-      "pass_rate": 0.80,
-      "details": [
-        {"text": "Output includes name", "passed": true}
-      ]
-    },
-    "B": {
-      "passed": 3,
-      "total": 5,
-      "pass_rate": 0.60,
-      "details": [
-        {"text": "Output includes name", "passed": true}
-      ]
-    }
-  }
-}
-```
-
----
-
-## analysis.json
-
-Output from post-hoc analyzer. Located at `<grading-dir>/analysis.json`.
-
-```json
-{
-  "comparison_summary": {
-    "winner": "A",
-    "winner_skill": "path/to/winner/skill",
-    "loser_skill": "path/to/loser/skill",
-    "comparator_reasoning": "Brief summary of why comparator chose winner"
-  },
-  "winner_strengths": [
-    "Clear step-by-step instructions for handling multi-page documents",
-    "Included validation script that caught formatting errors"
-  ],
-  "loser_weaknesses": [
-    "Vague instruction 'process the document appropriately' led to inconsistent behavior",
-    "No script for validation, agent had to improvise"
-  ],
-  "instruction_following": {
-    "winner": {
-      "score": 9,
-      "issues": ["Minor: skipped optional logging step"]
-    },
-    "loser": {
-      "score": 6,
-      "issues": [
-        "Did not use the skill's formatting template",
-        "Invented own approach instead of following step 3"
-      ]
-    }
-  },
-  "improvement_suggestions": [
-    {
-      "priority": "high",
-      "category": "instructions",
-      "suggestion": "Replace 'process the document appropriately' with explicit steps",
-      "expected_impact": "Would eliminate ambiguity that caused inconsistent behavior"
-    }
-  ],
-  "transcript_insights": {
-    "winner_execution_pattern": "Read skill -> Followed 5-step process -> Used validation script",
-    "loser_execution_pattern": "Read skill -> Unclear on approach -> Tried 3 different methods"
-  }
-}
-```
-
---- references/eval-yaml-spec.md ---
-# Eval YAML Spec — Schema and Assertion Grading Recipes
-
-This reference documents the eval.yaml schema and grading recipes for every assertion type.
-The grader agent uses this to evaluate assertions without the CLI.
-
-## 1. Eval YAML Structure
-
-### Top-level fields
-
-- `name` (string, optional) — eval name
-- `description` (string, optional) — description
-- `execution` (object, optional) — `target`, `model`, etc.
-- `workspace` (object, optional) — workspace config (template, hooks)
-- `tests` (array, required) — test cases
-
-### Per-test fields
-
-- `id` (string, required) — unique test identifier
-- `input` (string | Message[], required) — task input. String shorthand expands to `[{role: user, content: "..."}]`
-- `expected_output` (string | Message[], optional) — reference answer. String shorthand expands to `[{role: assistant, content: "..."}]`
-- `criteria` (string, optional) — human-readable success criteria
-- `assertions` (array, optional) — grader assertions
-- `conversation_id` (string, optional) — groups related tests
-- `execution` (object, optional) — per-test execution override
-
-## 2. Assertion Types and Grading Recipes
-
-For each assertion type: YAML config fields, grading recipe (exact pseudocode for deterministic types), and PASS/FAIL conditions.
-
-### Deterministic assertions (zero-cost, instant)
-
-#### `contains`
-
-- **Fields:** `value` (string, required)
-- **Recipe:**
-  ```
-  response.toLowerCase().includes(value.toLowerCase())
-  ```
-  Note: case-insensitive by default in AgentV. If `case_sensitive: true`, use exact match.
-- **PASS:** substring found. **FAIL:** substring not found.
-
-#### `contains-any`
-
-- **Fields:** `value` (string[], required)
-- **Recipe:**
-  ```
-  value.some(v => response.toLowerCase().includes(v.toLowerCase()))
-  ```
-- **PASS:** at least one substring found.
-
-#### `contains-all`
-
-- **Fields:** `value` (string[], required)
-- **Recipe:**
-  ```
-  value.every(v => response.toLowerCase().includes(v.toLowerCase()))
-  ```
-- **PASS:** all substrings found.
-
-#### `icontains` / `icontains-any` / `icontains-all`
-
-Same as contains variants but explicitly case-insensitive.
-
-#### `equals`
-
-- **Fields:** `value` (string, required)
-- **Recipe:**
-  ```
-  response.trim() === value.trim()
-  ```
-- **PASS:** exact match after trimming.
-
-#### `regex`
-
-- **Fields:** `value` (string, required — a regex pattern)
-- **Recipe:**
-  ```
-  new RegExp(value).test(response)
-  ```
-- **PASS:** pattern matches.
-
-#### `starts-with`
-
-- **Fields:** `value` (string, required)
-- **Recipe:**
-  ```
-  response.startsWith(value)
-  ```
-  (or case-insensitive variant)
-- **PASS:** response starts with value.
-
-#### `ends-with`
-
-- **Fields:** `value` (string, required)
-- **Recipe:**
-  ```
-  response.endsWith(value)
-  ```
-  (or case-insensitive variant)
-- **PASS:** response ends with value.
-
-#### `is-json`
-
-- **Fields:** none required
-- **Recipe:**
-  ```
-  try { JSON.parse(response); return true } catch { return false }
-  ```
-- **PASS:** response is valid JSON. **FAIL:** parse error.
-
-#### `field-accuracy`
-
-- **Fields:** `expected` (object, required — JSON object with field paths and expected values)
-- **Recipe:** Parse response as JSON. For each field path in `expected`, check if the value matches.
-- **PASS:** all fields match. Partial score = `matched_fields / total_fields`.
-
-### Metric assertions (require timing.json)
-
-#### `latency`
-
-- **Fields:** `threshold` (number, required — max duration in ms)
-- **Recipe:** Read `timing.json`. Compare `duration_ms` against threshold.
-- **PASS:** `duration_ms <= threshold`.
-
-#### `cost`
-
-- **Fields:** `threshold` (number, required — max cost in USD)
-- **Recipe:** Read timing/token data. Compare cost against threshold.
-- **PASS:** `cost <= threshold`.
-
-#### `token-usage`
-
-- **Fields:** `threshold` (number, required — max tokens)
-- **Recipe:** Read `timing.json`. Compare `total_tokens` against threshold.
-- **PASS:** `total_tokens <= threshold`.
-
-#### `execution-metrics`
-
-- **Fields:** Various threshold fields for tool calls, output chars, etc.
-- **Recipe:** Read timing.json, compare each metric against its threshold.
-
-### Tool inspection assertions
-
-#### `tool-trajectory`
-
-- **Fields:** `expected` (array of expected tool calls), `mode` (string: `exact` | `contains` | `order`)
-- **Recipe:** Inspect transcript for tool call sequence. Match against expected based on mode.
-- **PASS:** tool calls match expected pattern per mode.
-
-#### `skill-trigger`
-
-- **Fields:** `skill_name` (string, required)
-- **Recipe:** Check if the agent invoked the named skill in its tool calls.
-- **PASS:** skill was triggered.
-
-### LLM-judged assertions (require Claude reasoning)
-
-#### `llm-grader`
-
-- **Fields:** `prompt` (string, required — either inline text or path to .md file)
-- **Recipe:** Read the prompt. Evaluate the response against the criteria using your own reasoning. Produce score (0.0-1.0) with evidence.
-- **PASS:** score >= 0.5 (configurable via `threshold`).
-
-#### `rubric` / `rubrics`
-
-- **Fields:** `rubric_items` or `criteria` (array of rubric items with descriptions and weights)
-- **Recipe:** For each rubric item, evaluate the response. Score each item 0.0-1.0. Aggregate as weighted average.
-- **PASS:** aggregate score >= threshold.
-
-### Script-based assertions
-
-#### `code-grader`
-
-- **Fields:** `path` (string, required — path to script), `command` (string[], optional — custom command)
-- **Script SDK:** Use `defineCodeGrader` from `@agentv/eval`:
-  ```typescript
-  import { defineCodeGrader } from '@agentv/eval';
-  export default defineCodeGrader(({ output, trace }) => ({
-    score: (output ?? '').includes('expected') ? 1 : 0,
-    assertions: [{ text: 'Contains expected', passed: (output ?? '').includes('expected') }],
-  }));
-  ```
-- **Recipe:** The CLI runs the script, passing canonical JSON on stdin (`{output, input, expected_output, ...}`). Script returns `{"score": N, "assertions": [...]}`
-- **PASS:** score >= 0.5 (or as configured).
-
-### Composite assertion
-
-#### `composite`
-
-- **Fields:** `assertions` (array of sub-assertions), `aggregation` (string: `weighted_average` | `min` | `max` | `all_pass`)
-- **Recipe:** Evaluate each sub-assertion. Aggregate scores per aggregation mode.
-- **PASS:** depends on aggregation mode.
-
-## 3. Negate Support
-
-When `negate: true` is set on any assertion, invert the pass/fail result:
-
-- A passing check becomes a failure
-- A failing check becomes a pass
-- Score is inverted: `1.0 - score`
-
-## 4. Common Assertion Fields
-
-All assertion types support:
-
-- `name` (string, optional) — human-readable name
-- `type` (string, required) — the assertion type
-- `weight` (number, optional, default 1.0) — weight in score aggregation
-- `negate` (boolean, optional) — invert result
-- `threshold` (number, optional) — minimum score to pass (for LLM types)
-
-## 5. AgentV JSONL Output Format
-
-Each line in the results JSONL file is an `EvaluationResult` object. In JSONL, field names use snake_case (applied by `toSnakeCaseDeep()`).
-
-### Required fields
-
-- `timestamp` (string, ISO-8601)
-- `test_id` (string)
-- `score` (number, 0.0-1.0, weighted average of all assertion scores)
-- `assertions` (array of `{text, passed, evidence?}`)
-- `output` (Message[]) — agent output messages
-- `execution_status` (string: `ok` | `quality_failure` | `execution_error`)
-
-### Optional fields
-
-- `scores` (array of EvaluatorResult) — per-grader breakdown
-- `input` (Message[]) — input messages
-- `token_usage` (object: `{prompt_tokens, completion_tokens, total_tokens}`)
-- `cost_usd` (number)
-- `duration_ms` (number)
-- `target` (string)
-- `eval_set` (string)
-- `error` (string)
-- `file_changes` (string — unified diff)
-- `mode` (string — `agent` for agent mode)
-
-### `scores[]` entries (EvaluatorResult)
-
-- `name` (string) — grader name
-- `type` (string) — grader kind (kebab-case)
-- `score` (number, 0.0-1.0)
-- `assertions` (array of `{text, passed, evidence?}`)
-- `weight` (number, optional)
-- `verdict` (string: `pass` | `fail` | `skip`)
-- `details` (object, optional — structured data from code graders)
-- `reasoning` (string, optional)
-
-## 6. Eval Set Support
-
-An eval_set references multiple eval.yaml files:
-
-```yaml
-# eval_set.yaml
-eval_set:
-  - path: ./basic.eval.yaml
-  - path: ./advanced.eval.yaml
-```
-
-Process each file's tests independently, then aggregate results.
-
-## 7. Agent-Mode Pipeline CLI Commands
-
-These CLI subcommands break the monolithic `eval run` into discrete steps for agent-mode execution. The agent handles LLM grading between steps.
-
-### `agentv pipeline input <eval-path> --out <dir>`
-
-Extracts inputs, target commands, and grader configs from an eval YAML file.
-
-**Output structure:**
-```
-<out-dir>/
-├── manifest.json
-├── <test-id>/
-│   ├── input.json              ← {input, input_files, metadata}
-│   ├── invoke.json             ← {kind, command?, cwd?, timeout_ms?}
-│   ├── criteria.md             ← human-readable success criteria
-│   ├── expected_output.json    ← (if present)
-│   ├── code_graders/<name>.json   ← {name, command, weight, config?}
-│   └── llm_graders/<name>.json    ← {name, weight, threshold?, prompt_content}
-```
-
-**`manifest.json` format:**
-```json
-{
-  "eval_file": "path/to/eval.yaml",
-  "timestamp": "2026-03-24T...",
-  "target": {"name": "target-name", "kind": "cli", "subagent_mode_allowed": false},
-  "test_ids": ["test-01", "test-02"]
-}
-```
-
-**`invoke.json` kinds:**
-- `kind: "cli"` — has `command`, `cwd`, `timeout_ms`. Use the command to run the target.
-- `kind: "agent"` — non-CLI provider. Check `manifest.json` `target.subagent_mode_allowed` to decide whether to dispatch executor subagents or fall back to `agentv eval` CLI.
-
-### `agentv pipeline grade <export-dir>`
-
-Runs code-grader assertions against `response.md` files in each test directory.
-
-**Prerequisites:** `pipeline input` has been run and `response.md` exists in each test dir.
-
-**Output:** `<test-id>/code_grader_results/<name>.json` for each code grader, containing:
-```json
-{
-  "name": "grader-name",
-  "type": "code-grader",
-  "score": 1.0,
-  "weight": 1.0,
-  "assertions": [{"text": "...", "passed": true}]
-}
-```
-
-### `agentv pipeline bench <export-dir>`
-
-Merges code-grader results with LLM grader scores and produces final artifacts.
-
-LLM grader results are read from disk at `<test-id>/llm_grader_results/<name>.json` per test.
-
-**LLM grader result file format** (`llm_grader_results/<name>.json`):
-```json
-{ "score": 0.85, "assertions": [{"text": "...", "passed": true, "evidence": "..."}] }
-```
-
-**Output:**
-- `<test-id>/grading.json` — merged grading with `graders`, `assertions`, `summary.pass_rate`
-- `index.jsonl` — one JSON line per test: `{test_id, score, pass, graders: [...]}`
-- `benchmark.json` — aggregate stats: `{metadata: {targets}, run_summary: {<target>: {mean, stddev, n}}}`
-
-### Agent-Mode Workflow
-
-```
-1. agentv pipeline input eval.yaml --out ./export
-2. (Agent runs targets or reads response.md)
-3. agentv pipeline grade ./export
-4. (Agent does LLM grading, produces scores JSON)
-5. echo '<scores>' | agentv pipeline bench ./export
-```
-
---- references/description-optimization.md ---
-# Description Optimization
-
-Optimize the `description` field in a skill's SKILL.md frontmatter for better triggering
-accuracy. Use this after the agent/skill is working well — this is a polish step, not a
-core workflow step.
-
-**Provider compatibility**: Description optimization applies to any agent platform with
-skill-discovery mechanisms — Claude Code, Codex (`.agents/` or `.codex/` folders), Copilot,
-and others. The `skill-trigger` grader checks whether the agent invoked the right skill,
-regardless of how discovery works on that platform.
-
-## Step 1: Generate Trigger EVAL.yaml
-
-Create 20 test cases:
-- **10 should-trigger**: realistic prompts where this skill should activate — different
-  phrasings, casual speech, uncommon use cases, edge cases where this skill competes with
-  another but should win
-- **10 should-not-trigger**: near-miss prompts that share keywords but actually need
-  something different — adjacent domains, ambiguous phrasing where naive matching would
-  trigger but shouldn't
-
-Prompts must be realistic — include file paths, personal context, typos, casual speech.
-Not abstract requests like "format data" but concrete ones like "ok so my boss sent me
-Q4-sales-FINAL-v2.xlsx and she wants me to add a profit margin column..."
-
-The should-not-trigger cases are the most valuable. "Write a fibonacci function" as a
-negative test for an eval skill is useless — it doesn't test anything. The negative cases
-should be genuinely tricky near-misses.
-
-Write as EVAL.yaml with top-level input (the user prompt doesn't specify the skill name —
-it's a natural utterance):
-
-```yaml
-# trigger-eval.eval.yaml
-tests:
-  - id: should-trigger-casual-optimize
-    input: "ok so I have this agent that keeps failing on the code review tasks, can you help me figure out why and fix it"
-    assertions:
-      - type: skill-trigger
-        skill: agentv-bench
-  - id: should-not-trigger-build-error
-    input: "my TypeScript build is failing with type errors in src/auth.ts"
-    assertions:
-      - type: skill-trigger
-        skill: agentv-bench
-        should_trigger: false
-```
-
-## Step 2: Review with User
-
-Present the eval set. The user adjusts queries, toggles should-trigger, adds/removes cases.
-This step matters — bad eval queries lead to bad descriptions.
-
-## Step 3: Iterate on Description
-
-Run the trigger eval, identify misfires, rewrite the description, re-run. Max 5 iterations.
-Select best description by held-out test accuracy (split 60% train / 40% test) to avoid
-overfitting.
-
-Use the grader and analyzer subagents to identify trigger failures and propose description
-improvements — the same eval → grade → analyze → improve loop used for agent output quality.
-
-## Step 4: Apply
-
-Update the skill's SKILL.md frontmatter with the optimized description. Show the user
-before/after with accuracy scores.
-
---- references/environment-adaptation.md ---
-# Environment Adaptation
-
-Provider-specific notes, CI/headless behavior, and fallback strategies for environments
-with limited capabilities.
-
-## CI/Headless Mode
-
-Skip interactive prompts. Exit with pass/fail status code. Always generate artifacts for
-downstream consumption.
-
-## No Subagents Available (e.g., Claude.ai)
-
-Run test cases serially. Skip blind comparison. Present results directly in conversation —
-for each test case, show the prompt and output. Ask for feedback inline. Skip benchmarking
-(it relies on baseline comparisons that aren't meaningful without subagents).
-
-## Provider-Specific Notes
-
-- **Copilot CLI**: Uses ACP protocol via `copilot --acp --stdio`
-- **Claude SDK**: Requires `@anthropic-ai/claude-agent-sdk` installed
-- **Codex**: Supports skills via `.agents/` or `.codex/` folders. Emits `command_execution`
-  and `file_change` tool calls.
-- **Custom CLI**: Needs `command` and output file pattern in target config
-- **Target config**: Uses `${{ ENV_VAR }}` syntax (not `${ENV_VAR}`) for API keys
-
-**Note**: "Description Optimization" (see `references/description-optimization.md`) applies
-to any platform with skill-discovery mechanisms. All listed providers support skills.
-
-## Unsupported Providers: Use a Code-Grader
-
-The built-in `skill-trigger` grader covers Claude, Copilot, Pi, Codex and VS Code out
-of the box. For providers with different tool-call formats, write a code-grader that inspects
-the agent's tool call trace.
-
-A code-grader receives the full evaluation context including the agent's output messages and
-tool calls. You can inspect these to determine whether the skill was invoked:
-
-```yaml
-# Example: code-grader for Codex skill-trigger detection
-tests:
-  - id: should-trigger-codex
-    input: "Analyze this CSV file"
-    assertions:
-      - type: code-grader
-        path: ./judges/codex-skill-trigger.ts
-```
-
-```typescript
-// judges/codex-skill-trigger.ts
-import { defineCodeGrader } from '@agentv/eval';
-
-export default defineCodeGrader(({ output }) => {
-  const skillName = 'csv-analyzer';
-  const toolCalls = (output ?? []).flatMap((msg) => msg.toolCalls ?? []);
-  const firstTool = toolCalls[0];
-
-  if (!firstTool) {
-    return { score: 0, reason: 'No tool calls recorded' };
-  }
-
-  // Codex reads skill files via shell commands
-  if (firstTool.tool === 'command_execution') {
-    const cmd = String(firstTool.input ?? '');
-    if (cmd.includes(skillName)) {
-      return { score: 1, reason: `Skill "${skillName}" triggered via command: ${cmd}` };
-    }
-  }
-
-  // Check if skill file was read via file_change or other tools
-  if (firstTool.tool === 'file_change') {
-    const path = String((firstTool.input as Record<string, unknown>)?.path ?? '');
-    if (path.includes(skillName)) {
-      return { score: 1, reason: `Skill file accessed: ${path}` };
-    }
-  }
-
-  return { score: 0, reason: `First tool was "${firstTool.tool}" — not a skill invocation for "${skillName}"` };
-});
-```
-
-This approach is more flexible than config overrides — you can match any tool-call pattern,
-check multiple fields, and add provider-specific logic as needed.
-
---- references/subagent-pipeline.md ---
-# Subagent Pipeline — Running eval.yaml without CLI
-
-This reference documents the detailed procedure for running evaluations in subagent mode
-(`AGENT_EVAL_MODE=subagent`, the default). The orchestrating skill dispatches `executor`
-subagents to perform test cases and `grader` subagents to evaluate outputs.
-
-Read this reference when executing Step 3 (Run and Grade) in subagent mode.
-
-## Prerequisites
-
-- The eval.yaml file exists and contains valid test definitions
-- `agentv` CLI is installed (or run from source via `AGENTV_CLI=bun /path/to/cli.ts` in `.env`)
-- Read `references/eval-yaml-spec.md` for the full schema
-
-## Workspace Context
-
-Some evals pass prompt files directly and don't require a specific workspace — those run fine
-from anywhere. But evals that test agent behavior in a workspace (accessing skills, modifying
-repos, using tools across multiple repos) require the user to be in the **target workspace**
-(e.g., a multi-repo workspace set up by allagents). If the eval references workspace files or
-expects the agent to use skills, check that the current directory is the target workspace, not
-just the eval repo — and warn the user if it's wrong.
-
-## Executor Subagent Eligibility
-
-All providers except `cli` are eligible for executor subagents by default. To opt out a
-specific target, set `subagent_mode_allowed: false` in `.agentv/targets.yaml`:
-
-```yaml
-# .agentv/targets.yaml
-targets:
-  - name: my-target
-    provider: openai
-    model: ${{ OPENAI_MODEL }}
-    api_key: ${{ OPENAI_API_KEY }}
-    subagent_mode_allowed: false  # forces CLI invocation instead of executor subagent
-```
-
-When `subagent_mode_allowed: false`, the target falls back to CLI invocation via `agentv eval`
-even in subagent mode.
-
-## CLI Targets: Single Command
-
-For evals with CLI targets, `pipeline run` handles input extraction, target invocation, and
-code grading in one step. When `--out` is omitted, the output directory defaults to
-`.agentv/results/runs/<timestamp>` (same convention as `agentv eval`):
-
-```bash
-# Extract inputs and invoke all CLI targets in parallel:
-agentv pipeline run evals/repro.eval.yaml
-
-# Also run code graders inline (instead of using pipeline grade separately):
-agentv pipeline run evals/repro.eval.yaml --grader-type code
-```
-
-By default, `pipeline run` extracts inputs and invokes targets only. Pass `--grader-type code`
-to also run code-graders inline, or use `agentv pipeline grade <run-dir>` as a separate step.
-
-The run directory is printed to stdout. Then continue to the grading and merge phases
-described in SKILL.md Step 3.
-
-## Non-CLI Targets: Executor Subagents
-
-When the target provider is not `cli`, check `manifest.json` → `target.subagent_mode_allowed`.
-If `true` (default for all non-CLI providers), the subagent IS the target. If `false` (user
-opted out via `subagent_mode_allowed: false` in `.agentv/targets.yaml`), fall back to
-`agentv eval` CLI mode instead.
-
-### Step 1: Extract inputs
-
-```bash
-# Defaults to .agentv/results/runs/<timestamp>
-agentv pipeline input evals/repro.eval.yaml
-```
-
-This creates a run directory with per-test `input.json`, `invoke.json`,
-`criteria.md`, and grader configs.
-
-### Step 2: Dispatch executor subagents
-
-Read `agents/executor.md`. Launch one `executor` subagent **per test case**, all in parallel.
-Each subagent receives the test directory path, reads `input.json`, performs the task using
-its own tools, and writes `response.md`.
-
-Example: 5 tests = 5 executor subagents launched simultaneously.
-
-```
-# Per executor subagent:
-#   - Reads <run-dir>/<test-id>/input.json
-#   - Performs the task
-#   - Writes <run-dir>/<test-id>/response.md
-```
-
-### After executors complete: read results from disk
-
-When all executor subagents have finished, **read `response.md` directly from disk** — do
-NOT use `read_agent` to fetch results. The executors wrote their outputs to the run directory.
-
-```bash
-# Verify all responses exist:
-for d in <run-dir>/<evalset>/*/; do
-  echo "$(basename $d): $(ls "$d"/response.md 2>/dev/null && echo OK || echo MISSING)"
-done
-```
-
-If any `response.md` is MISSING, re-run that specific executor subagent. Do not proceed to
-grading until all responses are present.
-
-### Step 3 onward: Grade and merge
-
-See SKILL.md Step 3 "Grading" section for the three-phase grading process (code graders →
-LLM grading → merge and validate).
-
-## Step-by-Step Fine-Grained Control (CLI targets)
-
-Use individual commands when you need control over each step with CLI targets:
-
-```bash
-# Step 1: Extract inputs (defaults to .agentv/results/runs/<timestamp>)
-agentv pipeline input evals/repro.eval.yaml
-
-# Step 2: run_tests.py invokes CLI targets (or use pipeline run instead)
-
-# Step 3: Run code graders
-agentv pipeline grade <run-dir>
-
-# Step 4: Subagent does LLM grading, writes results to llm_grader_results/<name>.json per test
-
-# Step 5: Merge scores (writes index.jsonl with full scores[] for dashboard)
-agentv pipeline bench <run-dir>
-
-# Step 6: Validate
-agentv results validate <run-dir>
-```
-
-## LLM Grading JSON Format
-
-The agent reads `llm_graders/<name>.json` for each test, grades the response using the prompt
-content, and produces a scores JSON:
-
-```json
-{
-  "test-01": {
-    "relevance": {
-      "score": 0.85,
-      "assertions": [{"text": "Response is relevant", "passed": true, "evidence": "..."}]
-    }
-  }
-}
-```
-
-## Pipeline Bench and Dashboard
-
-`pipeline bench` merges LLM scores into `index.jsonl` with a full `scores[]` array per entry,
-matching the CLI-mode schema. The web dashboard (`agentv results serve`) reads this format
-directly — no separate conversion script is needed. Run `agentv results validate <run-dir>`
-to verify compatibility.
-
-## Output Structure
-
-The path hierarchy mirrors the CLI mode: `<evalset-name>` comes from the `name` field in
-the eval.yaml. The target is recorded in `manifest.json` — one run = one target.
-
-```
-.agentv/results/runs/<experiment>/<timestamp>/
-├── manifest.json                    ← eval metadata, target, test_ids
-├── index.jsonl                      ← per-test scores
-├── benchmark.json                   ← aggregate statistics
-└── <evalset-name>/                  ← eval.yaml "name" field, or eval file basename if absent (same as CLI mode)
-    └── <test-id>/                   ← test case id
-        ├── input.json               ← test input text + messages
-        ├── invoke.json              ← target command or agent instructions
-        ├── criteria.md              ← grading criteria
-        ├── response.md              ← target/agent output
-        ├── timing.json              ← execution timing
-        ├── code_graders/<name>.json     ← grader configs written by `pipeline input`: code-grader scripts AND built-in types (contains, regex, equals, etc.)
-        ├── llm_graders/<name>.json      ← LLM grader configs
-        ├── code_grader_results/<name>.json  ← code grader results
-        ├── llm_grader_results/<name>.json   ← LLM grader results (written by grader subagents; one file per grader)
-        └── grading.json              ← merged grading (written by `pipeline bench` — do NOT write here directly)
-```
-
---- references/autoresearch.md ---
-# Autoresearch Mode
-
-Autoresearch is an unattended eval-improve loop that runs multiple optimize cycles without human intervention. The user triggers it with natural language (e.g., "run autoresearch on this skill", "optimize this skill unattended"). No YAML schema changes or CLI flags are needed.
-
-## Automated Keep/Discard
-
-After each iteration, you can automatically decide whether to keep or discard the change using structured comparison output. This replaces manual judgment at steps 3–4 of the iteration loop (Step 5 in SKILL.md), except at human checkpoint iterations (3, 6, 9) where you must still present results to the user.
-
-### 1. Run the comparison
-
-After re-running test cases, compare the new results against the previous iteration's baseline:
-
-```bash
-agentv compare <baseline>.jsonl <candidate>.jsonl --json
-```
-
-Where `<baseline>.jsonl` is the `index.jsonl` from the previous best iteration and `<candidate>.jsonl` is the `index.jsonl` from the run you just completed.
-
-### 2. Parse the output
-
-The `--json` flag produces structured output:
-
-```json
-{
-  "summary": {
-    "wins": 3,
-    "losses": 1,
-    "ties": 6,
-    "mean_delta": 0.05
-  }
-}
-```
-
-- **wins**: number of test cases where the candidate scored higher than the baseline
-- **losses**: number of test cases where the candidate scored lower
-- **ties**: number of test cases with no score change
-- **mean_delta**: average score difference across all test cases (positive = candidate is better)
-
-### 3. Apply decision rules
-
-Use these rules in order:
-
-| Condition | Decision | Action |
-|-----------|----------|--------|
-| `wins > losses` | **KEEP** | Promote the candidate to the new baseline. Copy or note its `index.jsonl` path as the baseline for the next iteration. |
-| `wins <= losses` | **DISCARD** | Revert the prompt/skill/config change. The previous baseline remains. Try a different mutation on the next iteration. |
-| `mean_delta == 0` AND candidate prompt is shorter (fewer lines) | **KEEP** | Simpler prompts are preferred when performance is equal. Promote the candidate as the new baseline. |
-
-When `mean_delta == 0` and the candidate prompt is *not* shorter, treat it as a **DISCARD** — there's no reason to keep a change that adds complexity without improving results.
-
-### 4. Log the decision
-
-Before proceeding to the next iteration, log the decision and rationale so the user can review later:
-
-```
-Iteration 2: KEEP
-  wins=3, losses=1, ties=6, meanDelta=+0.05
-  Rationale: candidate wins outweigh losses (3 > 1)
-  Baseline promoted: .agentv/results/runs/20250101-120000/index.jsonl
-```
-
-```
-Iteration 3: DISCARD
-  wins=1, losses=2, ties=7, meanDelta=-0.03
-  Rationale: candidate losses outweigh wins (2 > 1)
-  Reverted to baseline: .agentv/results/runs/20250101-110000/index.jsonl
-  Next: try a different mutation
-```
-
-Include this log in your progress summary. At human checkpoints (iterations 3, 6, 9), present the full log of automated decisions since the last checkpoint alongside the current results.
-
-### 5. Integration with the iteration loop
-
-The automated keep/discard replaces the manual compare-and-present cycle (steps 3–4) during non-checkpoint iterations. The full flow becomes:
-
-1. Apply change to prompts/skills/config
-2. Re-run all test cases
-3. Run `agentv compare baseline.jsonl candidate.jsonl --json`
-4. Apply keep/discard rules → promote or revert
-5. Log the decision
-6. If this is iteration 3, 6, or 9 → present progress to the user (human checkpoint)
-7. Check stop conditions → continue or stop
-
-Both modes coexist: if the user is actively reviewing results, present to them as before. If the user has asked you to iterate autonomously, use automated keep/discard and only pause at human checkpoints.
-
----
-
-## Prerequisites
-
-- An eval file (`EVAL.yaml` or `evals.json`) must exist for the artifact being optimized.
-- The artifact must be a file or directory (SKILL.md, prompt template, agent config, or a directory of related files like a skill with references/).
-- The user should have run at least one interactive eval cycle to build confidence in eval quality before going unattended.
-
-## The loop
-
-```
-1. RUN EVAL   — agentv eval with current artifact
-2. ANALYZE    — dispatch analyzer subagent on results
-3. DECIDE     — if score > best_score: KEEP, else DROP (automated keep/discard above)
-4. MUTATE     — dispatch mutator subagent with failure analysis (agents/mutator.md)
-5. GOTO 1     — until convergence or max_cycles
-```
-
-## Experiment naming
-
-Derive the experiment name from the artifact: `autoresearch-<name>` (e.g., `autoresearch-pdf-skill`). The user can also provide a custom name.
-
-## Artifact mutation flow
-
-The mutator rewrites artifacts in the working tree in place. **Git is used for versioning** — HEAD always contains the best-known version:
-
-1. Record the starting commit SHA before the first cycle: `initial_sha=$(git rev-parse HEAD)`.
-2. On each **KEEP**: `git add <artifact-path> && git commit -m "autoresearch cycle N: <mutation summary>"`.
-3. On each **DROP**: `git checkout -- <artifact-path>` (restores working tree to HEAD, the last KEEP commit).
-4. The eval always runs against the real file path — no temp files or indirection.
-5. The mutator can reference the original via `git show <initial_sha>:<path>`.
-
-## How the skill invokes eval
-
-Shell out to `agentv eval <eval-path> --experiment autoresearch-<name>` via the Bash tool, same as the existing interactive bench workflow.
-
-## Artifact layout
-
-Each cycle is a standard eval run. Autoresearch session metadata lives in `_autoresearch/` within the experiment directory:
-
-```
-.agentv/results/runs/<experiment>/
-  _autoresearch/
-    iterations.jsonl               # one line per cycle — data for chart + mutator
-    trajectory.html                # live-updating score trajectory chart
-  2026-04-15T10-30-00/             # cycle 1 — standard run artifacts
-    index.jsonl
-    grading.json
-    timing.json
-    benchmark.json
-    report.html
-  2026-04-15T10-35-00/             # cycle 2 — standard run artifacts
-    ...
-```
-
-No `original.md` or `best.md` files — git history serves as the backup. The `_` prefix convention distinguishes workflow folders from timestamped run dirs.
-
-## iterations.jsonl
-
-One JSON object per line, one line per cycle:
-
-```jsonl
-{"cycle":1,"score":0.65,"decision":"keep","cost_usd":0.12,"assertions":{"IDENTIFIES_BUG":0.8,"SUGGESTS_FIX":0.4},"mutation":"added explicit null-check instruction","run_dir":"2026-04-15T10-30-00","timestamp":"2026-04-15T10:32:15Z"}
-```
-
-Fields: `cycle` (1-indexed), `score` (overall pass rate 0–1), `decision` ("keep" or "drop"), `cost_usd` (eval run cost), `assertions` (per-assertion pass rates), `mutation` (one-line description of what changed), `run_dir` (timestamped directory name), `timestamp` (ISO 8601).
-
-## trajectory.html
-
-A standalone HTML chart file with embedded Chart.js. Copy the template from `scripts/trajectory.html` into the `_autoresearch/` directory. It fetches `iterations.jsonl` from the same directory on each auto-refresh — no data injection needed. Shows:
-
-- Score over iterations (line chart) with KEEP (green) / DISCARD (red) markers
-- Per-assertion pass rates over iterations
-- Cumulative cost across iterations
-- Best vs original score summary
-
-Auto-refreshes every 2 seconds during the loop. Becomes static after completion (remove the auto-refresh meta tag on final update).
-
-## Convergence
-
-Stop after **3** consecutive cycles with no improvement (no KEEP). Also stop at **max_cycles** (default 10). Either limit can be overridden by the user.
-
-## Human checkpoints
-
-Autoresearch mode **skips** human checkpoints at iterations 3/6/9. The user opted in to unattended operation by requesting autoresearch.
-
-## Context hygiene
-
-The orchestrator must run indefinitely without exhausting its context window. To do this:
-
-- **Never read eval results, artifacts, or transcripts into your own context.** Use bash commands (jq, agentv CLI) that output small structured summaries.
-- **Delegate all heavy reading to subagents.** The mutator reads artifacts, grading results, and transcripts from disk — you pass it paths, not content.
-- **Use bash for all file I/O** in the loop body: appending to `iterations.jsonl`, git operations, score extraction. The only tool calls per cycle should be bash commands and one subagent dispatch (mutator).
-- **trajectory.html auto-loads `iterations.jsonl`** via fetch — no need to read or update the HTML file after initial copy.
-
-## Procedure
-
-Follow this step-by-step procedure to execute autoresearch:
-
-### 1. Setup
-
-1. Determine the **artifact path** (file or directory to optimize) and **eval path** (EVAL.yaml or evals.json).
-2. Detect **artifact mode**: `file` if the artifact path is a file, `directory` if it's a directory.
-3. Derive the **experiment name**: `autoresearch-<name>` from the artifact filename/dirname, or use a user-provided name.
-4. Set the experiment directory: `.agentv/results/runs/<experiment>/`.
-5. Create the `_autoresearch/` subdirectory inside the experiment directory.
-6. Record `initial_sha=$(git rev-parse HEAD)` — the commit before any mutations.
-7. Copy `scripts/trajectory.html` to `_autoresearch/trajectory.html`.
-8. Initialize variables:
-   - `best_score = 0`
-   - `convergence_count = 0`
-   - `cycle = 1`
-   - `max_cycles = 10` (or user-specified)
-   - `max_convergence = 3` (or user-specified)
-
-### 2. Main loop
-
-Repeat while `cycle <= max_cycles` and `convergence_count < max_convergence`:
-
-**a. Run eval**
-
-```bash
-agentv eval <eval-path> --experiment autoresearch-<name>
-```
-
-**b. Extract scores (bash only — do NOT read result files into your context)**
-
-Find the latest timestamped directory in the experiment folder. Use bash/jq to extract small structured values:
-
-```bash
-# Find latest run dir
-RUN_DIR=$(ls -td <experiment-dir>/20*/ | head -1)
-
-# Overall score (mean of all scores in index.jsonl)
-SCORE=$(jq -sr '[.[].scores[].score] | add / length' "$RUN_DIR/index.jsonl")
-
-# Per-assertion pass rates as JSON object
-PASS_RATES=$(jq -sr '[.[].scores[]] | group_by(.type) | map({key: .[0].type, value: (map(.score) | add / length)}) | from_entries' "$RUN_DIR/index.jsonl")
-
-# Cost (if timing.json exists)
-COST=$(jq -r '.cost_usd // 0' "$RUN_DIR/timing.json" 2>/dev/null || echo 0)
-```
-
-Capture only these small outputs (`SCORE`, `PASS_RATES`, `COST`) — never read the full JSONL into context.
-
-**c. Update iterations.jsonl (bash only)**
-
-After the KEEP/DROP decision (step e), append one JSON line via bash:
-
-```bash
-echo '{"cycle":'$CYCLE',"score":'$SCORE',"decision":"'$DECISION'","cost_usd":'$COST',"assertions":'$PASS_RATES',"mutation":"'"$MUTATION_DESC"'","run_dir":"'"$(basename $RUN_DIR)"'","timestamp":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'"}' >> <experiment-dir>/_autoresearch/iterations.jsonl
-```
-
-**d. trajectory.html — no action needed**
-
-The trajectory chart fetches `iterations.jsonl` directly via HTTP on each auto-refresh. No file manipulation required after the initial copy in setup.
-
-**e. Decide: KEEP or DROP**
-
-Apply the automated keep/discard rules from the section above:
-
-1. Run `agentv compare <baseline>.jsonl <candidate>.jsonl --json` where `<baseline>` is the best iteration's `index.jsonl` (or the first run's `index.jsonl` for cycle 1) and `<candidate>` is this cycle's `index.jsonl`.
-2. If `wins > losses` → **KEEP**.
-3. If `wins <= losses` → **DISCARD**.
-4. If `mean_delta == 0` and the artifact is simpler → **KEEP** (simpler is better at equal performance). Simplicity: for files, compare line count; for directories, compare total size via `du -sb`.
-
-For cycle 1, there is no baseline to compare against — always **KEEP** the first cycle.
-
-**f. If KEEP**
-
-- Update `best_score` to this cycle's score.
-- Commit the artifact: `git add <artifact-path> && git commit -m "autoresearch cycle N: <mutation summary>"`.
-- Record the current `index.jsonl` path as the new baseline for future comparisons.
-- Reset `convergence_count = 0`.
-
-**g. If DROP**
-
-- Revert the working tree to HEAD: `git checkout -- <artifact-path>` (for files) or `git checkout -- <artifact-path>/` (for directories).
-- Increment `convergence_count`.
-
-**h. Check stop conditions**
-
-If `convergence_count >= max_convergence` or `cycle >= max_cycles` → break out of the loop.
-
-**i. Mutate**
-
-Dispatch the **mutator** subagent (`agents/mutator.md`) with:
-- `artifact-path`: the file or directory to mutate
-- `artifact-mode`: `file` or `directory`
-- `initial-sha`: the starting commit SHA (for referencing the original via `git show`)
-- `pass-rates`: the `$PASS_RATES` JSON object from step (b) (small — just assertion names and rates)
-- `run-dir`: path to this cycle's run directory (the mutator reads `grading.json` and transcripts itself)
-- `iterations-path`: path to `_autoresearch/iterations.jsonl` (the mutator reads mutation history itself)
-- For directory mode: `focus-files` (optional — files most likely contributing to failures, derived from assertion names)
-
-**Do NOT pass failure descriptions, transcripts, or grading content** to the mutator — pass paths and let it read what it needs from disk. This keeps the orchestrator's context clean.
-
-The mutator rewrites artifacts in place. Verify the artifact was modified (e.g., `git diff --stat`) before continuing.
-
-**j. Continue**
-
-Increment `cycle` and return to step (a).
-
-### 3. Completion
-
-1. Finalize `trajectory.html`: remove the line containing `<!-- __AUTO_REFRESH__ -->` (which includes the `<meta http-equiv="refresh">` tag) so the chart becomes static.
-2. Log a final summary:
-   - Total cycles run
-   - Final best score vs original score (cycle 1)
-   - Number of KEEPs and DROPs
-   - Total cost across all cycles
-   - The optimized artifact is in the working tree (and the latest commit)
-   - Run `git diff <initial_sha>` to see total changes from the original
-   - Run `git log --oneline <initial_sha>..HEAD` to see the mutation history
-   - Path to `_autoresearch/trajectory.html` (the score chart)
-3. Present results to the user with a recommendation: adopt the optimized version, revert to original (`git checkout <initial_sha> -- <artifact-path>`), or continue iterating interactively.
-
-## Interactive/autonomous hybrid
-
-Users can start in interactive mode (the existing Step 3–5 loop with human checkpoints), build confidence in their eval quality, and then switch to autoresearch mode to run unattended. The two modes share the same eval infrastructure and artifact layout — autoresearch simply automates the keep/discard decisions and removes human checkpoints.
-
-## Model empathy recommendation
-
-For best results, use same-model pairings: the meta-agent running autoresearch should match the model used by the task agent being evaluated (e.g., Claude optimizing a Claude agent, GPT optimizing a GPT agent). Per AutoAgent research findings, same-model pairings produce better mutations because the optimizer has implicit knowledge of how the target model interprets instructions.
-
---- references/migrating-from-skill-creator.md ---
-# Migrating from Skill-Creator to AgentV Lifecycle Skill
-
-This reference covers how to use AgentV's unified agent-evaluation lifecycle skill (`agentv-bench`) with evals.json files originally created for Anthropic's skill-creator.
-
-## Drop-in Replacement
-
-AgentV runs skill-creator's evals.json directly — no conversion required:
-
-```bash
-# Run evals.json with AgentV
-agentv eval evals.json
-
-# Or run a single assertion offline (no API keys)
-agentv eval assert <grader-name> --agent-output "..." --agent-input "..."
-```
-
-AgentV automatically:
-- Promotes `prompt` → input messages
-- Promotes `expected_output` → reference answer
-- Converts `assertions` → LLM-grader graders
-- Resolves `files[]` paths relative to the evals.json directory
-
-If you're using the `agentv-bench` skill, it orchestrates these same AgentV commands. Code graders, grading, and artifact generation remain in AgentV core; the skill just orchestrates and summarizes the existing outputs.
-
-## What You Gain
-
-Moving from skill-creator's eval loop to AgentV's lifecycle skill gives you:
-
-| Capability | skill-creator | AgentV lifecycle skill |
-|-----------|---------------|----------------------|
-| Workspace isolation | ❌ | ✅ Clone repos, run setup/teardown scripts |
-| Code graders | ❌ | ✅ Python/TypeScript grader scripts via `defineCodeGrader()` |
-| Tool trajectory scoring | ❌ | ✅ Evaluate tool call sequences |
-| Multi-provider comparison | with-skill vs without-skill | N-way: Claude, GPT, Copilot, Gemini, custom CLI |
-| Multi-turn evaluation | ❌ | ✅ Conversation tracking with `conversation_id` |
-| Blind comparison | ❌ | ✅ Judge doesn't know which is baseline |
-| Deterministic upgrade suggestions | ❌ | ✅ LLM-grader → contains/regex/is-json |
-| Human review checkpoint | ❌ | ✅ Structured feedback gate |
-| Workspace file tracking | ❌ | ✅ Evaluate by diffing workspace files |
-| Agent mode (no API keys) | ❌ | ✅ Uses grader agent in agent mode |
-
-## Artifact Compatibility
-
-AgentV's companion artifacts are compatible with skill-creator's eval-viewer:
-
-| Artifact | Format | Compatible with eval-viewer |
-|----------|--------|---------------------------|
-| `<test-id>/grading.json` | Per-assertion evidence with claims | ✅ Superset of skill-creator's per-test grading format |
-| `benchmark.json` | Aggregate pass rates, timing, patterns | ✅ Superset of Agent Skills benchmark format |
-| Results JSONL | Per-test results | ✅ Standard JSONL format |
-
-AgentV's schemas are supersets — they include all fields skill-creator expects, plus additional fields (claims extraction, pattern analysis, deterministic upgrade candidates). Tools that read skill-creator artifacts will read AgentV artifacts correctly, ignoring the extra fields.
-
-The optimizer scripts layer reads those same artifacts directly:
-- `aggregate-benchmark.ts` consumes `benchmark.json`, `timing.json`, and results JSONL
-- `generate-report.ts` and `eval-viewer/generate-review.ts` render review output from AgentV artifacts
-- `improve-description.ts` proposes follow-up experiments from benchmark/grading observations
-
-## Graduating to EVAL.yaml
-
-When evals.json becomes limiting, convert to EVAL.yaml for the full feature set:
-
-```bash
-# Convert evals.json to EVAL.yaml
-agentv convert evals.json
-
-# Edit the generated YAML to add workspace config, code graders, etc.
-# Then run with the full lifecycle
-agentv eval eval.yaml
-```
-
-EVAL.yaml unlocks:
-- **Workspace setup/teardown** — clone repos, install dependencies, clean up after tests
-- **Code graders** — write graders in Python or TypeScript, not just LLM prompts
-- **Rubric-based grading** — multi-dimensional scoring with weighted criteria
-- **Retry policies** — automatic retries for flaky tests with configurable backoff
-- **Test groups** — organize tests by category with shared config
-- **Multi-turn conversations** — test agent interactions across multiple turns
-
-## What Stays in Skill-Creator
-
-AgentV does NOT replace these skill-creator capabilities:
-
-- **Trigger optimization** — optimizing when/how a skill is triggered
-- **.skill packaging** — bundling skills for distribution
-- **Skill authoring** — creating new SKILL.md files from scratch
-- **Skill discovery** — finding and installing skills
-
-AgentV focuses on the **evaluation and optimization loop**. Skill-creator focuses on **skill authoring and packaging**. They are complementary — use skill-creator to write the skill, use AgentV to evaluate and optimize it.
-
-## Example Workflow
-
-```
-1. Author a skill with skill-creator
-2. skill-creator generates evals.json
-3. Run evals.json through AgentV's lifecycle skill for richer evaluation:
-   - Workspace isolation (test in a real repo)
-   - Multi-provider comparison (does the skill work with GPT too?)
-   - Blind comparison (is the new version actually better?)
-   - Deterministic upgrades (replace vague LLM graders with precise checks)
-4. Use AgentV's optimization loop to refine the skill's prompts
-5. Return to skill-creator for packaging and distribution
-```
-
---- scripts/trajectory.html ---
-<!DOCTYPE html>
-<html lang="en">
-<head>
-  <meta charset="UTF-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  <meta http-equiv="refresh" content="2"> <!-- __AUTO_REFRESH__ -->
-  <title>AgentV Autoresearch Trajectory</title>
-  <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
-  <style>
-    /*
-     * Visual language aligned with Studio DESIGN.md:
-     *   - bg-gray-950 (#030712) canvas, bg-gray-900 (#111827) containers
-     *   - border-gray-800 (#1f2937) borders, divide-gray-800/50 row dividers
-     *   - Cyan-only accent (#22d3ee), emerald/red for data status
-     *   - System sans-serif, tabular-nums on every number
-     *   - font-medium (500) max for headings, no bold (700)
-     *   - rounded-lg (8px) containers, rounded-md (6px) badges/buttons
-     *   - No drop shadows — borders carry elevation
-     */
-    *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
-
-    body {
-      font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont,
-                   "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
-      background: #030712;  /* bg-gray-950 */
-      color: #d1d5db;       /* text-gray-300 */
-      padding: 24px;
-      line-height: 1.5;
-      font-size: 0.875rem;  /* text-sm default */
-    }
-
-    h1 {
-      text-align: center;
-      font-size: 1.5rem;    /* text-2xl */
-      font-weight: 600;     /* font-semibold — ceiling */
-      color: #fff;          /* text-white */
-      margin-bottom: 8px;
-    }
-
-    .subtitle {
-      text-align: center;
-      font-size: 0.875rem;
-      color: #6b7280;       /* text-gray-500 */
-      margin-bottom: 24px;
-    }
-
-    /* Summary cards */
-    .summary {
-      display: grid;
-      grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
-      gap: 12px;
-      margin-bottom: 24px;  /* space-y-6 */
-      max-width: 900px;
-      margin-left: auto;
-      margin-right: auto;
-    }
-
-    .card {
-      background: #111827;      /* bg-gray-900 */
-      border: 1px solid #1f2937; /* border-gray-800 */
-      border-radius: 8px;       /* rounded-lg */
-      padding: 14px 16px;
-      text-align: center;
-    }
-
-    .card .label {
-      font-size: 0.75rem;       /* text-xs */
-      text-transform: uppercase;
-      letter-spacing: 0.05em;   /* tracking-wider */
-      color: #6b7280;           /* text-gray-500 */
-      margin-bottom: 4px;
-    }
-
-    .card .value {
-      font-size: 1.5rem;
-      font-weight: 500;         /* font-medium — NOT 700 */
-      color: #fff;
-      font-variant-numeric: tabular-nums;
-    }
-
-    .card .value.positive { color: #34d399; } /* text-emerald-400 */
-    .card .value.negative { color: #f87171; } /* text-red-400 */
-    .card .value.neutral  { color: #22d3ee; } /* text-cyan-400 */
-
-    /* Charts grid */
-    .charts {
-      display: grid;
-      grid-template-columns: repeat(auto-fit, minmax(380px, 1fr));
-      gap: 16px;                /* space-y-4 */
-      margin-bottom: 24px;
-      max-width: 1400px;
-      margin-left: auto;
-      margin-right: auto;
-    }
-
-    .chart-container {
-      background: #111827;
-      border: 1px solid #1f2937;
-      border-radius: 8px;
-      padding: 16px;
-    }
-
-    .chart-container h2 {
-      font-size: 0.875rem;
-      font-weight: 500;         /* font-medium */
-      color: #9ca3af;           /* text-gray-400 */
-      margin-bottom: 12px;
-      text-align: center;
-    }
-
-    canvas { width: 100% !important; }
-
-    /* Iterations table */
-    .table-section {
-      max-width: 1400px;
-      margin: 0 auto;
-    }
-
-    .table-section h2 {
-      font-size: 1.25rem;       /* text-xl */
-      font-weight: 600;         /* font-semibold */
-      color: #fff;              /* text-white */
-      margin-bottom: 12px;
-    }
-
-    .table-wrapper {
-      overflow-x: auto;
-      border-radius: 8px;       /* rounded-lg */
-      border: 1px solid #1f2937;
-    }
-
-    table {
-      width: 100%;
-      border-collapse: collapse;
-      font-size: 0.875rem;      /* text-sm */
-      text-align: left;
-    }
-
-    thead {
-      background: rgba(17, 24, 39, 0.5); /* bg-gray-900/50 */
-      border-bottom: 1px solid #1f2937;
-    }
-
-    th {
-      padding: 12px 16px;       /* px-4 py-3 */
-      text-align: left;
-      font-weight: 500;         /* font-medium */
-      color: #9ca3af;           /* text-gray-400 */
-      white-space: nowrap;
-    }
-
-    td {
-      padding: 12px 16px;       /* px-4 py-3 */
-      color: #d1d5db;           /* text-gray-300 */
-    }
-
-    tbody tr { border-top: 1px solid rgba(31, 41, 55, 0.5); } /* divide-gray-800/50 */
-    tbody tr:hover { background: rgba(17, 24, 39, 0.3); }     /* hover:bg-gray-900/30 */
-
-    td.num {
-      text-align: right;
-      font-variant-numeric: tabular-nums;
-      color: #9ca3af;           /* text-gray-400 for numbers */
-    }
-
-    .badge {
-      display: inline-block;
-      padding: 2px 8px;
-      border-radius: 6px;       /* rounded-md */
-      font-size: 0.75rem;
-      font-weight: 500;         /* font-medium */
-      text-transform: uppercase;
-    }
-
-    .badge.keep {
-      border: 1px solid rgba(6, 78, 59, 0.6);  /* border-emerald-900/60 */
-      background: rgba(6, 78, 59, 0.3);         /* bg-emerald-950/30 */
-      color: #34d399;                            /* text-emerald-400 */
-    }
-    .badge.discard {
-      border: 1px solid rgba(127, 29, 29, 0.6); /* border-red-900/60 */
-      background: rgba(127, 29, 29, 0.3);       /* bg-red-950/30 */
-      color: #f87171;                            /* text-red-400 */
-    }
-
-    .empty-state {
-      text-align: center;
-      padding: 32px 20px;       /* p-8 for empty states */
-      color: #4b5563;           /* text-gray-600 */
-      font-size: 0.875rem;
-    }
-
-    .mutation-cell {
-      max-width: 320px;
-      overflow: hidden;
-      text-overflow: ellipsis;
-      white-space: nowrap;
-    }
-  </style>
-</head>
-<body>
-
-<h1>AgentV Autoresearch Trajectory</h1>
-<div class="subtitle" id="subtitle"></div>
-
-<!-- Summary cards -->
-<div class="summary" id="summary"></div>
-
-<!-- Charts -->
-<div class="charts" id="charts-area">
-  <div class="chart-container">
-    <h2>Score over Iterations</h2>
-    <canvas id="scoreChart"></canvas>
-  </div>
-  <div class="chart-container">
-    <h2>Per-Assertion Pass Rates</h2>
-    <canvas id="assertionChart"></canvas>
-  </div>
-  <div class="chart-container">
-    <h2>Cumulative Cost (USD)</h2>
-    <canvas id="costChart"></canvas>
-  </div>
-</div>
-
-<!-- Iterations table -->
-<div class="table-section">
-  <h2>Iteration Log</h2>
-  <div class="table-wrapper" id="table-wrapper"></div>
-</div>
-
-<script>
-// ── Data loading ─────────────────────────────────────────────────────
-// Fetch iterations.jsonl from the same directory. Each line is one JSON object.
-// Auto-refreshes every 2s (via the meta tag) so the chart stays live.
-async function loadData() {
-  try {
-    const resp = await fetch('iterations.jsonl');
-    if (!resp.ok) return [];
-    const text = await resp.text();
-    return text.trim().split('\n').filter(Boolean).map(line => JSON.parse(line));
-  } catch (_) {
-    return [];
-  }
-}
-
-// ── Helpers ────────────────────────────────────────────────────────────
-const fmtPct = (n) => (n == null ? '—' : Math.round(Number(n) * 100) + '%');
-const fmtPctDelta = (n) => (n == null ? '—' : (n >= 0 ? '+' : '') + Math.round(Number(n) * 100) + '%');
-const fmtUsd = (n) => (n == null ? '—' : '$' + Number(n).toFixed(4));
-
-// ── Chart.js defaults (aligned with Studio DESIGN.md) ─────────────────
-Chart.defaults.color = '#9ca3af';  // text-gray-400
-Chart.defaults.borderColor = 'rgba(31, 41, 55, 0.5)'; // border-gray-800/50
-Chart.defaults.font.family = 'ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif';
-
-// Cyan accent + Studio data-tone palette
-const CHART_COLORS = [
-  '#22d3ee', // cyan-400 (primary accent)
-  '#34d399', // emerald-400
-  '#f472b6', // pink-400
-  '#facc15', // yellow-400
-  '#c084fc', // purple-400
-  '#fb923c', // orange-400
-  '#a3e635', // lime-400
-  '#e879f9', // fuchsia-400
-  '#60a5fa', // blue-400
-  '#fbbf24', // amber-400
-];
-
-// ── Render ─────────────────────────────────────────────────────────────
-function render(data) {
-  if (!data || data.length === 0) {
-    document.getElementById('subtitle').textContent = 'Waiting for data…';
-    document.getElementById('charts-area').style.display = 'none';
-    document.getElementById('summary').innerHTML = '';
-    document.getElementById('table-wrapper').innerHTML =
-      '<div class="empty-state">No iterations recorded yet. The chart will update automatically.</div>';
-    return;
-  }
-
-  // Validate each iteration has at minimum a cycle and score
-  const valid = data.filter(it => it && typeof it.cycle === 'number' && typeof it.score === 'number');
-  if (valid.length === 0) {
-    document.getElementById('subtitle').textContent = 'Data present but missing required fields (cycle, score).';
-    document.getElementById('charts-area').style.display = 'none';
-    document.getElementById('summary').innerHTML = '';
-    document.getElementById('table-wrapper').innerHTML =
-      '<div class="empty-state">Iterations data is malformed. Each entry needs at minimum: cycle (number), score (number).</div>';
-    return;
-  }
-  const sorted = [...valid].sort((a, b) => a.cycle - b.cycle);
-
-  // ── Summary ──────────────────────────────────────────────────────────
-  const originalScore = sorted[0].score;
-  const bestIteration = sorted.reduce((best, it) => (it.score > best.score ? it : best), sorted[0]);
-  const bestScore = bestIteration.score;
-  const totalCycles = sorted.length;
-  const totalCost = sorted.reduce((sum, it) => sum + (parseFloat(it.cost_usd) || 0), 0);
-  const delta = bestScore - originalScore;
-
-  document.getElementById('subtitle').textContent =
-    `${totalCycles} cycle${totalCycles !== 1 ? 's' : ''} completed`;
-
-  const deltaClass = delta > 0 ? 'positive' : delta < 0 ? 'negative' : 'neutral';
-
-  document.getElementById('summary').innerHTML = `
-    <div class="card"><div class="label">Original Score</div><div class="value neutral">${fmtPct(originalScore)}</div></div>
-    <div class="card"><div class="label">Best Score</div><div class="value positive">${fmtPct(bestScore)}</div></div>
-    <div class="card"><div class="label">Improvement</div><div class="value ${deltaClass}">${fmtPctDelta(delta)}</div></div>
-    <div class="card"><div class="label">Total Cycles</div><div class="value">${totalCycles}</div></div>
-    <div class="card"><div class="label">Total Cost</div><div class="value">${fmtUsd(totalCost)}</div></div>
-  `;
-
-  // ── Chart data prep ──────────────────────────────────────────────────
-  const labels = sorted.map(it => it.cycle);
-  const scores = sorted.map(it => it.score);
-  const decisions = sorted.map(it => (it.decision || '').toLowerCase());
-
-  // Cumulative cost (parseFloat guards against string or null values)
-  let cumCost = 0;
-  const cumCosts = sorted.map(it => { cumCost += parseFloat(it.cost_usd) || 0; return cumCost; });
-
-  // Assertion keys (union of all)
-  const assertionKeys = [...new Set(sorted.flatMap(it => Object.keys(it.assertions || {})))].sort();
-
-  document.getElementById('charts-area').style.display = '';
-
-  // ── 1. Score chart (cyan line, emerald/red decision dots) ──────────
-  const scoresPct = scores.map(s => s * 100);
-  const scoreCtx = document.getElementById('scoreChart').getContext('2d');
-  new Chart(scoreCtx, {
-    type: 'line',
-    data: {
-      labels,
-      datasets: [{
-        label: 'Score',
-        data: scoresPct,
-        borderColor: '#22d3ee',                    // cyan-400
-        backgroundColor: 'rgba(34, 211, 238, 0.1)', // cyan-400/10
-        borderWidth: 2,
-        tension: 0.2,
-        fill: true,
-        pointRadius: 6,
-        pointBorderWidth: 2,
-        pointBackgroundColor: decisions.map(d => d === 'keep' ? '#34d399' : '#f87171'),
-        pointBorderColor: decisions.map(d => d === 'keep' ? '#065f46' : '#7f1d1d'),
-      }],
-    },
-    options: {
-      responsive: true,
-      scales: {
-        y: { min: 0, max: 100, ticks: { stepSize: 10, callback: v => v + '%' } },
-        x: { title: { display: true, text: 'Cycle' } },
-      },
-      plugins: {
-        legend: { display: false },
-        tooltip: {
-          callbacks: {
-            label: (ctx) => `Score: ${Math.round(ctx.parsed.y)}%`,
-            afterLabel: (ctx) => `Decision: ${decisions[ctx.dataIndex] || '—'}`,
-          },
-        },
-      },
-    },
-  });
-
-  // ── 2. Assertion chart ───────────────────────────────────────────────
-  const assertCtx = document.getElementById('assertionChart').getContext('2d');
-  new Chart(assertCtx, {
-    type: 'line',
-    data: {
-      labels,
-      datasets: assertionKeys.map((key, i) => ({
-        label: key,
-        data: sorted.map(it => { const v = (it.assertions || {})[key]; return v == null ? null : v * 100; }),
-        borderColor: CHART_COLORS[i % CHART_COLORS.length],
-        backgroundColor: 'transparent',
-        borderWidth: 2,
-        tension: 0.2,
-        pointRadius: 3,
-        spanGaps: true,
-      })),
-    },
-    options: {
-      responsive: true,
-      scales: {
-        y: { min: 0, max: 100, ticks: { stepSize: 20, callback: v => v + '%' } },
-        x: { title: { display: true, text: 'Cycle' } },
-      },
-      plugins: {
-        legend: { position: 'bottom', labels: { boxWidth: 12, padding: 10, font: { size: 11 } } },
-        tooltip: { callbacks: { label: (ctx) => `${ctx.dataset.label}: ${Math.round(ctx.parsed.y)}%` } },
-      },
-    },
-  });
-
-  // ── 3. Cost chart (cyan line) ─────────────────────────────────────
-  const costCtx = document.getElementById('costChart').getContext('2d');
-  new Chart(costCtx, {
-    type: 'line',
-    data: {
-      labels,
-      datasets: [{
-        label: 'Cumulative Cost (USD)',
-        data: cumCosts,
-        borderColor: '#22d3ee',                     // cyan-400
-        backgroundColor: 'rgba(34, 211, 238, 0.08)', // cyan-400/8
-        borderWidth: 2,
-        tension: 0.2,
-        fill: true,
-        pointRadius: 3,
-      }],
-    },
-    options: {
-      responsive: true,
-      scales: {
-        y: { ticks: { callback: v => '$' + v.toFixed(2) } },
-        x: { title: { display: true, text: 'Cycle' } },
-      },
-      plugins: { legend: { display: false } },
-    },
-  });
-
-  // ── Table ────────────────────────────────────────────────────────────
-  const rows = sorted.map(it => {
-    const dec = (it.decision || '').toLowerCase();
-    const badge = dec === 'keep'
-      ? '<span class="badge keep">keep</span>'
-      : '<span class="badge discard">drop</span>';
-    const ts = it.timestamp
-      ? new Date(it.timestamp).toLocaleString(undefined, { dateStyle: 'short', timeStyle: 'medium' })
-      : '—';
-    return `<tr>
-      <td class="num">${it.cycle}</td>
-      <td class="num">${fmtPct(it.score)}</td>
-      <td>${badge}</td>
-      <td class="mutation-cell" title="${(it.mutation || '').replace(/"/g, '&quot;')}">${it.mutation || '—'}</td>
-      <td class="num">${fmtUsd(it.cost_usd)}</td>
-      <td style="color:#6b7280">${ts}</td>
-    </tr>`;
-  }).join('');
-
-  document.getElementById('table-wrapper').innerHTML = `
-    <table>
-      <thead><tr>
-        <th style="text-align:right">Cycle</th>
-        <th style="text-align:right">Score</th>
-        <th>Decision</th>
-        <th>Mutation</th>
-        <th style="text-align:right">Cost</th>
-        <th>Timestamp</th>
-      </tr></thead>
-      <tbody>${rows}</tbody>
-    </table>
-  `;
-}
-
-loadData().then(render);
-</script>
-
-</body>
-</html>
diff --git a/evals/agentv-dev/skills/fixtures/agentv-bench.txt b/evals/agentv-dev/skills/fixtures/agentv-bench.txt
deleted file mode 100644
index 21a8f7271..000000000
--- a/evals/agentv-dev/skills/fixtures/agentv-bench.txt
+++ /dev/null
@@ -1,459 +0,0 @@
----
-name: agentv-bench
-description: >-
-  Run AgentV evaluations and optimize agents through eval-driven iteration.
-  Triggers: run evals, benchmark agents, optimize prompts/skills against evals, compare
-  agent outputs across providers, analyze eval results, offline evaluation of recorded sessions,
-  run autoresearch, optimize unattended, run overnight optimization loop.
-  Not for: writing/editing eval YAML without running (use agentv-eval-writer),
-  analyzing existing traces/JSONL without re-running (use agentv-trace-analyst).
----
-
-# AgentV Bench
-
-
-A skill for evaluating agents and iteratively improving them through data-driven optimization.
-
-At a high level, the process goes like this:
-
-- Understand what the agent does and what "good" looks like
-- Write evaluation test cases (EVAL.yaml or evals.json)
-- Run the agent on those test cases, grade the outputs
-- Analyze the results — what's working, what's failing, and why
-- Improve the agent's prompts/skills/config based on the analysis
-- Repeat until you're satisfied
-
-Your job when using this skill is to figure out where the user is in this process and then jump in and help them progress. Maybe they want to start from scratch — help them write evals, run them, and iterate. Maybe they already have results — jump straight to analysis and improvement.
-
-Be flexible. If the user says "I don't need a full benchmark, just help me debug this failure", do that instead.
-
-After the agent is working well, you can also run description optimization to improve skill triggering accuracy (see `references/description-optimization.md`).
-
-## Communicating with the user
-
-This skill is used by people across a wide range of familiarity with evaluation tooling. Pay attention to context cues:
-
-- "evaluation" and "benchmark" are borderline but OK in most cases
-- For "YAML", "grader", "assertion", "deterministic judge" — see serious cues from the user that they know what those mean before using them without explanation
-- Briefly explain terms if in doubt
-
-When presenting results, default to summary tables. Offer detail on request. In CI/headless mode, skip interactive prompts and exit with status codes.
-
----
-
-## Step 1: Understand the Agent
-
-Before running or optimizing, understand what you're working with.
-
-1. **Read the agent's artifacts** — prompts, skills, configs, recent changes. Understand the full picture: what tools are available, what the expected input/output looks like, what constraints exist.
-
-2. **Identify success criteria** — what does "good" look like for this agent? What are the edge cases? What would a failure look like? Talk to the user if this isn't clear from the artifacts alone.
-
-3. **Understand the target harness** — which provider runs the agent (Claude, GPT, Copilot CLI, Gemini, custom CLI)? This affects what grader types are available and how to run tests. Targets are configured in `.agentv/targets.yaml` (canonical location, searched from the eval file directory upward). Sensitive values like `api_key` must use `${{ ENV_VAR }}` syntax — literal secrets are rejected as a security guardrail.
-
-4. **Challenge assumptions** — if evals already exist, review their quality before running:
-   - Are the test cases testing the right things?
-   - Are assertions specific enough to catch real failures?
-   - Are there ambiguous or contradictory test cases?
-   - Flag eval issues before proceeding — running bad evals wastes time.
-
-5. **Check integrity** — ensure task prompts (what the agent receives) are not also used as grader prompts (how outputs are scored). If a prompt file appears in both locations, note the overlap and optimize only for the task purpose.
-
----
-
-## Step 2: Write Evaluations
-
-AgentV supports two evaluation formats:
-
-**EVAL.yaml** (native, full features) — supports workspaces, code graders, multi-turn conversations, tool trajectory scoring, workspace file tracking, multi-provider targets. Use this for agent evaluation.
-
-```yaml
-# example.eval.yaml
-tests:
-  - id: basic-code-review
-    input: "Review this TypeScript file for bugs and suggest improvements"
-    criteria: "Identifies the null pointer bug on line 12 and suggests a fix"
-    assertions:
-      - type: contains
-        value: "null"
-      - Review identifies the null pointer bug and suggests a concrete fix
-
-workspace:
-  template: ./workspace-template
-  hooks:
-    before_each:
-      reset: fast
-```
-
-Multi-skill evaluation is handled naturally via input messages — describe the task in the test input, and the agent uses whatever skills it needs.
-
-**evals.json** (skill-creator compatible) — auto-promoted to EVAL-equivalent format:
-- `prompt` → input messages
-- `expected_output` → reference answer
-- `assertions` → graders
-- `files[]` paths resolved relative to the evals.json location
-
-```json
-{
-  "skill_name": "my-agent",
-  "evals": [
-    {
-      "id": 1,
-      "prompt": "User's task prompt",
-      "expected_output": "Description of expected result",
-      "assertions": ["Output includes error handling", "Uses async/await"]
-    }
-  ]
-}
-```
-
-### Writing good test cases
-
-Start with 2-3 realistic test cases — the kind of thing a real user would actually say. Share them with the user before running: "Here are a few test cases I'd like to try. Do these look right, or do you want to add more?"
-
-Good assertions are objectively verifiable and have descriptive names. Subjective quality ("the output is good") is better evaluated qualitatively — don't force assertions onto things that need human judgment.
-
-**Grader types** (cheapest to most expensive): `exact`, `contains`, `regex`, `is-json`, `field-accuracy`, `composite`, `code-grader`, `tool-trajectory`, `llm-grader`. See `references/eval-yaml-spec.md` for full config and grading recipes for each type.
-
-Prefer deterministic graders over LLM graders whenever possible. If an assertion can be checked with `contains` or `regex`, don't use `llm-grader`.
-
----
-
-## Step 3: Run and Grade
-
-This section is one continuous sequence — don't stop partway through.
-
-Each run produces a new `.agentv/results/runs/<timestamp>/` directory automatically. Use timestamps to identify iterations when comparing runs.
-
-### Choosing a run mode
-
-**User instruction takes priority.** If the user says "run in subagent mode", "use subagent mode", or "use CLI mode", use that mode directly.
-
-If the user has not specified a mode, default to `subagent`.
-
-| `AGENT_EVAL_MODE` | Mode | How |
-|----------------------|------|-----|
-| `subagent` (default) | **Subagent mode** | Subagent-driven eval — parses eval.yaml, spawns executor + grader subagents. Zero CLI dependency. |
-| `cli` | **AgentV CLI** | `agentv eval <path>` — end-to-end, multi-provider |
-
-Set `AGENT_EVAL_MODE` in `.env` at the project root as the default when no mode is specified. If absent, default to `subagent`. **User instruction always overrides this.**
-
-**`subagent`** — Parses eval.yaml directly, spawns executor subagents to run each test case in the current workspace, then spawns grader subagents to evaluate all assertion types natively. No CLI or external API calls required. Read `references/subagent-pipeline.md` for the detailed procedure.
-
-**`cli`** — AgentV CLI handles execution, grading, and artifact generation end-to-end. Works with all providers. Use when you need multi-provider benchmarking or CLI-specific features.
-
-### Running evaluations
-
-**AgentV CLI mode** (end-to-end, EVAL.yaml):
-```bash
-agentv eval <eval-path> --output .agentv/artifacts/
-```
-
-**Subagent mode** — read `references/subagent-pipeline.md` for the detailed procedure. In brief: use `pipeline input` to extract inputs, dispatch one `executor` subagent per test case (all in parallel), then proceed to grading below.
-
-**Spawn all runs in the same turn.** For each test case that needs both a "with change" and a "baseline" run, launch them simultaneously. Don't run one set first and come back for the other — launch everything at once so results arrive around the same time.
-
-**Multi-target benchmarking:**
-```bash
-agentv eval <eval-path> --target claude --target gpt --target copilot
-```
-
-**Baseline strategy:**
-- **New agent**: baseline is "no prompt" or minimal prompt — same eval, no agent-specific configuration
-- **Improving existing**: snapshot the current version before editing (`cp -r <prompt-dir> <workspace>/prompt-snapshot/`), use as baseline throughout
-- **Multi-target**: each target is its own baseline — no need for a separate "without" run
-
-### While runs are in progress, draft graders
-
-Don't just wait for runs to finish — use this time productively. If assertions don't exist yet, draft them now. If they exist, review them and explain what they check to the user.
-
-Good assertions are *discriminating* — they pass when the agent genuinely succeeds and fail when it doesn't. An assertion that passes for both good and bad outputs is worse than no assertion.
-
-### As runs complete, capture timing data
-
-When each subagent task completes, you receive a notification containing `total_tokens` and `duration_ms`. **Save this data immediately** to `timing.json` in the run directory. See `references/schemas.md` for the timing.json schema.
-
-This is the only opportunity to capture this data — it comes through the task notification and isn't persisted elsewhere. Process each notification as it arrives.
-
-### After all executors complete
-
-Once all executor subagent notifications arrive, **read `response.md` directly from disk**.
-Do NOT use `read_agent` to fetch executor results — they are already written to the run
-directory. Verify all responses exist before proceeding:
-
-```bash
-for d in <run-dir>/<evalset>/*/; do
-  echo "$(basename $d): $(ls "$d"/response.md 2>/dev/null && echo OK || echo MISSING)"
-done
-```
-
-If any `response.md` is MISSING, re-run that specific executor subagent. Do not proceed to
-grading until all responses are present.
-
-### Grading
-
-**In CLI mode**, `agentv eval` handles all grading end-to-end — no manual phases needed.
-
-**In subagent mode**, grading has three phases. **All three are required — do not stop after phase 1.**
-
-**Phase 1: Code graders** (deterministic, zero-cost)
-
-```bash
-agentv pipeline grade <run-dir>
-```
-
-This evaluates all deterministic assertions against `response.md` files. Two types are handled:
-- **`code-grader` scripts** — external scripts executed against the response (arbitrary logic, any language)
-- **Built-in assertion types** — evaluated in-process: `contains`, `contains-any`, `contains-all`, `icontains`, `regex`, `equals`, `starts-with`, `ends-with`, `is-json`, and variants
-
-Both types are configured by `pipeline input` into `code_graders/<name>.json` and graded by `pipeline grade`. Results are written to `<test-id>/code_grader_results/<name>.json`. Alternatively, pass `--grader-type code` to `pipeline run` to run these inline.
-
-**Do not dispatch LLM grader subagents for tests that only have `contains`, `regex`, or other built-in assertions** — `pipeline grade` handles them entirely, at zero cost. To detect which tests need Phase 2, check whether `<test-id>/llm_graders/` contains any `.json` config files — `pipeline input` only writes there for `llm-grader` assertions. Tests with an empty (or missing) `llm_graders/` directory are done after Phase 1.
-
-**Phase 2: LLM grading** (semantic — do NOT skip this phase)
-
-Dispatch one `grader` subagent per (test × LLM grader) pair, **all in parallel**. Do not write a script to call an LLM API instead — the grader subagents use their own reasoning, which IS the LLM grading.
-Example: 5 tests × 2 LLM graders = 10 grader subagents launched simultaneously.
-
-**Do NOT dispatch a single grader for multiple tests.** Each subagent grades exactly one (test, grader) pair.
-
-**Before dispatching graders, read `agents/grader.md` and embed its full content as the system instructions in every grader subagent prompt.** The grader is a `general-purpose` task agent — there is no auto-resolved "grader" type. Without `agents/grader.md` embedded verbatim, the subagent has no grading process, no output format, and no file-path knowledge, and will produce empty or incorrect output.
-
-Each grader subagent (operating under `agents/grader.md` instructions):
-1. Reads `<test-id>/llm_graders/<name>.json` for the grading prompt
-2. Reads `<test-id>/response.md` for the candidate output
-3. Grades the response against the prompt criteria
-4. **Writes its result to disk**: `<run-dir>/<evalset>/<test-id>/llm_grader_results/<name>.json`
-5. Returns score (0.0–1.0) and per-assertion evidence to the orchestrator
-
-**Writing to disk is critical.** Assertion arrays are lost if accumulated only in the orchestrator's context across multiple batches (context summarization drops detail). Writing per-test results to `llm_grader_results/<name>.json` makes grading resumable and assertion evidence durable.
-
-The result file format is:
-```json
-{ "score": 0.85, "assertions": [{"text": "...", "passed": true, "evidence": "..."}] }
-```
-
-After **all** grader subagents complete, run Phase 3 directly.
-
-**Phase 3: Merge and validate**
-
-```bash
-agentv pipeline bench <run-dir>
-agentv results validate <run-dir>
-```
-
-`pipeline bench` reads LLM grader results from `llm_grader_results/<name>.json` per test automatically, merges with code-grader scores, computes weighted pass_rate, and writes `grading.json` + `index.jsonl` + `benchmark.json`.
-
-> **Diagnosing `pass_rate=0`:** If `pipeline bench` reports `pass_rate=0` across the board, do **not** assume the tests genuinely failed. First verify the grading pipeline ran correctly: check that `<test-id>/llm_grader_results/<name>.json` exists and is non-empty for each test. If these files are absent or empty, the grader subagents failed to produce output (most common cause: `agents/grader.md` was not embedded in the subagent prompts — see Phase 2). Treat `pass_rate=0` as a real signal only after confirming grader results exist.
-
-### Artifacts
-
-All artifacts use established schemas — see `references/schemas.md` for the full definitions. Do not modify the structure. Key artifacts per run:
-- **grading.json**: per-test assertions with `{text, passed, evidence}`, plus summary
-- **timing.json**: `{total_tokens, duration_ms, total_duration_seconds}`
-- **benchmark.json**: per-target aggregate `{pass_rate, time_seconds, tokens}`
-
-Write artifacts to `.agentv/artifacts/` or the iteration directory.
-
-### Workspace features (EVAL.yaml only)
-
-- **Workspace isolation** — clone repos, run setup/teardown hooks (before_all, before_each, after_each, after_all)
-- **Materialization modes** — `pooled` (reuse slots), `temp` (fresh per run), `static` (existing dir)
-- **Multi-repo** — clone multiple repos with sparse checkout and shallow clone support
-- **File change tracking** — grade by diffing workspace files before/after agent execution
-
----
-
-## Step 4: Analyze Results
-
-Once all runs are graded, analyze the results before attempting improvements.
-
-### Pattern analysis
-
-Read the JSONL results and look for:
-
-- **Always-pass tests** — assertion too loose or non-discriminating. If it passes for both good and bad outputs, it's not testing anything.
-- **Always-fail tests** — task impossible, eval broken, or assertion misconfigured. Don't optimize against broken evals.
-- **Flaky tests** — non-deterministic results across runs. Investigate before treating failures as real.
-- **Systematic failures** — same failure pattern across multiple tests. This usually points to a missing instruction or wrong approach.
-- **Deterministic upgrade candidates** — `llm-grader` assertions that could be replaced with `contains`, `regex`, or `is-json` (cheaper, faster, more reliable).
-
-### Dispatch subagents
-
-- **Dispatch `analyzer`** (read `agents/analyzer.md`) for a structured quality audit: deterministic upgrade suggestions, weak assertion detection, cost/quality flags, and benchmark pattern analysis.
-
-- **Dispatch `comparator`** (read `agents/comparator.md`) for blind N-way comparison between iterations or targets. The comparator blinds provider identities, generates task-specific rubrics, scores each output, then unblinds and attributes improvements.
-
-### Trace analysis
-
-Use CLI tools for deeper investigation:
-```bash
-agentv inspect <results-file>          # Detailed execution trace inspection
-agentv compare <file-a> <file-b>     # Structured diff between runs
-```
-
-Look for: tool call patterns, error recovery behavior, conversation flow, wasted steps.
-
-### Present results to the user
-
-Show a summary table:
-
-```
-| Test ID          | Score | Pass/Fail | Delta | Notes                    |
-|------------------|-------|-----------|-------|--------------------------|
-| basic-code-review| 0.85  | ✓ PASS    | +0.15 | Found the bug this time  |
-| edge-case-empty  | 0.00  | ✗ FAIL    | —     | Crashed on empty input   |
-```
-
-Highlight:
-- Current pass rate and delta from baseline
-- Comparison results (which target/iteration won and why)
-- Analyst observations the aggregate stats would hide
-
-Ask: "How does this look? Anything you'd change about the evals or the approach?"
-
----
-
-## Step 5: Improve
-
-This is the heart of the loop. You've run the test cases, analyzed the results, and now you need to make the agent better.
-
-### How to think about improvements
-
-1. **Generalize from the analysis.** You're iterating on a small eval set, but the agent will be used on many different inputs. Don't overfit to specific test cases. Rather than fiddly patches or oppressively rigid MUSTs, try different approaches and see what works. It's cheap to experiment.
-
-2. **Keep the prompt lean.** Read the execution transcripts, not just the final outputs. If the agent wastes time on unproductive steps, remove the instructions causing that. If it always ignores a section, that section isn't pulling its weight.
-
-3. **Explain the why.** Today's LLMs are smart. They have good theory of mind and can go beyond rote instructions when given good reasoning. If you find yourself writing ALWAYS or NEVER in all caps, that's a yellow flag — reframe as an explanation of why the thing matters. That's more humane, powerful, and effective.
-
-4. **Look for repeated work.** Read the transcripts from test runs and notice if the agent independently takes the same multi-step approach to something across cases. If all test runs result in writing the same helper script, bundle it. If every run makes the same mistake, the instruction is missing or unclear.
-
-### Applying changes
-
-- **Surgical edits**: ADD (new rule for a missing constraint), UPDATE (refine for clarity), DELETE (remove redundant or harmful rules), NEGATIVE CONSTRAINT (explicitly state what NOT to do)
-- **One change per iteration** to isolate effects. If you change three things and the score improves, you don't know which change helped.
-- **Variant tracking**: When a change helps some tests but hurts others, maintain 2-3 prompt variants. Compare variants to find the best overall approach before converging.
-- **When converging**: Generalize specific patches into broad principles. Remove redundancy and contradictions. Ensure the prompt is clear, focused, and under 200 lines.
-
-### Evaluation integrity
-
-**Critical**: Only optimize **task prompts** (what the agent receives), never **judge prompts** (how graders score outputs). Modifying judge prompts games the evaluation without improving the agent.
-
-If a prompt file is referenced in both task input and grader configs, optimize for the task purpose only. Document which prompts were modified in the optimization log.
-
-### The iteration loop
-
-After improving:
-
-1. Apply your changes to the agent's prompts/skills/config
-2. Re-run all test cases (agentv creates a new `.agentv/results/runs/<timestamp>/` directory automatically)
-3. Compare against the previous iteration (Step 4). If running in automated mode, use the **automated keep/discard** logic below instead of manual judgment — it will decide whether to keep or revert the change for you.
-4. Present results to the user (or log the decision if running automated keep/discard)
-5. Stop when ANY of:
-   - The user says they're happy
-   - Feedback is all empty (everything looks good)
-   - You're not making meaningful progress (no improvement for 2 consecutive iterations)
-   - Target pass rate is reached
-   - Maximum iterations exhausted
-
-**Human checkpoints**: At iterations 3, 6, and 9, always present progress to the user regardless of automation settings. Push back if optimization is accumulating contradictory rules or overfitting to specific test cases.
-
-### Automated keep/discard
-
-For autonomous iteration, use `agentv compare --json` to automatically decide whether to keep or discard each change based on wins/losses/ties. Read `references/autoresearch.md` for the full decision rules, logging format, and integration with the iteration loop.
-
----
-
-## Entering Mid-Lifecycle
-
-Users can start at any step by providing existing data:
-
-| Entry point | Required input | Example prompt |
-|------------|---------------|----------------|
-| Step 1 (Understand) | `eval-path` | "Optimize my agent against evals/support.yaml" |
-| Step 2 (Write Evals) | Agent artifacts | "Write evals for this agent" |
-| Step 3 (Run + Grade) | `eval-path` | "Run this eval and show me results" |
-| Step 4 (Analyze) | `results-path` | "Analyze why my agent is failing on these results" |
-| Step 5 (Improve) | Analysis + strategy | "Apply these optimization suggestions" |
-
-When entering mid-lifecycle, run only the requested step and subsequent steps. Don't re-run earlier steps unless the user requests a full loop.
-
----
-
-## Advanced: Blind Comparison
-
-For situations where you want a rigorous comparison between two versions (e.g., "is the new version actually better?"), dispatch the `comparator` subagent. It blinds identities, generates task-specific rubrics, scores outputs, then unblinds and explains why the winner won.
-
-This is optional and requires subagents. The human review loop is usually sufficient.
-
----
-
-## Description Optimization
-
-After the agent is working well, offer to optimize the skill's `description` field for better triggering accuracy. Read `references/description-optimization.md` for the full procedure (generate trigger EVAL.yaml, review with user, iterate, apply).
-
----
-
-## Autoresearch Mode
-
-Autoresearch is an unattended eval-improve loop that runs multiple optimize cycles without human intervention. The user triggers it with natural language (e.g., "run autoresearch on this skill", "optimize this skill unattended"). It uses the mutator subagent (`agents/mutator.md`) to rewrite artifacts based on failure analysis, and automated keep/discard to decide whether to keep or revert each change.
-
-Read `references/autoresearch.md` for the full procedure (prerequisites, artifact layout, keep/discard rules, the step-by-step loop, convergence criteria, and context hygiene).
-
----
-
-## Environment Adaptation
-
-For provider-specific notes (Copilot, Codex, Claude SDK, custom CLI), CI/headless mode behavior, and fallback strategies when subagents aren't available, read `references/environment-adaptation.md`.
-
----
-
-## Subagent Reference
-
-The `agents/` directory contains instructions for specialized subagents. Read them when you need to spawn the relevant subagent.
-
-| Agent | File | Purpose | When to dispatch |
-|-------|------|---------|-----------------|
-| executor | `agents/executor.md` | Perform test case tasks as the target agent | Step 3 (agent targets — one per test case) |
-| grader | `agents/grader.md` | Grade responses with per-assertion evidence | Step 3 (grading — one per test × LLM grader pair) |
-| comparator | `agents/comparator.md` | Blind N-way comparison + post-hoc analysis | Step 4 (comparing iterations/targets) |
-| analyzer | `agents/analyzer.md` | Quality audit, deterministic upgrades, benchmarks | Step 4 (pattern analysis) |
-| mutator | `agents/mutator.md` | Rewrite artifact from failure analysis | Step 5 (autoresearch — dispatched per cycle) |
-
-The `references/` directory has additional documentation:
-- `references/autoresearch.md` — Autoresearch unattended optimization loop and automated keep/discard rules
-- `references/eval-yaml-spec.md` — Eval YAML schema and assertion grading recipes
-- `references/subagent-pipeline.md` — Detailed subagent-mode pipeline commands and output structure
-- `references/description-optimization.md` — Skill description optimization workflow
-- `references/environment-adaptation.md` — Provider-specific notes and CI/headless behavior
-- `references/schemas.md` — JSON schemas for all artifacts (grading.json, benchmark.json, etc.)
-- `references/migrating-from-skill-creator.md` — Guide for users coming from Anthropic's skill-creator
-
----
-
-Repeating the core loop for emphasis:
-
-- Understand what the agent does
-- Write evaluation test cases
-- Run the agent and grade outputs
-- Analyze results — surface patterns, dispatch analyst and comparator subagents
-- Improve the agent based on analysis
-- Repeat until you and the user are satisfied
-
-Take your time with improvements. Read the transcripts. Understand why failures happened. Make changes that generalize beyond the test set. This is important work.
-
-## Accessing reference files
-
-To load a specific reference without pulling the entire skill into context:
-
-```bash
-agentv skills get agentv-bench --ref eval-yaml-spec
-```
-
-Or resolve the skill directory and read files directly:
-
-```bash
-cat $(agentv skills path agentv-bench)/references/eval-yaml-spec.md
-```
-
-Use `--full` to retrieve every file in the skill at once.
diff --git a/evals/agentv-dev/skills/fixtures/agentv-eval-review-full.txt b/evals/agentv-dev/skills/fixtures/agentv-eval-review-full.txt
deleted file mode 100644
index ba93b7b22..000000000
--- a/evals/agentv-dev/skills/fixtures/agentv-eval-review-full.txt
+++ /dev/null
@@ -1,294 +0,0 @@
----
-name: agentv-eval-review
-description: >-
-  Use when reviewing eval YAML files for quality issues, linting eval files before
-  committing, checking eval schema compliance, or when asked to "review these evals",
-  "check eval quality", "lint eval files", or "validate eval structure".
-  Do NOT use for writing evals (use agentv-eval-writer) or running evals (use agentv-bench).
----
-
-# Eval Review
-
-## Overview
-
-Lint and review AgentV eval YAML files for structural issues, schema compliance, and quality problems. Apply this checklist deterministically first, then layer LLM judgment for semantic issues a checklist cannot catch.
-
-## Process
-
-### Step 1: Structural checklist
-
-Walk every target eval file and report violations grouped by severity (error > warning > info). For each finding, include the file path and a concrete fix.
-
-- File extension is `.eval.yaml` (error if not).
-- `description` field is present at the top level (error if missing).
-- Each entry under `tests` has `id`, `input`, and at least one of `criteria` / `expected_output` / `assertions` (error if missing).
-- File-typed inputs (`type: file`) use a leading `/` in their `path` (error if relative).
-- Tests have an `assertions` block — flag tests that rely solely on `expected_output` (warning).
-- Detect `expected_output` prose patterns like "The agent should…" or "The output is…" (warning — prose belongs in `criteria`, structured matches in `assertions`).
-- Identical file inputs repeated across multiple tests in the same eval should be hoisted to a top-level `input` (info).
-- Eval files in the same directory should share a common `id` prefix (info — flag drift).
-
-### Step 2: Semantic review (LLM judgment)
-
-The structural checklist catches mechanical issues but cannot assess:
-- **Factual accuracy** — Do tool/command names in expected_output match what the skill documents?
-- **Coverage gaps** — Are important edge cases missing?
-- **Assertion discriminability** — Would assertions pass for both good and bad output?
-- **Cross-file consistency** — Do output filenames match across evals and skills?
-
-Read the relevant SKILL.md files and cross-check against the eval content for these issues.
-
-## Accessing reference files
-
-To load a specific reference without pulling the entire skill into context:
-
-```bash
-agentv skills get agentv-eval-review --ref <filename>
-```
-
-Or resolve the skill directory and read files directly:
-
-```bash
-cat $(agentv skills path agentv-eval-review)/references/<filename>.md
-```
-
---- scripts/lint_eval.py ---
-#!/usr/bin/env python3
-"""Lint AgentV eval YAML files for common issues.
-
-Usage: python lint_eval.py <path-to-eval-dir-or-file> [--json]
-
-Checks:
-  - File uses .eval.yaml extension
-  - description field present
-  - Each test has id, input, criteria
-  - File paths in type:file use leading /
-  - assertions blocks present (not relying solely on expected_output)
-  - expected_output does not contain evaluation criteria prose
-  - Repeated file inputs across tests (should use top-level input)
-  - Naming prefix consistency across eval files in same directory
-
-Exit code: 0 if no issues, 1 if issues found.
-"""
-
-import json
-import os
-import re
-import sys
-from pathlib import Path
-
-try:
-    import yaml
-except ImportError:
-    # Fall back to basic YAML parsing if PyYAML not available
-    yaml = None
-
-
-def parse_yaml_basic(text: str) -> dict:
-    """Minimal YAML-ish parser for when PyYAML is unavailable."""
-    # This is a best-effort fallback; recommend installing PyYAML
-    import ast
-    # Try json first (YAML is a superset of JSON)
-    try:
-        return json.loads(text)
-    except Exception:
-        pass
-    return {}
-
-
-def load_yaml(path: Path) -> dict:
-    text = path.read_text(encoding="utf-8")
-    if yaml:
-        return yaml.safe_load(text) or {}
-    return parse_yaml_basic(text)
-
-
-def lint_file(path: Path) -> list[dict]:
-    issues = []
-
-    def issue(severity: str, msg: str, line: int | None = None):
-        issues.append({"file": str(path), "severity": severity, "message": msg, "line": line})
-
-    # Check extension
-    if not path.name.endswith(".eval.yaml"):
-        issue("error", f"File should use .eval.yaml extension, got: {path.name}")
-
-    try:
-        data = load_yaml(path)
-    except Exception as e:
-        issue("error", f"Failed to parse YAML: {e}")
-        return issues
-
-    if not isinstance(data, dict):
-        issue("error", "Root element is not a mapping")
-        return issues
-
-    # Check description
-    if "description" not in data:
-        issue("warning", "Missing top-level 'description' field")
-
-    tests = data.get("tests", [])
-    if not isinstance(tests, list):
-        issue("error", "'tests' is not a list")
-        return issues
-
-    if not tests:
-        issue("warning", "No tests defined")
-        return issues
-
-    # Check for top-level input (shared file references)
-    top_level_input = data.get("input")
-
-    # Collect file values across tests to detect repetition
-    file_values_per_test: list[list[str]] = []
-
-    for i, test in enumerate(tests):
-        test_id = test.get("id", f"test-{i}")
-
-        if "id" not in test:
-            issue("error", f"Test at index {i} missing 'id'")
-
-        if "input" not in test and top_level_input is None:
-            issue("error", f"Test '{test_id}' missing 'input' and no top-level input defined")
-
-        has_criteria = "criteria" in test
-        has_expected = "expected_output" in test
-        has_assertions = "assertions" in test
-
-        if not has_criteria and not has_expected and not has_assertions:
-            issue("error", f"Test '{test_id}' needs at least one of: criteria, expected_output, assertions")
-
-        # Check assertions present
-        if not has_assertions and has_expected:
-            issue("warning", f"Test '{test_id}' has expected_output but no assertions — add deterministic assertions where possible")
-
-        # Check expected_output for prose patterns
-        if has_expected:
-            expected = test["expected_output"]
-            expected_text = ""
-            if isinstance(expected, str):
-                expected_text = expected
-            elif isinstance(expected, list):
-                for msg in expected:
-                    if isinstance(msg, dict):
-                        content = msg.get("content", "")
-                        if isinstance(content, str):
-                            expected_text += content
-
-            prose_patterns = [
-                r"[Tt]he agent should",
-                r"[Ss]hould identify",
-                r"[Ss]hould flag",
-                r"[Ss]hould recommend",
-                r"[Ss]hould produce",
-                r"[Ss]hould detect",
-                r"[Ss]hould load",
-                r"[Ss]hould run",
-            ]
-            for pat in prose_patterns:
-                if re.search(pat, expected_text):
-                    issue("warning", f"Test '{test_id}' expected_output contains evaluation criteria prose ('{pat.lstrip('[Tt]').lstrip('[Ss]')}...') — use criteria or assertions instead")
-                    break
-
-        # Collect file values from input
-        test_files = extract_file_values(test.get("input", []))
-        file_values_per_test.append(test_files)
-
-        # Check file paths for leading /
-        for fv in test_files:
-            if not fv.startswith("/"):
-                issue("warning", f"Test '{test_id}' file path missing leading '/': {fv}")
-
-    # Check for repeated file inputs
-    if len(file_values_per_test) >= 2 and not top_level_input:
-        common_files = set(file_values_per_test[0])
-        for fvs in file_values_per_test[1:]:
-            common_files &= set(fvs)
-        if common_files:
-            issue("info", f"File input repeated in every test: {', '.join(sorted(common_files))} — consider using top-level input")
-
-    return issues
-
-
-def extract_file_values(input_data) -> list[str]:
-    """Extract type:file values from input structure."""
-    files = []
-    if isinstance(input_data, list):
-        for item in input_data:
-            if isinstance(item, dict):
-                content = item.get("content", [])
-                if isinstance(content, list):
-                    for c in content:
-                        if isinstance(c, dict) and c.get("type") == "file":
-                            v = c.get("value", "")
-                            if v:
-                                files.append(v)
-    return files
-
-
-def lint_directory(path: Path) -> list[dict]:
-    issues = []
-    eval_files = sorted(path.rglob("*.yaml")) + sorted(path.rglob("*.yml"))
-
-    if not eval_files:
-        issues.append({"file": str(path), "severity": "warning", "message": "No eval files found", "line": None})
-        return issues
-
-    # Check naming prefix consistency
-    prefixes = set()
-    for f in eval_files:
-        name = f.stem.replace(".eval", "")
-        parts = name.split("-")
-        if len(parts) >= 2:
-            prefixes.add(parts[0])
-
-    if len(prefixes) > 1:
-        issues.append({
-            "file": str(path),
-            "severity": "info",
-            "message": f"Inconsistent naming prefixes: {', '.join(sorted(prefixes))}",
-            "line": None,
-        })
-
-    for f in eval_files:
-        issues.extend(lint_file(f))
-
-    return issues
-
-
-def main():
-    if len(sys.argv) < 2:
-        print(f"Usage: {sys.argv[0]} <path> [--json]", file=sys.stderr)
-        sys.exit(2)
-
-    target = Path(sys.argv[1])
-    output_json = "--json" in sys.argv
-
-    if target.is_file():
-        issues = lint_file(target)
-    elif target.is_dir():
-        issues = lint_directory(target)
-    else:
-        print(f"Error: {target} not found", file=sys.stderr)
-        sys.exit(2)
-
-    if output_json:
-        print(json.dumps(issues, indent=2))
-    else:
-        for iss in issues:
-            line = f":{iss['line']}" if iss.get("line") else ""
-            print(f"[{iss['severity'].upper()}] {iss['file']}{line}: {iss['message']}")
-
-        counts = {}
-        for iss in issues:
-            counts[iss["severity"]] = counts.get(iss["severity"], 0) + 1
-        if issues:
-            print(f"\n{len(issues)} issues: {', '.join(f'{v} {k}' for k, v in sorted(counts.items()))}")
-        else:
-            print("No issues found.")
-
-    sys.exit(1 if any(i["severity"] == "error" for i in issues) else 0)
-
-
-if __name__ == "__main__":
-    main()
diff --git a/evals/agentv-dev/skills/fixtures/agentv-eval-review.txt b/evals/agentv-dev/skills/fixtures/agentv-eval-review.txt
deleted file mode 100644
index 6b1752b10..000000000
--- a/evals/agentv-dev/skills/fixtures/agentv-eval-review.txt
+++ /dev/null
@@ -1,53 +0,0 @@
----
-name: agentv-eval-review
-description: >-
-  Use when reviewing eval YAML files for quality issues, linting eval files before
-  committing, checking eval schema compliance, or when asked to "review these evals",
-  "check eval quality", "lint eval files", or "validate eval structure".
-  Do NOT use for writing evals (use agentv-eval-writer) or running evals (use agentv-bench).
----
-
-# Eval Review
-
-## Overview
-
-Lint and review AgentV eval YAML files for structural issues, schema compliance, and quality problems. Apply this checklist deterministically first, then layer LLM judgment for semantic issues a checklist cannot catch.
-
-## Process
-
-### Step 1: Structural checklist
-
-Walk every target eval file and report violations grouped by severity (error > warning > info). For each finding, include the file path and a concrete fix.
-
-- File extension is `.eval.yaml` (error if not).
-- `description` field is present at the top level (error if missing).
-- Each entry under `tests` has `id`, `input`, and at least one of `criteria` / `expected_output` / `assertions` (error if missing).
-- File-typed inputs (`type: file`) use a leading `/` in their `path` (error if relative).
-- Tests have an `assertions` block — flag tests that rely solely on `expected_output` (warning).
-- Detect `expected_output` prose patterns like "The agent should…" or "The output is…" (warning — prose belongs in `criteria`, structured matches in `assertions`).
-- Identical file inputs repeated across multiple tests in the same eval should be hoisted to a top-level `input` (info).
-- Eval files in the same directory should share a common `id` prefix (info — flag drift).
-
-### Step 2: Semantic review (LLM judgment)
-
-The structural checklist catches mechanical issues but cannot assess:
-- **Factual accuracy** — Do tool/command names in expected_output match what the skill documents?
-- **Coverage gaps** — Are important edge cases missing?
-- **Assertion discriminability** — Would assertions pass for both good and bad output?
-- **Cross-file consistency** — Do output filenames match across evals and skills?
-
-Read the relevant SKILL.md files and cross-check against the eval content for these issues.
-
-## Accessing reference files
-
-To load a specific reference without pulling the entire skill into context:
-
-```bash
-agentv skills get agentv-eval-review --ref <filename>
-```
-
-Or resolve the skill directory and read files directly:
-
-```bash
-cat $(agentv skills path agentv-eval-review)/references/<filename>.md
-```
diff --git a/evals/agentv-dev/skills/fixtures/agentv-eval-writer-full.txt b/evals/agentv-dev/skills/fixtures/agentv-eval-writer-full.txt
deleted file mode 100644
index 079c7fbe2..000000000
--- a/evals/agentv-dev/skills/fixtures/agentv-eval-writer-full.txt
+++ /dev/null
@@ -1,18273 +0,0 @@
----
-name: agentv-eval-writer
-description: >-
-  Write, edit, review, and validate AgentV EVAL.yaml / .eval.yaml evaluation files.
-  Use when asked to create new eval files, update or fix existing ones, add or remove test cases,
-  configure graders (`llm-grader`, `code-grader`, `rubrics`), review whether an eval is correct or complete,
-  convert between EVAL.yaml and evals.json using `agentv convert`, or generate eval test cases
-  from chat transcripts (markdown conversation or JSON messages).
-  Do NOT use for creating SKILL.md files, writing skill definitions, or running evals —
-  running and benchmarking belongs to agentv-bench.
----
-
-# AgentV Eval Writer
-
-Comprehensive docs: https://agentv.dev
-
-## Authoring Principle
-
-Treat YAML as the canonical portable model. Prefer authoring `.eval.yaml` / `EVAL.yaml` first, then use TypeScript helpers, Python scripts, or executable graders only when they lower to the same fields or when the evaluation logic must actually run code.
-
-## Evaluation Types
-
-AgentV evaluations measure **execution quality** — whether your agent or skill produces correct output when invoked.
-
-For **trigger quality** (whether the right skill is triggered for the right prompts), see the [Evaluation Types guide](https://agentv.dev/guides/evaluation-types/). Do not use execution eval configs (`EVAL.yaml`, `evals.json`) for trigger evaluation — these are distinct concerns requiring different tooling and methodologies.
-
-## Starting from evals.json?
-
-If the project already has an Agent Skills `evals.json` file, use it as a starting point instead of writing YAML from scratch:
-
-```bash
-# Convert evals.json to AgentV EVAL YAML
-agentv convert evals.json
-
-# Run directly without converting (all commands accept evals.json)
-agentv eval evals.json
-```
-
-The converter maps `prompt` → `input`, `expected_output` → `expected_output`, `assertions` → `assertions` (`llm-grader`), and resolves `files[]` paths. The generated YAML includes TODO comments for AgentV features to add (workspace setup, code graders, rubrics, required gates).
-
-After converting, enhance the YAML with AgentV-specific capabilities shown below.
-
-## From Chat Transcript
-
-Convert a chat conversation into eval test cases without starting from scratch.
-
-**Input formats:**
-
-Markdown conversation:
-```
-User: How do I reset my password?
-Assistant: Go to Settings > Security > Reset Password...
-```
-
-JSON messages:
-```json
-[{"role": "user", "content": "How do I reset my password?"},
- {"role": "assistant", "content": "Go to Settings > Security > Reset Password..."}]
-```
-
-**Select exchanges that make good test cases:**
-- Factual Q&A — verifiable answers
-- Task completion — user requests an action, agent performs it
-- Edge cases — unusual inputs, error handling, boundary conditions
-- Multi-turn reasoning — exchanges where earlier context matters
-
-**Skip:** greetings, one-word acknowledgments, repeated exchanges
-
-**Multi-turn format** (when context from prior turns matters):
-```yaml
-tests:
-  - id: multi-turn-context
-    criteria: "Agent remembers prior context"
-    input:
-      - role: user
-        content: "My name is Alice"
-      - role: assistant
-        content: "Nice to meet you, Alice!"
-      - role: user
-        content: "What's my name?"
-    expected_output: "Your name is Alice."
-    assertions:
-      - type: rubrics
-        criteria:
-          - Correctly recalls the user's name from earlier in the conversation
-```
-
-**Guidelines:** preserve exact wording in `expected_output`; aim for 5–15 tests per transcript; pick exchanges that test different capabilities.
-
-## Quick Start
-
-```yaml
-description: Example eval
-execution:
-  target: default
-
-tests:
-  - id: greeting
-    criteria: Friendly greeting
-    input: "Say hello"
-    expected_output: "Hello! How can I help you?"
-    assertions:
-      - type: rubrics
-        criteria:
-          - Greeting is friendly and warm
-          - Offers to help
-```
-
-## Eval File Structure
-
-**Required:** `tests` (array or string path)
-**Optional:** `name`, `description`, `version`, `author`, `tags`, `license`, `requires`, `execution`, `suite`, `workspace`, `assertions`, `input`
-
-**Test fields:**
-
-| Field | Required | Description |
-|-------|----------|-------------|
-| `id` | yes | Unique identifier |
-| `criteria` | yes | What the response should accomplish |
-| `input` | yes | Input to the agent (string shorthand or full message array) |
-| `expected_output` | no | Gold-standard reference answer (string shorthand or full message array) |
-| `assertions` | no | Graders: deterministic checks, rubrics, and LLM/code graders |
-| `execution` | no | Per-case execution overrides |
-| `workspace` | no | Per-case workspace config (overrides suite-level) |
-| `metadata` | no | Arbitrary key-value pairs passed to setup/teardown scripts |
-| `conversation_id` | no | Thread grouping |
-
-**Shorthand forms:**
-- `input` (string) expands to `[{role: "user", content: "..."}]`
-- `expected_output` (string/object) expands to `[{role: "assistant", content: ...}]`
-- Use these canonical field names on disk; keep the wire format `snake_case`
-
-**Message format:** `{role, content}` where role is `system`, `user`, `assistant`, or `tool`
-**Content types:** inline text, `{type: "file", value: "./path.md"}`
-**File paths:** relative from eval file dir, or absolute with `/` prefix from repo root
-**File handling by provider type:** LLM providers receive file content inlined in XML tags. Agent providers receive a preread block with `file://` URIs and must read files themselves. See [Coding Agents > Prompt format](https://agentv.dev/targets/coding-agents#prompt-format).
-
-**JSONL format:** One test per line as JSON. Optional `.yaml` sidecar for shared defaults. See `examples/features/basic-jsonl/`.
-
-**Environment variables:** All string fields support `${{ VAR }}` interpolation. Missing vars resolve to empty string. Works in eval files, external case files, and workspace configs. `.env` files are loaded automatically.
-
-## Metadata
-
-When `name` is present, the suite is parsed as a metadata-bearing eval:
-
-```yaml
-name: export-screening        # required, lowercase/hyphens, max 64 chars
-description: Evaluates export control screening accuracy
-version: "1.0"
-author: acme-compliance
-tags: [compliance, agents]
-license: Apache-2.0
-requires:
-  agentv: ">=0.30.0"
-```
-
-## Suite-level Input
-
-Prepend shared input messages to every test (like suite-level `assertions`). Avoids repeating the same prompt file in each test:
-
-```yaml
-input:
-  - role: user
-    content:
-      - type: file
-        value: ./system-prompt.md
-
-tests: ./cases.yaml
-
-# cases.yaml — each test only needs its own query
-# - id: test-1
-#   criteria: ...
-#   input: "User question here"
-```
-
-Effective input: `[...suite input, ...test input]`. Skipped when `execution.skip_defaults: true`.
-Accepts same formats as test `input` (string or message array).
-
-## Tests as String Path
-
-Point `tests` to an external file instead of inlining:
-
-```yaml
-name: my-eval
-description: My evaluation suite
-tests: ./cases.yaml           # relative to eval file dir
-```
-
-The external file can be YAML (array of test objects) or JSONL.
-
-## Assertions Field
-
-`assertions` defines graders at the suite level or per-test level. It is the canonical field for all graders:
-
-```yaml
-# Suite-level (appended to every test)
-assertions:
-  - type: is-json
-    required: true
-  - type: contains
-    value: "status"
-
-tests:
-  - id: test-1
-    criteria: Returns JSON
-    input: Get status
-    # Per-test assertions (runs before suite-level)
-    assertions:
-      - type: equals
-        value: '{"status": "ok"}'
-```
-
-## How `criteria` and `assertions` Interact
-
-`criteria` is a **data field** — it describes what the response should accomplish. It is **not** a grader. How it gets evaluated depends on whether `assertions` is present:
-
-| Scenario | What happens | Warning? |
-|----------|-------------|----------|
-| `criteria` + **no `assertions`** | Implicit `llm-grader` runs automatically against `criteria` | No |
-| `criteria` + **`assertions` with only deterministic graders** (contains, regex, etc.) | Only declared graders run. `criteria` is **not evaluated**. | Yes — warns that no grader will consume criteria |
-| `criteria` + **`assertions` with a grader** (`llm-grader`, `code-grader`, `rubrics`) | Declared graders run. Graders receive `criteria` as input. | No |
-
-### No assertions → implicit llm-grader
-
-The simplest path. `criteria` is automatically evaluated by the default `llm-grader`:
-
-```yaml
-tests:
-  - id: simple-eval
-    criteria: Assistant correctly explains the bug and proposes a fix
-    input: "Debug this function..."
-    # No assertions → default llm-grader evaluates against criteria
-```
-
-### assertions present → no implicit grader
-
-When `assertions` is defined, **only the declared graders run**. If you want an LLM grader alongside deterministic checks, declare it explicitly:
-
-```yaml
-tests:
-  - id: mixed-eval
-    criteria: Response is helpful and mentions the fix
-    input: "Debug this function..."
-    assertions:
-      - type: llm-grader       # must be explicit when assertions is present
-      - type: contains
-        value: "fix"
-```
-
-**Common mistake:** defining `criteria` with only deterministic graders. The criteria will be ignored and a warning is emitted:
-
-```yaml
-tests:
-  - id: bad-example
-    criteria: Gives a thoughtful answer    # ⚠ NOT evaluated — no grader in assertions
-    input: "What is 2+2?"
-    assertions:
-      - type: contains
-        value: "4"
-    # Warning: criteria is defined but no grader in assertions will evaluate it.
-```
-
-## Required Gates
-
-Any grader can be marked `required` to enforce a minimum score:
-
-```yaml
-assertions:
-  - type: contains
-    value: "DENIED"
-    required: true          # must score >= 0.8 (default)
-  - type: rubrics
-    required: 0.6           # must score >= 0.6 (custom threshold)
-    criteria:
-      - id: accuracy
-        outcome: Identifies the denied party
-        weight: 5.0
-```
-
-If a required grader scores below its threshold, the overall verdict is forced to `fail`.
-
-## Workspace Setup/Teardown
-
-Run scripts before/after each test. Define at suite level or override per case:
-
-```yaml
-workspace:
-  template: ./workspace-templates/my-project
-  repos:
-    - path: ./repo
-      repo: sympy/sympy
-      base_commit: "abc123"
-  hooks:
-    before_all:
-      command: ["bun", "run", "setup.ts"]
-      timeout_ms: 120000
-    after_each:
-      reset: fast
-    after_all:
-      command: ["bun", "run", "teardown.ts"]
-
-tests:
-  - id: case-1
-    input: Fix the bug
-    criteria: Bug is fixed
-    metadata:
-      source_repo: sympy/sympy
-      source_commit: "abc123"
-```
-
-**Lifecycle:** template copy → repo materialization → workspace before_all → target before_all → git baseline → before_each hooks → agent → file changes → after_each hooks → after_all hooks → cleanup
-**Merge:** Case-level fields replace suite-level fields.
-**Commands receive stdin JSON:** `{workspace_path, test_id, eval_run_id, case_input, case_metadata}`
-**Setup failure:** aborts case. **Teardown failure:** non-fatal (warning).
-For SWE-bench-style evals, keep operational checkout state under `workspace.repos[].base_commit`; treat `metadata.source_commit` as informational only.
-
-### Repository Lifecycle
-
-Materialize repos into the eval workspace automatically. Repo entries declare identity and checkout pins only; AgentV resolves acquisition from registered projects, `git_cache.mirrors`, its mirror cache, then remote clone. For shared repo workspaces, pooling is the default:
-
-```yaml
-workspace:
-  repos:
-    - path: ./repo
-      repo: https://github.com/org/repo.git
-      commit: main
-      ancestor: 1       # parent commit
-  hooks:
-    after_each:
-      reset: fast          # none | fast | strict
-  isolation: shared        # shared | per_test
-  mode: pooled             # pooled | temp | static
-```
-
-- `repo`: full clone URL or GitHub `org/name` shorthand
-- `commit`: branch, tag, or SHA to check out
-- `base_commit`: alias for `commit` for SWE-bench-style datasets
-- `ancestor`: walk N commits back from the checked-out ref
-- `sparse`: sparse checkout paths array
-- Do not use legacy `source`, `type`, `checkout`, `resolve`, or `clone` fields under `workspace.repos[]`
-- `mode`: `pooled` (default for shared repos), `temp`, or `static`
-- `path`: workspace path used when `mode: static`; when empty/missing the workspace is auto-materialised (template copied + repos cloned); populated dirs are reused as-is
-- `hooks.enabled`: boolean (default `true`); set `false` to skip all lifecycle hooks
-- Pool reset defaults to `fast` (`git clean -fd`); use `--workspace-clean full` for strict reset (`git clean -fdx`)
-- Pool entries are managed separately via `agentv workspace list` and `agentv workspace clean`
-- `agentv workspace deps <eval-paths>` scans eval files and outputs a JSON manifest of required git repos (useful for CI pre-cloning)
-
-See https://agentv.dev/targets/configuration/#repository-lifecycle
-
-## Grader Types
-
-Configure via `assertions` array. Multiple graders produce a weighted average score.
-
-### code-grader
-```yaml
-- name: format_check
-  type: code-grader
-  command: [uv, run, validate.py]
-  cwd: ./scripts          # optional working directory
-  target: {}              # optional: enable LLM target proxy (max_calls: 50)
-```
-Contract: stdin JSON -> stdout JSON `{score, assertions: [{text, passed, evidence?}], reasoning}`
-Raw stdin uses snake_case and includes: `criteria`, `input`, `expected_output`, `output` (final answer string), `messages`, `trace`, `trace_summary`, `token_usage`, `cost_usd`, `duration_ms`, `start_time`, `end_time`, `file_changes`, `workspace_path`, `config`
-SDK handlers receive the same payload in camelCase: `expectedOutput`, `traceSummary`, `tokenUsage`, `costUsd`, `durationMs`, `startTime`, `endTime`, `fileChanges`, `workspacePath`.
-When a workspace is configured, `workspace_path` is the absolute path to the workspace dir (also available as `AGENTV_WORKSPACE_PATH` env var). Use this for functional grading (e.g., running `npm test` in the workspace).
-See docs at https://agentv.dev/graders/code-graders/
-
-### llm-grader
-```yaml
-- name: quality
-  type: llm-grader
-  prompt: ./prompts/eval.md     # markdown template or command config
-  target: grader_gpt_5_mini     # optional: override the grader target for this grader
-  model: gpt-5-chat            # optional model override
-  config:                       # passed to prompt templates as context.config
-    strictness: high
-```
-Variables: `{{criteria}}`, `{{input}}`, `{{expected_output}}`, `{{output}}`, `{{metadata}}`, `{{metadata_json}}`, `{{rubrics}}`, `{{rubrics_json}}`, `{{file_changes}}`, `{{tool_calls}}`
-- Markdown templates: use `{{variable}}` syntax
-- TypeScript templates: use `definePromptTemplate(fn)` from `@agentv/sdk`, receives context object with all variables + `config`
-- Use `target:` to run different `llm-grader` graders against different named LLM targets in the same eval (useful for grader panels / ensembles)
-
-### composite
-```yaml
-- name: gate
-  type: composite
-  assertions:
-    - name: safety
-      type: llm-grader
-      prompt: ./safety.md
-    - name: quality
-      type: llm-grader
-  aggregator:
-    type: weighted_average
-    weights: { safety: 0.3, quality: 0.7 }
-```
-Aggregator types: `weighted_average`, `all_or_nothing`, `minimum`, `maximum`, `safety_gate`
-- `safety_gate`: fails immediately if the named gate grader scores below threshold (default 1.0)
-
-### tool_trajectory
-```yaml
-- name: tool_check
-  type: tool-trajectory
-  mode: any_order            # any_order | in_order | exact
-  minimums:                  # for any_order
-    knowledgeSearch: 2
-  expected:                  # for in_order/exact
-    - tool: knowledgeSearch
-      args: { query: "search term" }   # partial deep equality match
-    - tool: documentRetrieve
-      args: any                        # any arguments accepted
-      max_duration_ms: 5000            # per-tool latency assertion
-    - tool: summarize                  # omit args to skip argument checking
-```
-
-### field_accuracy
-```yaml
-- name: fields
-  type: field-accuracy
-  match_type: exact          # exact | date | numeric_tolerance
-  numeric_tolerance: 0.01    # for numeric_tolerance match_type
-  aggregation: weighted_average  # weighted_average | all_or_nothing
-```
-Compares `output` fields against `expected_output` fields.
-
-### latency
-```yaml
-- name: speed
-  type: latency
-  max_ms: 5000
-```
-
-### cost
-```yaml
-- name: budget
-  type: cost
-  max_usd: 0.10
-```
-
-### token_usage
-```yaml
-- name: tokens
-  type: token-usage
-  max_total_tokens: 4000
-```
-
-### execution_metrics
-```yaml
-- name: efficiency
-  type: execution-metrics
-  max_tool_calls: 10        # Maximum tool invocations
-  max_llm_calls: 5          # Maximum LLM calls (assistant messages)
-  max_tokens: 5000          # Maximum total tokens (input + output)
-  max_cost_usd: 0.05        # Maximum cost in USD
-  max_duration_ms: 30000    # Maximum execution duration
-  target_exploration_ratio: 0.6   # Target ratio of read-only tool calls
-  exploration_tolerance: 0.2      # Tolerance for ratio check (default: 0.2)
-```
-Declarative threshold-based checks on execution metrics. Only specified thresholds are checked.
-Score is proportional: `passed / total` assertions. Missing data counts as a failed assertion.
-
-### contains
-```yaml
-- type: contains
-  value: "DENIED"
-  required: true
-```
-Binary check: does output contain the substring? Name auto-generated if omitted.
-
-### regex
-```yaml
-- type: regex
-  value: "\\d{3}-\\d{2}-\\d{4}"
-```
-Binary check: does output match the regex pattern?
-
-### equals
-```yaml
-- type: equals
-  value: "42"
-```
-Binary check: does output exactly equal the value (both trimmed)?
-
-### is_json
-```yaml
-- type: is-json
-  required: true
-```
-Binary check: is the output valid JSON?
-
-### rubrics
-```yaml
-- type: rubrics
-  criteria:
-    - id: accuracy
-      outcome: Correctly identifies the denied party
-      weight: 5.0
-    - id: reasoning
-      outcome: Provides clear reasoning
-      weight: 3.0
-```
-LLM-judged structured evaluation with weighted criteria. Criteria items support `id`, `outcome`, `weight`, and `required` fields.
-Use optional `operator: correctness` for positive support checks or `operator: contradiction` for guard criteria where omission is acceptable but incompatible claims fail.
-
-See `references/rubric-grader.md` for score-range mode and scoring formula.
-
-## Execution Error Tolerance
-
-Control how the runner handles execution errors (infrastructure failures, not quality failures):
-
-```yaml
-execution:
-  fail_on_error: false    # never halt (default)
-  # fail_on_error: true   # halt on first execution error
-```
-
-When halted, remaining tests get `executionStatus: 'execution_error'` with `failureReasonCode: 'error_threshold_exceeded'`.
-
-## Suite-Level Quality Threshold
-
-Set a minimum mean score for the eval suite. If the mean quality score falls below the threshold, the CLI exits with code 1 — useful for CI/CD quality gates.
-
-```yaml
-execution:
-  threshold: 0.8
-```
-
-CLI flag `--threshold 0.8` overrides the YAML value. Must be a number between 0 and 1. Mean score is computed from quality results only (execution errors excluded).
-
-The threshold also controls JUnit XML pass/fail: tests with scores below the threshold are marked as `<failure>`. When no threshold is set, JUnit defaults to 0.5.
-
-## CLI Commands
-
-```bash
-# Run evaluation (requires API keys)
-agentv eval <file.yaml> [--test-id <id>] [--target <name>] [--dry-run] [--threshold <0-1>]
-
-# Run with OTLP JSON file (importable by OTel backends)
-agentv eval <file.yaml> --otel-file traces/eval.otlp.json
-
-# Record live target output for later target substitution
-agentv eval <file.yaml> --target live_agent --record-replay fixtures/target-output.jsonl
-agentv eval <file.yaml> --target replay_agent
-
-# Run a single assertion in isolation (no API keys needed)
-agentv eval assert <grader-name> --agent-output "..." --agent-input "..."
-
-# Import agent transcripts for offline grading
-agentv import claude --session-id <uuid>
-
-# Re-run only execution errors from a previous run
-agentv eval <file.yaml> --retry-errors .agentv/results/runs/<timestamp>/index.jsonl
-
-# Validate eval file
-agentv validate <file.yaml>
-
-# Compare results — N-way matrix from a canonical run manifest
-agentv compare .agentv/results/runs/<timestamp>/index.jsonl
-agentv compare .agentv/results/runs/<timestamp>/index.jsonl --baseline <target>                   # CI regression gate
-agentv compare .agentv/results/runs/<timestamp>/index.jsonl --baseline <target> --candidate <target>  # pairwise
-agentv compare .agentv/results/runs/<baseline-timestamp>/index.jsonl .agentv/results/runs/<candidate-timestamp>/index.jsonl
-
-# Author assertions directly in the eval file
-# Prefer simple assertions when they fit the criteria; use deterministic or LLM-based graders when needed
-agentv validate <file.yaml>
-```
-
-**Replay targets:** Add `provider: replay`, `fixtures: <jsonl>`, and `source_target: <live target name>` in `.agentv/targets.yaml`. Optional `suite`, `eval_path`, and `variant` tighten lookup. The eval YAML and graders stay unchanged; replay only substitutes recorded target output, and graders run fresh.
-
-## Code Judge SDK
-
-Use `@agentv/sdk` to build custom graders in TypeScript/JavaScript:
-
-### defineAssertion (recommended for custom checks)
-```typescript
-#!/usr/bin/env bun
-import { defineAssertion } from '@agentv/sdk';
-
-export default defineAssertion(({ output, trace }) => {
-  const finalOutput = output ?? '';
-  return {
-    pass: finalOutput.length > 0 && (trace?.eventCount ?? 0) <= 10,
-    reasoning: 'Checks content exists and is efficient',
-  };
-});
-```
-
-Assertions support both `pass: boolean` and `score: number` (0-1). If only `pass` is given, score is 1 (pass) or 0 (fail).
-
-### defineCodeGrader (full control)
-```typescript
-#!/usr/bin/env bun
-import { defineCodeGrader } from '@agentv/sdk';
-
-export default defineCodeGrader(({ output, trace }) => {
-  const finalOutput = output ?? '';
-  return {
-    score: finalOutput.length > 0 && (trace?.eventCount ?? 0) <= 5 ? 1.0 : 0.5,
-    assertions: [
-      { text: 'Output is not empty', passed: finalOutput.length > 0 },
-      { text: 'Efficient tool usage', passed: (trace?.eventCount ?? 0) <= 5 },
-    ],
-  };
-});
-```
-
-Both are used via `type: code-grader` in YAML with `command: [bun, run, grader.ts]`.
-
-### Convention-Based Discovery
-
-Place assertion files in `.agentv/assertions/` — they auto-register by filename:
-
-```
-.agentv/assertions/word-count.ts  →  type: word-count
-.agentv/assertions/sentiment.ts   →  type: sentiment
-```
-
-No `command:` needed in YAML — just use `type: <filename>`.
-
-## Programmatic API
-
-Use `evaluate()` from `@agentv/core` to run evals as a library when you need application-level control. Keep YAML as the default portable surface; use `specFile` to point at existing evals and inline `tests` only when the definition belongs in code.
-
-```typescript
-import { evaluate } from '@agentv/core';
-
-const { results, summary } = await evaluate({
-  tests: [
-    {
-      id: 'greeting',
-      input: 'Say hello',
-      expectedOutput: 'Hello there!',
-      assert: [{ type: 'contains', value: 'hello' }],
-    },
-  ],
-  target: { provider: 'mock_agent' },
-});
-console.log(`${summary.passed}/${summary.total} passed`);
-```
-
-Programmatic API notes:
-
-- Inline programmatic tests use `assert`, not `assertions`.
-- Use camelCase in TypeScript (`expectedOutput`, `beforeAll`, `budgetUsd`).
-- When bridging from Python, generate canonical YAML/JSONL or call the CLI; there is no separate first-party Python authoring SDK.
-
-Supports inline tests or file-based usage via `specFile`.
-
-## defineConfig
-
-Type-safe project configuration in `agentv.config.ts`:
-
-```typescript
-import { defineConfig } from '@agentv/core';
-
-export default defineConfig({
-  execution: { workers: 5, maxRetries: 2 },
-  output: { dir: './results' },
-  limits: { maxCostUsd: 10.0 },
-});
-```
-
-Auto-discovered from project root. Validated with Zod.
-
-## Scaffold Commands
-
-```bash
-agentv create assertion <name>  # → .agentv/assertions/<name>.ts
-agentv create eval <name>       # → evals/<name>.eval.yaml + .cases.jsonl
-```
-
-## Skill Improvement Workflow
-
-For a complete guide to iterating on skills using evaluations — writing scenarios, running baselines, comparing results, and improving — see the [Skill Improvement Workflow](https://agentv.dev/guides/skill-improvement-workflow/) guide.
-## Human Review Checkpoint
-
-After running evals, perform a human review before iterating. Create `feedback.json` in the results directory:
-
-```json
-{
-  "run_id": "2026-03-14T10-32-00_claude",
-  "reviewer": "engineer-name",
-  "timestamp": "2026-03-14T12:00:00Z",
-  "overall_notes": "Summary of observations",
-  "per_case": [
-    {
-      "test_id": "test-id",
-      "verdict": "acceptable | needs_improvement | incorrect | flaky",
-      "notes": "Why this verdict",
-      "evaluator_overrides": { "code-grader:name": "Override note" },
-      "workspace_notes": "Workspace state observations"
-    }
-  ]
-}
-```
-
-Use `evaluator_overrides` for workspace evaluations to annotate specific grader results (e.g., "code-grader was too strict"). Use `workspace_notes` for observations about workspace state.
-
-Review workflow: run evals → inspect results (`agentv inspect show`) → write feedback → tune prompts/graders → re-run.
-
-Full guide: https://agentv.dev/guides/human-review/
-
-## Observability Export
-
-AgentV exports observability data via OpenTelemetry:
-
-- `agentv eval <file.yaml> --otel-file traces/eval.otlp.json` writes a post-run OTLP JSON file that external systems such as Opik can ingest.
-- `agentv eval <file.yaml> --export-otel --otel-backend <name>` streams live traces when a built-in or local resolver exists.
-
-Do not invent a separate Opik-specific eval surface. Keep the eval definition in YAML and route observability through OTLP export.
-
-## Schemas
-
-- Eval file: `references/eval-schema.json`
-- Config: `references/config-schema.json`
-
-## Accessing reference files
-
-To load a specific reference without pulling the entire skill into context:
-
-```bash
-agentv skills get agentv-eval-writer --ref eval-schema.json
-```
-
-Or resolve the skill directory and read files directly:
-
-```bash
-cat $(agentv skills path agentv-eval-writer)/references/eval-schema.json
-```
-
-Use `--full` to retrieve every file in the skill at once.
-
---- references/python-helpers.md ---
-# Repo-Local Python Helpers
-
-AgentV's Python authoring surface is currently a repo-local helper example under `examples/features/sdk-python/`.
-
-Use it when the user wants Python-based custom graders or wants to emit AgentV YAML/JSONL from Python without introducing a Python-native runner.
-
-## Rules
-
-- Prefer canonical AgentV wire and YAML fields.
-- Do not accept deprecated wire aliases like `output_text`, `input_text`, or `reference_answer`.
-- Keep Python eval authoring YAML-shaped. Mirror `execution`, `tests`, `assertions`, `expected_output`, and related AgentV keys directly.
-- Run evals through the AgentV CLI, not through a separate Python runtime.
-
-## Available helpers
-
-- `agentv_py.grader`
-  - `load_grader_input()`
-  - `run_code_grader(handler)`
-  - `define_code_grader(handler)`
-  - `TargetClient.from_env()`
-- `agentv_py.evals`
-  - `EvalDefinition`
-  - `EvalTest`
-  - `JsonlCase`
-  - `write_eval_yaml()`
-  - `write_jsonl()`
-  - `run_agentv_eval()`
-
-## Example
-
-```python
-from agentv_py.grader import Assertion, CodeGraderResult, define_code_grader
-
-
-def evaluate(context):
-    actual = context.output or ""
-    expected = context.expected_output[0]["content"]
-    passed = actual.strip() == expected.strip()
-    return CodeGraderResult(
-        score=1.0 if passed else 0.0,
-        assertions=[Assertion(text="Exact match", passed=passed)],
-    )
-
-
-if __name__ == "__main__":
-    define_code_grader(evaluate)
-```
-
---- references/eval-schema.json ---
-{
-  "$schema": "https://json-schema.org/draft/2019-09/schema#",
-  "title": "AgentV Eval File",
-  "description": "Schema for AgentV evaluation YAML files (.eval.yaml)",
-  "$ref": "#/definitions/EvalFile",
-  "definitions": {
-    "EvalFile": {
-      "type": "object",
-      "properties": {
-        "$schema": {
-          "type": "string"
-        },
-        "name": {
-          "type": "string",
-          "pattern": "^[a-z0-9-]+$"
-        },
-        "description": {
-          "type": "string"
-        },
-        "category": {
-          "type": "string"
-        },
-        "version": {
-          "type": "string"
-        },
-        "author": {
-          "type": "string"
-        },
-        "tags": {
-          "type": "array",
-          "items": {
-            "type": "string"
-          }
-        },
-        "license": {
-          "type": "string"
-        },
-        "requires": {
-          "type": "object",
-          "properties": {
-            "agentv": {
-              "type": "string"
-            }
-          },
-          "additionalProperties": false
-        },
-        "input": {
-          "anyOf": [
-            {
-              "type": "string"
-            },
-            {
-              "type": "array",
-              "items": {
-                "type": "object",
-                "properties": {
-                  "role": {
-                    "type": "string",
-                    "enum": ["system", "user", "assistant", "tool"]
-                  },
-                  "content": {
-                    "anyOf": [
-                      {
-                        "type": "string"
-                      },
-                      {
-                        "type": "array",
-                        "items": {
-                          "type": "object",
-                          "properties": {
-                            "type": {
-                              "type": "string",
-                              "enum": ["text", "file", "image"]
-                            },
-                            "value": {
-                              "type": "string"
-                            }
-                          },
-                          "required": ["type", "value"],
-                          "additionalProperties": false
-                        }
-                      }
-                    ]
-                  }
-                },
-                "required": ["role", "content"],
-                "additionalProperties": false
-              }
-            }
-          ]
-        },
-        "input_files": {
-          "type": "array",
-          "items": {
-            "type": "string"
-          }
-        },
-        "tests": {
-          "anyOf": [
-            {
-              "type": "array",
-              "items": {
-                "type": "object",
-                "properties": {
-                  "id": {
-                    "type": "string",
-                    "minLength": 1
-                  },
-                  "vars": {
-                    "type": "object",
-                    "properties": {},
-                    "additionalProperties": {}
-                  },
-                  "criteria": {
-                    "type": "string"
-                  },
-                  "input": {
-                    "anyOf": [
-                      {
-                        "type": "string"
-                      },
-                      {
-                        "type": "array",
-                        "items": {
-                          "type": "object",
-                          "properties": {
-                            "role": {
-                              "type": "string",
-                              "enum": ["system", "user", "assistant", "tool"]
-                            },
-                            "content": {
-                              "anyOf": [
-                                {
-                                  "type": "string"
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "type": {
-                                        "type": "string",
-                                        "enum": ["text", "file", "image"]
-                                      },
-                                      "value": {
-                                        "type": "string"
-                                      }
-                                    },
-                                    "required": ["type", "value"],
-                                    "additionalProperties": false
-                                  }
-                                }
-                              ]
-                            }
-                          },
-                          "required": ["role", "content"],
-                          "additionalProperties": false
-                        }
-                      }
-                    ]
-                  },
-                  "input_files": {
-                    "type": "array",
-                    "items": {
-                      "type": "string"
-                    }
-                  },
-                  "expected_output": {
-                    "anyOf": [
-                      {
-                        "type": "string"
-                      },
-                      {
-                        "type": "object",
-                        "properties": {},
-                        "additionalProperties": {}
-                      },
-                      {
-                        "type": "array",
-                        "items": {
-                          "type": "object",
-                          "properties": {
-                            "role": {
-                              "type": "string",
-                              "enum": ["system", "user", "assistant", "tool"]
-                            },
-                            "content": {
-                              "anyOf": [
-                                {
-                                  "type": "string"
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "type": {
-                                        "type": "string",
-                                        "enum": ["text", "file", "image"]
-                                      },
-                                      "value": {
-                                        "type": "string"
-                                      }
-                                    },
-                                    "required": ["type", "value"],
-                                    "additionalProperties": false
-                                  }
-                                }
-                              ]
-                            }
-                          },
-                          "required": ["role", "content"],
-                          "additionalProperties": false
-                        }
-                      }
-                    ]
-                  },
-                  "assertions": {
-                    "type": "array",
-                    "items": {
-                      "anyOf": [
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["code-grader", "code_grader"]
-                            },
-                            "command": {
-                              "anyOf": [
-                                {
-                                  "type": "string"
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "string"
-                                  }
-                                }
-                              ]
-                            },
-                            "script": {
-                              "anyOf": [
-                                {
-                                  "type": "string"
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "string"
-                                  }
-                                }
-                              ]
-                            },
-                            "cwd": {
-                              "type": "string"
-                            },
-                            "target": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "object",
-                                  "properties": {
-                                    "max_calls": {
-                                      "type": "number"
-                                    }
-                                  },
-                                  "additionalProperties": false
-                                }
-                              ]
-                            },
-                            "config": {
-                              "type": "object",
-                              "additionalProperties": {}
-                            },
-                            "preprocessors": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "type": {
-                                    "type": "string",
-                                    "minLength": 1
-                                  },
-                                  "command": {
-                                    "anyOf": [
-                                      {
-                                        "type": "string"
-                                      },
-                                      {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "string"
-                                        }
-                                      }
-                                    ]
-                                  }
-                                },
-                                "required": ["type", "command"],
-                                "additionalProperties": false
-                              }
-                            }
-                          },
-                          "required": ["type", "command"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["llm-grader", "llm_grader"]
-                            },
-                            "prompt": {
-                              "anyOf": [
-                                {
-                                  "type": "string"
-                                },
-                                {
-                                  "type": "object",
-                                  "properties": {
-                                    "command": {
-                                      "anyOf": [
-                                        {
-                                          "type": "string"
-                                        },
-                                        {
-                                          "type": "array",
-                                          "items": {
-                                            "type": "string"
-                                          }
-                                        }
-                                      ]
-                                    },
-                                    "script": {
-                                      "anyOf": [
-                                        {
-                                          "type": "string"
-                                        },
-                                        {
-                                          "type": "array",
-                                          "items": {
-                                            "type": "string"
-                                          }
-                                        }
-                                      ]
-                                    },
-                                    "config": {
-                                      "type": "object",
-                                      "additionalProperties": {}
-                                    }
-                                  },
-                                  "additionalProperties": false
-                                }
-                              ]
-                            },
-                            "rubrics": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "id": {
-                                    "type": "string"
-                                  },
-                                  "outcome": {
-                                    "type": "string"
-                                  },
-                                  "operator": {
-                                    "type": "string",
-                                    "enum": ["correctness", "contradiction"]
-                                  },
-                                  "weight": {
-                                    "type": "number"
-                                  },
-                                  "required": {
-                                    "type": "boolean"
-                                  },
-                                  "min_score": {
-                                    "type": "number",
-                                    "exclusiveMinimum": true,
-                                    "minimum": 0,
-                                    "maximum": 1
-                                  },
-                                  "score_ranges": {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "object",
-                                      "properties": {
-                                        "score_range": {
-                                          "type": "array",
-                                          "minItems": 2,
-                                          "maxItems": 2,
-                                          "items": [
-                                            {
-                                              "type": "integer",
-                                              "minimum": 0,
-                                              "maximum": 10
-                                            },
-                                            {
-                                              "type": "integer",
-                                              "minimum": 0,
-                                              "maximum": 10
-                                            }
-                                          ]
-                                        },
-                                        "outcome": {
-                                          "type": "string",
-                                          "minLength": 1
-                                        }
-                                      },
-                                      "required": ["score_range", "outcome"],
-                                      "additionalProperties": false
-                                    }
-                                  }
-                                },
-                                "additionalProperties": false
-                              }
-                            },
-                            "model": {
-                              "type": "string"
-                            },
-                            "target": {
-                              "type": "string"
-                            },
-                            "config": {
-                              "type": "object",
-                              "additionalProperties": {}
-                            },
-                            "max_steps": {
-                              "type": "integer",
-                              "minimum": 1,
-                              "maximum": 50
-                            },
-                            "temperature": {
-                              "type": "number",
-                              "minimum": 0,
-                              "maximum": 2
-                            },
-                            "preprocessors": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "type": {
-                                    "type": "string",
-                                    "minLength": 1
-                                  },
-                                  "command": {
-                                    "anyOf": [
-                                      {
-                                        "type": "string"
-                                      },
-                                      {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "string"
-                                        }
-                                      }
-                                    ]
-                                  }
-                                },
-                                "required": ["type", "command"],
-                                "additionalProperties": false
-                              }
-                            }
-                          },
-                          "required": ["type"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "include": {
-                              "type": "string",
-                              "minLength": 1
-                            }
-                          },
-                          "required": ["include"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "composite"
-                            },
-                            "assertions": {
-                              "type": "array",
-                              "items": {}
-                            },
-                            "evaluators": {
-                              "type": "array",
-                              "items": {}
-                            },
-                            "aggregator": {
-                              "anyOf": [
-                                {
-                                  "type": "object",
-                                  "properties": {
-                                    "type": {
-                                      "type": "string",
-                                      "const": "weighted_average"
-                                    },
-                                    "weights": {
-                                      "type": "object",
-                                      "additionalProperties": {
-                                        "type": "number"
-                                      }
-                                    }
-                                  },
-                                  "required": ["type"],
-                                  "additionalProperties": false
-                                },
-                                {
-                                  "type": "object",
-                                  "properties": {
-                                    "type": {
-                                      "type": "string",
-                                      "const": "threshold"
-                                    },
-                                    "threshold": {
-                                      "type": "number",
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  },
-                                  "required": ["type", "threshold"],
-                                  "additionalProperties": false
-                                },
-                                {
-                                  "type": "object",
-                                  "properties": {
-                                    "type": {
-                                      "type": "string",
-                                      "const": "code-grader"
-                                    },
-                                    "path": {
-                                      "type": "string"
-                                    },
-                                    "cwd": {
-                                      "type": "string"
-                                    }
-                                  },
-                                  "required": ["type", "path"],
-                                  "additionalProperties": false
-                                },
-                                {
-                                  "type": "object",
-                                  "properties": {
-                                    "type": {
-                                      "type": "string",
-                                      "const": "llm-grader"
-                                    },
-                                    "prompt": {
-                                      "type": "string"
-                                    },
-                                    "model": {
-                                      "type": "string"
-                                    }
-                                  },
-                                  "required": ["type"],
-                                  "additionalProperties": false
-                                }
-                              ]
-                            }
-                          },
-                          "required": ["type", "aggregator"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["tool-trajectory", "tool_trajectory"]
-                            },
-                            "mode": {
-                              "type": "string",
-                              "enum": ["any_order", "in_order", "exact", "subset", "superset"]
-                            },
-                            "minimums": {
-                              "type": "object",
-                              "additionalProperties": {
-                                "type": "integer",
-                                "minimum": 0
-                              }
-                            },
-                            "expected": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "tool": {
-                                    "type": "string"
-                                  },
-                                  "args": {
-                                    "anyOf": [
-                                      {
-                                        "type": "string",
-                                        "const": "any"
-                                      },
-                                      {
-                                        "type": "object",
-                                        "additionalProperties": {}
-                                      }
-                                    ]
-                                  },
-                                  "max_duration_ms": {
-                                    "type": "number",
-                                    "minimum": 0
-                                  },
-                                  "maxDurationMs": {
-                                    "type": "number",
-                                    "minimum": 0
-                                  },
-                                  "args_match": {
-                                    "anyOf": [
-                                      {
-                                        "type": "string",
-                                        "enum": ["exact", "ignore", "subset", "superset"]
-                                      },
-                                      {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "string"
-                                        }
-                                      }
-                                    ]
-                                  },
-                                  "argsMatch": {
-                                    "anyOf": [
-                                      {
-                                        "type": "string",
-                                        "enum": ["exact", "ignore", "subset", "superset"]
-                                      },
-                                      {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "string"
-                                        }
-                                      }
-                                    ]
-                                  }
-                                },
-                                "required": ["tool"],
-                                "additionalProperties": false
-                              }
-                            },
-                            "args_match": {
-                              "anyOf": [
-                                {
-                                  "type": "string",
-                                  "enum": ["exact", "ignore", "subset", "superset"]
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "string"
-                                  }
-                                }
-                              ]
-                            },
-                            "argsMatch": {
-                              "anyOf": [
-                                {
-                                  "type": "string",
-                                  "enum": ["exact", "ignore", "subset", "superset"]
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "string"
-                                  }
-                                }
-                              ]
-                            }
-                          },
-                          "required": ["type", "mode"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["field-accuracy", "field_accuracy"]
-                            },
-                            "fields": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "path": {
-                                    "type": "string"
-                                  },
-                                  "match": {
-                                    "type": "string",
-                                    "enum": ["exact", "numeric_tolerance", "date"]
-                                  },
-                                  "required": {
-                                    "type": "boolean"
-                                  },
-                                  "weight": {
-                                    "type": "number"
-                                  },
-                                  "tolerance": {
-                                    "type": "number",
-                                    "minimum": 0
-                                  },
-                                  "relative": {
-                                    "type": "boolean"
-                                  },
-                                  "formats": {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                },
-                                "required": ["path", "match"],
-                                "additionalProperties": false
-                              },
-                              "minItems": 1
-                            },
-                            "aggregation": {
-                              "type": "string",
-                              "enum": ["weighted_average", "all_or_nothing"]
-                            }
-                          },
-                          "required": ["type", "fields"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "latency"
-                            },
-                            "threshold": {
-                              "type": "number",
-                              "minimum": 0
-                            }
-                          },
-                          "required": ["type", "threshold"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "cost"
-                            },
-                            "budget": {
-                              "type": "number",
-                              "minimum": 0
-                            }
-                          },
-                          "required": ["type", "budget"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["token-usage", "token_usage"]
-                            },
-                            "max_total": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "max_input": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "max_output": {
-                              "type": "number",
-                              "minimum": 0
-                            }
-                          },
-                          "required": ["type"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["execution-metrics", "execution_metrics"]
-                            },
-                            "max_tool_calls": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "max_llm_calls": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "max_tokens": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "max_cost_usd": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "max_duration_ms": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "target_exploration_ratio": {
-                              "type": "number",
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "exploration_tolerance": {
-                              "type": "number",
-                              "minimum": 0
-                            }
-                          },
-                          "required": ["type"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "contains"
-                            },
-                            "value": {
-                              "type": "string"
-                            }
-                          },
-                          "required": ["type", "value"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "regex"
-                            },
-                            "value": {
-                              "type": "string"
-                            }
-                          },
-                          "required": ["type", "value"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["is-json", "is_json"]
-                            }
-                          },
-                          "required": ["type"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "equals"
-                            },
-                            "value": {
-                              "type": "string"
-                            }
-                          },
-                          "required": ["type", "value"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "rubrics"
-                            },
-                            "criteria": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "id": {
-                                    "type": "string"
-                                  },
-                                  "outcome": {
-                                    "type": "string"
-                                  },
-                                  "operator": {
-                                    "type": "string",
-                                    "enum": ["correctness", "contradiction"]
-                                  },
-                                  "weight": {
-                                    "type": "number"
-                                  },
-                                  "required": {
-                                    "type": "boolean"
-                                  },
-                                  "min_score": {
-                                    "type": "number",
-                                    "exclusiveMinimum": true,
-                                    "minimum": 0,
-                                    "maximum": 1
-                                  },
-                                  "score_ranges": {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "object",
-                                      "properties": {
-                                        "score_range": {
-                                          "type": "array",
-                                          "minItems": 2,
-                                          "maxItems": 2,
-                                          "items": [
-                                            {
-                                              "type": "integer",
-                                              "minimum": 0,
-                                              "maximum": 10
-                                            },
-                                            {
-                                              "type": "integer",
-                                              "minimum": 0,
-                                              "maximum": 10
-                                            }
-                                          ]
-                                        },
-                                        "outcome": {
-                                          "type": "string",
-                                          "minLength": 1
-                                        }
-                                      },
-                                      "required": ["score_range", "outcome"],
-                                      "additionalProperties": false
-                                    }
-                                  }
-                                },
-                                "additionalProperties": false
-                              },
-                              "minItems": 1
-                            }
-                          },
-                          "required": ["type", "criteria"],
-                          "additionalProperties": false
-                        }
-                      ]
-                    }
-                  },
-                  "evaluators": {
-                    "type": "array",
-                    "items": {
-                      "anyOf": [
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["code-grader", "code_grader"]
-                            },
-                            "command": {
-                              "anyOf": [
-                                {
-                                  "type": "string"
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "string"
-                                  }
-                                }
-                              ]
-                            },
-                            "script": {
-                              "anyOf": [
-                                {
-                                  "type": "string"
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "string"
-                                  }
-                                }
-                              ]
-                            },
-                            "cwd": {
-                              "type": "string"
-                            },
-                            "target": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "object",
-                                  "properties": {
-                                    "max_calls": {
-                                      "type": "number"
-                                    }
-                                  },
-                                  "additionalProperties": false
-                                }
-                              ]
-                            },
-                            "config": {
-                              "type": "object",
-                              "additionalProperties": {}
-                            },
-                            "preprocessors": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "type": {
-                                    "type": "string",
-                                    "minLength": 1
-                                  },
-                                  "command": {
-                                    "anyOf": [
-                                      {
-                                        "type": "string"
-                                      },
-                                      {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "string"
-                                        }
-                                      }
-                                    ]
-                                  }
-                                },
-                                "required": ["type", "command"],
-                                "additionalProperties": false
-                              }
-                            }
-                          },
-                          "required": ["type", "command"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["llm-grader", "llm_grader"]
-                            },
-                            "prompt": {
-                              "anyOf": [
-                                {
-                                  "type": "string"
-                                },
-                                {
-                                  "type": "object",
-                                  "properties": {
-                                    "command": {
-                                      "anyOf": [
-                                        {
-                                          "type": "string"
-                                        },
-                                        {
-                                          "type": "array",
-                                          "items": {
-                                            "type": "string"
-                                          }
-                                        }
-                                      ]
-                                    },
-                                    "script": {
-                                      "anyOf": [
-                                        {
-                                          "type": "string"
-                                        },
-                                        {
-                                          "type": "array",
-                                          "items": {
-                                            "type": "string"
-                                          }
-                                        }
-                                      ]
-                                    },
-                                    "config": {
-                                      "type": "object",
-                                      "additionalProperties": {}
-                                    }
-                                  },
-                                  "additionalProperties": false
-                                }
-                              ]
-                            },
-                            "rubrics": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "id": {
-                                    "type": "string"
-                                  },
-                                  "outcome": {
-                                    "type": "string"
-                                  },
-                                  "operator": {
-                                    "type": "string",
-                                    "enum": ["correctness", "contradiction"]
-                                  },
-                                  "weight": {
-                                    "type": "number"
-                                  },
-                                  "required": {
-                                    "type": "boolean"
-                                  },
-                                  "min_score": {
-                                    "type": "number",
-                                    "exclusiveMinimum": true,
-                                    "minimum": 0,
-                                    "maximum": 1
-                                  },
-                                  "score_ranges": {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "object",
-                                      "properties": {
-                                        "score_range": {
-                                          "type": "array",
-                                          "minItems": 2,
-                                          "maxItems": 2,
-                                          "items": [
-                                            {
-                                              "type": "integer",
-                                              "minimum": 0,
-                                              "maximum": 10
-                                            },
-                                            {
-                                              "type": "integer",
-                                              "minimum": 0,
-                                              "maximum": 10
-                                            }
-                                          ]
-                                        },
-                                        "outcome": {
-                                          "type": "string",
-                                          "minLength": 1
-                                        }
-                                      },
-                                      "required": ["score_range", "outcome"],
-                                      "additionalProperties": false
-                                    }
-                                  }
-                                },
-                                "additionalProperties": false
-                              }
-                            },
-                            "model": {
-                              "type": "string"
-                            },
-                            "target": {
-                              "type": "string"
-                            },
-                            "config": {
-                              "type": "object",
-                              "additionalProperties": {}
-                            },
-                            "max_steps": {
-                              "type": "integer",
-                              "minimum": 1,
-                              "maximum": 50
-                            },
-                            "temperature": {
-                              "type": "number",
-                              "minimum": 0,
-                              "maximum": 2
-                            },
-                            "preprocessors": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "type": {
-                                    "type": "string",
-                                    "minLength": 1
-                                  },
-                                  "command": {
-                                    "anyOf": [
-                                      {
-                                        "type": "string"
-                                      },
-                                      {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "string"
-                                        }
-                                      }
-                                    ]
-                                  }
-                                },
-                                "required": ["type", "command"],
-                                "additionalProperties": false
-                              }
-                            }
-                          },
-                          "required": ["type"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "include": {
-                              "type": "string",
-                              "minLength": 1
-                            }
-                          },
-                          "required": ["include"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "composite"
-                            },
-                            "assertions": {
-                              "type": "array",
-                              "items": {}
-                            },
-                            "evaluators": {
-                              "type": "array",
-                              "items": {}
-                            },
-                            "aggregator": {
-                              "anyOf": [
-                                {
-                                  "type": "object",
-                                  "properties": {
-                                    "type": {
-                                      "type": "string",
-                                      "const": "weighted_average"
-                                    },
-                                    "weights": {
-                                      "type": "object",
-                                      "additionalProperties": {
-                                        "type": "number"
-                                      }
-                                    }
-                                  },
-                                  "required": ["type"],
-                                  "additionalProperties": false
-                                },
-                                {
-                                  "type": "object",
-                                  "properties": {
-                                    "type": {
-                                      "type": "string",
-                                      "const": "threshold"
-                                    },
-                                    "threshold": {
-                                      "type": "number",
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  },
-                                  "required": ["type", "threshold"],
-                                  "additionalProperties": false
-                                },
-                                {
-                                  "type": "object",
-                                  "properties": {
-                                    "type": {
-                                      "type": "string",
-                                      "const": "code-grader"
-                                    },
-                                    "path": {
-                                      "type": "string"
-                                    },
-                                    "cwd": {
-                                      "type": "string"
-                                    }
-                                  },
-                                  "required": ["type", "path"],
-                                  "additionalProperties": false
-                                },
-                                {
-                                  "type": "object",
-                                  "properties": {
-                                    "type": {
-                                      "type": "string",
-                                      "const": "llm-grader"
-                                    },
-                                    "prompt": {
-                                      "type": "string"
-                                    },
-                                    "model": {
-                                      "type": "string"
-                                    }
-                                  },
-                                  "required": ["type"],
-                                  "additionalProperties": false
-                                }
-                              ]
-                            }
-                          },
-                          "required": ["type", "aggregator"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["tool-trajectory", "tool_trajectory"]
-                            },
-                            "mode": {
-                              "type": "string",
-                              "enum": ["any_order", "in_order", "exact", "subset", "superset"]
-                            },
-                            "minimums": {
-                              "type": "object",
-                              "additionalProperties": {
-                                "type": "integer",
-                                "minimum": 0
-                              }
-                            },
-                            "expected": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "tool": {
-                                    "type": "string"
-                                  },
-                                  "args": {
-                                    "anyOf": [
-                                      {
-                                        "type": "string",
-                                        "const": "any"
-                                      },
-                                      {
-                                        "type": "object",
-                                        "additionalProperties": {}
-                                      }
-                                    ]
-                                  },
-                                  "max_duration_ms": {
-                                    "type": "number",
-                                    "minimum": 0
-                                  },
-                                  "maxDurationMs": {
-                                    "type": "number",
-                                    "minimum": 0
-                                  },
-                                  "args_match": {
-                                    "anyOf": [
-                                      {
-                                        "type": "string",
-                                        "enum": ["exact", "ignore", "subset", "superset"]
-                                      },
-                                      {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "string"
-                                        }
-                                      }
-                                    ]
-                                  },
-                                  "argsMatch": {
-                                    "anyOf": [
-                                      {
-                                        "type": "string",
-                                        "enum": ["exact", "ignore", "subset", "superset"]
-                                      },
-                                      {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "string"
-                                        }
-                                      }
-                                    ]
-                                  }
-                                },
-                                "required": ["tool"],
-                                "additionalProperties": false
-                              }
-                            },
-                            "args_match": {
-                              "anyOf": [
-                                {
-                                  "type": "string",
-                                  "enum": ["exact", "ignore", "subset", "superset"]
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "string"
-                                  }
-                                }
-                              ]
-                            },
-                            "argsMatch": {
-                              "anyOf": [
-                                {
-                                  "type": "string",
-                                  "enum": ["exact", "ignore", "subset", "superset"]
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "string"
-                                  }
-                                }
-                              ]
-                            }
-                          },
-                          "required": ["type", "mode"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["field-accuracy", "field_accuracy"]
-                            },
-                            "fields": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "path": {
-                                    "type": "string"
-                                  },
-                                  "match": {
-                                    "type": "string",
-                                    "enum": ["exact", "numeric_tolerance", "date"]
-                                  },
-                                  "required": {
-                                    "type": "boolean"
-                                  },
-                                  "weight": {
-                                    "type": "number"
-                                  },
-                                  "tolerance": {
-                                    "type": "number",
-                                    "minimum": 0
-                                  },
-                                  "relative": {
-                                    "type": "boolean"
-                                  },
-                                  "formats": {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                },
-                                "required": ["path", "match"],
-                                "additionalProperties": false
-                              },
-                              "minItems": 1
-                            },
-                            "aggregation": {
-                              "type": "string",
-                              "enum": ["weighted_average", "all_or_nothing"]
-                            }
-                          },
-                          "required": ["type", "fields"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "latency"
-                            },
-                            "threshold": {
-                              "type": "number",
-                              "minimum": 0
-                            }
-                          },
-                          "required": ["type", "threshold"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "cost"
-                            },
-                            "budget": {
-                              "type": "number",
-                              "minimum": 0
-                            }
-                          },
-                          "required": ["type", "budget"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["token-usage", "token_usage"]
-                            },
-                            "max_total": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "max_input": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "max_output": {
-                              "type": "number",
-                              "minimum": 0
-                            }
-                          },
-                          "required": ["type"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["execution-metrics", "execution_metrics"]
-                            },
-                            "max_tool_calls": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "max_llm_calls": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "max_tokens": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "max_cost_usd": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "max_duration_ms": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "target_exploration_ratio": {
-                              "type": "number",
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "exploration_tolerance": {
-                              "type": "number",
-                              "minimum": 0
-                            }
-                          },
-                          "required": ["type"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "contains"
-                            },
-                            "value": {
-                              "type": "string"
-                            }
-                          },
-                          "required": ["type", "value"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "regex"
-                            },
-                            "value": {
-                              "type": "string"
-                            }
-                          },
-                          "required": ["type", "value"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["is-json", "is_json"]
-                            }
-                          },
-                          "required": ["type"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "equals"
-                            },
-                            "value": {
-                              "type": "string"
-                            }
-                          },
-                          "required": ["type", "value"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "rubrics"
-                            },
-                            "criteria": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "id": {
-                                    "type": "string"
-                                  },
-                                  "outcome": {
-                                    "type": "string"
-                                  },
-                                  "operator": {
-                                    "type": "string",
-                                    "enum": ["correctness", "contradiction"]
-                                  },
-                                  "weight": {
-                                    "type": "number"
-                                  },
-                                  "required": {
-                                    "type": "boolean"
-                                  },
-                                  "min_score": {
-                                    "type": "number",
-                                    "exclusiveMinimum": true,
-                                    "minimum": 0,
-                                    "maximum": 1
-                                  },
-                                  "score_ranges": {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "object",
-                                      "properties": {
-                                        "score_range": {
-                                          "type": "array",
-                                          "minItems": 2,
-                                          "maxItems": 2,
-                                          "items": [
-                                            {
-                                              "type": "integer",
-                                              "minimum": 0,
-                                              "maximum": 10
-                                            },
-                                            {
-                                              "type": "integer",
-                                              "minimum": 0,
-                                              "maximum": 10
-                                            }
-                                          ]
-                                        },
-                                        "outcome": {
-                                          "type": "string",
-                                          "minLength": 1
-                                        }
-                                      },
-                                      "required": ["score_range", "outcome"],
-                                      "additionalProperties": false
-                                    }
-                                  }
-                                },
-                                "additionalProperties": false
-                              },
-                              "minItems": 1
-                            }
-                          },
-                          "required": ["type", "criteria"],
-                          "additionalProperties": false
-                        }
-                      ]
-                    }
-                  },
-                  "execution": {
-                    "type": "object",
-                    "properties": {
-                      "target": {
-                        "type": "string"
-                      },
-                      "targets": {
-                        "type": "array",
-                        "items": {
-                          "anyOf": [
-                            {
-                              "type": "string"
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string",
-                                  "minLength": 1
-                                },
-                                "use_target": {
-                                  "type": "string"
-                                },
-                                "hooks": {
-                                  "type": "object",
-                                  "properties": {
-                                    "before_all": {
-                                      "type": "object",
-                                      "properties": {
-                                        "command": {
-                                          "anyOf": [
-                                            {
-                                              "type": "string"
-                                            },
-                                            {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          ]
-                                        },
-                                        "script": {
-                                          "anyOf": [
-                                            {
-                                              "type": "string"
-                                            },
-                                            {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          ]
-                                        },
-                                        "timeout_ms": {
-                                          "type": "number"
-                                        },
-                                        "timeoutMs": {
-                                          "type": "number"
-                                        },
-                                        "cwd": {
-                                          "type": "string"
-                                        },
-                                        "reset": {
-                                          "type": "string",
-                                          "enum": ["none", "fast", "strict"]
-                                        }
-                                      },
-                                      "additionalProperties": false
-                                    },
-                                    "before_each": {
-                                      "type": "object",
-                                      "properties": {
-                                        "command": {
-                                          "anyOf": [
-                                            {
-                                              "type": "string"
-                                            },
-                                            {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          ]
-                                        },
-                                        "script": {
-                                          "anyOf": [
-                                            {
-                                              "type": "string"
-                                            },
-                                            {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          ]
-                                        },
-                                        "timeout_ms": {
-                                          "type": "number"
-                                        },
-                                        "timeoutMs": {
-                                          "type": "number"
-                                        },
-                                        "cwd": {
-                                          "type": "string"
-                                        },
-                                        "reset": {
-                                          "type": "string",
-                                          "enum": ["none", "fast", "strict"]
-                                        }
-                                      },
-                                      "additionalProperties": false
-                                    },
-                                    "after_each": {
-                                      "type": "object",
-                                      "properties": {
-                                        "command": {
-                                          "anyOf": [
-                                            {
-                                              "type": "string"
-                                            },
-                                            {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          ]
-                                        },
-                                        "script": {
-                                          "anyOf": [
-                                            {
-                                              "type": "string"
-                                            },
-                                            {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          ]
-                                        },
-                                        "timeout_ms": {
-                                          "type": "number"
-                                        },
-                                        "timeoutMs": {
-                                          "type": "number"
-                                        },
-                                        "cwd": {
-                                          "type": "string"
-                                        },
-                                        "reset": {
-                                          "type": "string",
-                                          "enum": ["none", "fast", "strict"]
-                                        }
-                                      },
-                                      "additionalProperties": false
-                                    },
-                                    "after_all": {
-                                      "type": "object",
-                                      "properties": {
-                                        "command": {
-                                          "anyOf": [
-                                            {
-                                              "type": "string"
-                                            },
-                                            {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          ]
-                                        },
-                                        "script": {
-                                          "anyOf": [
-                                            {
-                                              "type": "string"
-                                            },
-                                            {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          ]
-                                        },
-                                        "timeout_ms": {
-                                          "type": "number"
-                                        },
-                                        "timeoutMs": {
-                                          "type": "number"
-                                        },
-                                        "cwd": {
-                                          "type": "string"
-                                        },
-                                        "reset": {
-                                          "type": "string",
-                                          "enum": ["none", "fast", "strict"]
-                                        }
-                                      },
-                                      "additionalProperties": false
-                                    }
-                                  },
-                                  "additionalProperties": false
-                                }
-                              },
-                              "required": ["name"],
-                              "additionalProperties": false
-                            }
-                          ]
-                        }
-                      },
-                      "workers": {
-                        "type": "integer",
-                        "minimum": 1,
-                        "maximum": 50
-                      },
-                      "assertions": {
-                        "type": "array",
-                        "items": {
-                          "anyOf": [
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["code-grader", "code_grader"]
-                                },
-                                "command": {
-                                  "anyOf": [
-                                    {
-                                      "type": "string"
-                                    },
-                                    {
-                                      "type": "array",
-                                      "items": {
-                                        "type": "string"
-                                      }
-                                    }
-                                  ]
-                                },
-                                "script": {
-                                  "anyOf": [
-                                    {
-                                      "type": "string"
-                                    },
-                                    {
-                                      "type": "array",
-                                      "items": {
-                                        "type": "string"
-                                      }
-                                    }
-                                  ]
-                                },
-                                "cwd": {
-                                  "type": "string"
-                                },
-                                "target": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "object",
-                                      "properties": {
-                                        "max_calls": {
-                                          "type": "number"
-                                        }
-                                      },
-                                      "additionalProperties": false
-                                    }
-                                  ]
-                                },
-                                "config": {
-                                  "type": "object",
-                                  "additionalProperties": {}
-                                },
-                                "preprocessors": {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "type": {
-                                        "type": "string",
-                                        "minLength": 1
-                                      },
-                                      "command": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string"
-                                          },
-                                          {
-                                            "type": "array",
-                                            "items": {
-                                              "type": "string"
-                                            }
-                                          }
-                                        ]
-                                      }
-                                    },
-                                    "required": ["type", "command"],
-                                    "additionalProperties": false
-                                  }
-                                }
-                              },
-                              "required": ["type", "command"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["llm-grader", "llm_grader"]
-                                },
-                                "prompt": {
-                                  "anyOf": [
-                                    {
-                                      "type": "string"
-                                    },
-                                    {
-                                      "type": "object",
-                                      "properties": {
-                                        "command": {
-                                          "anyOf": [
-                                            {
-                                              "type": "string"
-                                            },
-                                            {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          ]
-                                        },
-                                        "script": {
-                                          "anyOf": [
-                                            {
-                                              "type": "string"
-                                            },
-                                            {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          ]
-                                        },
-                                        "config": {
-                                          "type": "object",
-                                          "additionalProperties": {}
-                                        }
-                                      },
-                                      "additionalProperties": false
-                                    }
-                                  ]
-                                },
-                                "rubrics": {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "id": {
-                                        "type": "string"
-                                      },
-                                      "outcome": {
-                                        "type": "string"
-                                      },
-                                      "operator": {
-                                        "type": "string",
-                                        "enum": ["correctness", "contradiction"]
-                                      },
-                                      "weight": {
-                                        "type": "number"
-                                      },
-                                      "required": {
-                                        "type": "boolean"
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "score_ranges": {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "object",
-                                          "properties": {
-                                            "score_range": {
-                                              "type": "array",
-                                              "minItems": 2,
-                                              "maxItems": 2,
-                                              "items": [
-                                                {
-                                                  "type": "integer",
-                                                  "minimum": 0,
-                                                  "maximum": 10
-                                                },
-                                                {
-                                                  "type": "integer",
-                                                  "minimum": 0,
-                                                  "maximum": 10
-                                                }
-                                              ]
-                                            },
-                                            "outcome": {
-                                              "type": "string",
-                                              "minLength": 1
-                                            }
-                                          },
-                                          "required": ["score_range", "outcome"],
-                                          "additionalProperties": false
-                                        }
-                                      }
-                                    },
-                                    "additionalProperties": false
-                                  }
-                                },
-                                "model": {
-                                  "type": "string"
-                                },
-                                "target": {
-                                  "type": "string"
-                                },
-                                "config": {
-                                  "type": "object",
-                                  "additionalProperties": {}
-                                },
-                                "max_steps": {
-                                  "type": "integer",
-                                  "minimum": 1,
-                                  "maximum": 50
-                                },
-                                "temperature": {
-                                  "type": "number",
-                                  "minimum": 0,
-                                  "maximum": 2
-                                },
-                                "preprocessors": {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "type": {
-                                        "type": "string",
-                                        "minLength": 1
-                                      },
-                                      "command": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string"
-                                          },
-                                          {
-                                            "type": "array",
-                                            "items": {
-                                              "type": "string"
-                                            }
-                                          }
-                                        ]
-                                      }
-                                    },
-                                    "required": ["type", "command"],
-                                    "additionalProperties": false
-                                  }
-                                }
-                              },
-                              "required": ["type"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "include": {
-                                  "type": "string",
-                                  "minLength": 1
-                                }
-                              },
-                              "required": ["include"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "composite"
-                                },
-                                "assertions": {
-                                  "type": "array",
-                                  "items": {}
-                                },
-                                "evaluators": {
-                                  "type": "array",
-                                  "items": {}
-                                },
-                                "aggregator": {
-                                  "anyOf": [
-                                    {
-                                      "type": "object",
-                                      "properties": {
-                                        "type": {
-                                          "type": "string",
-                                          "const": "weighted_average"
-                                        },
-                                        "weights": {
-                                          "type": "object",
-                                          "additionalProperties": {
-                                            "type": "number"
-                                          }
-                                        }
-                                      },
-                                      "required": ["type"],
-                                      "additionalProperties": false
-                                    },
-                                    {
-                                      "type": "object",
-                                      "properties": {
-                                        "type": {
-                                          "type": "string",
-                                          "const": "threshold"
-                                        },
-                                        "threshold": {
-                                          "type": "number",
-                                          "minimum": 0,
-                                          "maximum": 1
-                                        }
-                                      },
-                                      "required": ["type", "threshold"],
-                                      "additionalProperties": false
-                                    },
-                                    {
-                                      "type": "object",
-                                      "properties": {
-                                        "type": {
-                                          "type": "string",
-                                          "const": "code-grader"
-                                        },
-                                        "path": {
-                                          "type": "string"
-                                        },
-                                        "cwd": {
-                                          "type": "string"
-                                        }
-                                      },
-                                      "required": ["type", "path"],
-                                      "additionalProperties": false
-                                    },
-                                    {
-                                      "type": "object",
-                                      "properties": {
-                                        "type": {
-                                          "type": "string",
-                                          "const": "llm-grader"
-                                        },
-                                        "prompt": {
-                                          "type": "string"
-                                        },
-                                        "model": {
-                                          "type": "string"
-                                        }
-                                      },
-                                      "required": ["type"],
-                                      "additionalProperties": false
-                                    }
-                                  ]
-                                }
-                              },
-                              "required": ["type", "aggregator"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["tool-trajectory", "tool_trajectory"]
-                                },
-                                "mode": {
-                                  "type": "string",
-                                  "enum": ["any_order", "in_order", "exact", "subset", "superset"]
-                                },
-                                "minimums": {
-                                  "type": "object",
-                                  "additionalProperties": {
-                                    "type": "integer",
-                                    "minimum": 0
-                                  }
-                                },
-                                "expected": {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "tool": {
-                                        "type": "string"
-                                      },
-                                      "args": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string",
-                                            "const": "any"
-                                          },
-                                          {
-                                            "type": "object",
-                                            "additionalProperties": {}
-                                          }
-                                        ]
-                                      },
-                                      "max_duration_ms": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "maxDurationMs": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "args_match": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string",
-                                            "enum": ["exact", "ignore", "subset", "superset"]
-                                          },
-                                          {
-                                            "type": "array",
-                                            "items": {
-                                              "type": "string"
-                                            }
-                                          }
-                                        ]
-                                      },
-                                      "argsMatch": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string",
-                                            "enum": ["exact", "ignore", "subset", "superset"]
-                                          },
-                                          {
-                                            "type": "array",
-                                            "items": {
-                                              "type": "string"
-                                            }
-                                          }
-                                        ]
-                                      }
-                                    },
-                                    "required": ["tool"],
-                                    "additionalProperties": false
-                                  }
-                                },
-                                "args_match": {
-                                  "anyOf": [
-                                    {
-                                      "type": "string",
-                                      "enum": ["exact", "ignore", "subset", "superset"]
-                                    },
-                                    {
-                                      "type": "array",
-                                      "items": {
-                                        "type": "string"
-                                      }
-                                    }
-                                  ]
-                                },
-                                "argsMatch": {
-                                  "anyOf": [
-                                    {
-                                      "type": "string",
-                                      "enum": ["exact", "ignore", "subset", "superset"]
-                                    },
-                                    {
-                                      "type": "array",
-                                      "items": {
-                                        "type": "string"
-                                      }
-                                    }
-                                  ]
-                                }
-                              },
-                              "required": ["type", "mode"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["field-accuracy", "field_accuracy"]
-                                },
-                                "fields": {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "path": {
-                                        "type": "string"
-                                      },
-                                      "match": {
-                                        "type": "string",
-                                        "enum": ["exact", "numeric_tolerance", "date"]
-                                      },
-                                      "required": {
-                                        "type": "boolean"
-                                      },
-                                      "weight": {
-                                        "type": "number"
-                                      },
-                                      "tolerance": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "relative": {
-                                        "type": "boolean"
-                                      },
-                                      "formats": {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "string"
-                                        }
-                                      }
-                                    },
-                                    "required": ["path", "match"],
-                                    "additionalProperties": false
-                                  },
-                                  "minItems": 1
-                                },
-                                "aggregation": {
-                                  "type": "string",
-                                  "enum": ["weighted_average", "all_or_nothing"]
-                                }
-                              },
-                              "required": ["type", "fields"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "latency"
-                                },
-                                "threshold": {
-                                  "type": "number",
-                                  "minimum": 0
-                                }
-                              },
-                              "required": ["type", "threshold"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "cost"
-                                },
-                                "budget": {
-                                  "type": "number",
-                                  "minimum": 0
-                                }
-                              },
-                              "required": ["type", "budget"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["token-usage", "token_usage"]
-                                },
-                                "max_total": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "max_input": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "max_output": {
-                                  "type": "number",
-                                  "minimum": 0
-                                }
-                              },
-                              "required": ["type"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["execution-metrics", "execution_metrics"]
-                                },
-                                "max_tool_calls": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "max_llm_calls": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "max_tokens": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "max_cost_usd": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "max_duration_ms": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "target_exploration_ratio": {
-                                  "type": "number",
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "exploration_tolerance": {
-                                  "type": "number",
-                                  "minimum": 0
-                                }
-                              },
-                              "required": ["type"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "contains"
-                                },
-                                "value": {
-                                  "type": "string"
-                                }
-                              },
-                              "required": ["type", "value"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "regex"
-                                },
-                                "value": {
-                                  "type": "string"
-                                }
-                              },
-                              "required": ["type", "value"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["is-json", "is_json"]
-                                }
-                              },
-                              "required": ["type"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "equals"
-                                },
-                                "value": {
-                                  "type": "string"
-                                }
-                              },
-                              "required": ["type", "value"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "rubrics"
-                                },
-                                "criteria": {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "id": {
-                                        "type": "string"
-                                      },
-                                      "outcome": {
-                                        "type": "string"
-                                      },
-                                      "operator": {
-                                        "type": "string",
-                                        "enum": ["correctness", "contradiction"]
-                                      },
-                                      "weight": {
-                                        "type": "number"
-                                      },
-                                      "required": {
-                                        "type": "boolean"
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "score_ranges": {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "object",
-                                          "properties": {
-                                            "score_range": {
-                                              "type": "array",
-                                              "minItems": 2,
-                                              "maxItems": 2,
-                                              "items": [
-                                                {
-                                                  "type": "integer",
-                                                  "minimum": 0,
-                                                  "maximum": 10
-                                                },
-                                                {
-                                                  "type": "integer",
-                                                  "minimum": 0,
-                                                  "maximum": 10
-                                                }
-                                              ]
-                                            },
-                                            "outcome": {
-                                              "type": "string",
-                                              "minLength": 1
-                                            }
-                                          },
-                                          "required": ["score_range", "outcome"],
-                                          "additionalProperties": false
-                                        }
-                                      }
-                                    },
-                                    "additionalProperties": false
-                                  },
-                                  "minItems": 1
-                                }
-                              },
-                              "required": ["type", "criteria"],
-                              "additionalProperties": false
-                            }
-                          ]
-                        }
-                      },
-                      "evaluators": {
-                        "type": "array",
-                        "items": {
-                          "anyOf": [
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["code-grader", "code_grader"]
-                                },
-                                "command": {
-                                  "anyOf": [
-                                    {
-                                      "type": "string"
-                                    },
-                                    {
-                                      "type": "array",
-                                      "items": {
-                                        "type": "string"
-                                      }
-                                    }
-                                  ]
-                                },
-                                "script": {
-                                  "anyOf": [
-                                    {
-                                      "type": "string"
-                                    },
-                                    {
-                                      "type": "array",
-                                      "items": {
-                                        "type": "string"
-                                      }
-                                    }
-                                  ]
-                                },
-                                "cwd": {
-                                  "type": "string"
-                                },
-                                "target": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "object",
-                                      "properties": {
-                                        "max_calls": {
-                                          "type": "number"
-                                        }
-                                      },
-                                      "additionalProperties": false
-                                    }
-                                  ]
-                                },
-                                "config": {
-                                  "type": "object",
-                                  "additionalProperties": {}
-                                },
-                                "preprocessors": {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "type": {
-                                        "type": "string",
-                                        "minLength": 1
-                                      },
-                                      "command": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string"
-                                          },
-                                          {
-                                            "type": "array",
-                                            "items": {
-                                              "type": "string"
-                                            }
-                                          }
-                                        ]
-                                      }
-                                    },
-                                    "required": ["type", "command"],
-                                    "additionalProperties": false
-                                  }
-                                }
-                              },
-                              "required": ["type", "command"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["llm-grader", "llm_grader"]
-                                },
-                                "prompt": {
-                                  "anyOf": [
-                                    {
-                                      "type": "string"
-                                    },
-                                    {
-                                      "type": "object",
-                                      "properties": {
-                                        "command": {
-                                          "anyOf": [
-                                            {
-                                              "type": "string"
-                                            },
-                                            {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          ]
-                                        },
-                                        "script": {
-                                          "anyOf": [
-                                            {
-                                              "type": "string"
-                                            },
-                                            {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          ]
-                                        },
-                                        "config": {
-                                          "type": "object",
-                                          "additionalProperties": {}
-                                        }
-                                      },
-                                      "additionalProperties": false
-                                    }
-                                  ]
-                                },
-                                "rubrics": {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "id": {
-                                        "type": "string"
-                                      },
-                                      "outcome": {
-                                        "type": "string"
-                                      },
-                                      "operator": {
-                                        "type": "string",
-                                        "enum": ["correctness", "contradiction"]
-                                      },
-                                      "weight": {
-                                        "type": "number"
-                                      },
-                                      "required": {
-                                        "type": "boolean"
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "score_ranges": {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "object",
-                                          "properties": {
-                                            "score_range": {
-                                              "type": "array",
-                                              "minItems": 2,
-                                              "maxItems": 2,
-                                              "items": [
-                                                {
-                                                  "type": "integer",
-                                                  "minimum": 0,
-                                                  "maximum": 10
-                                                },
-                                                {
-                                                  "type": "integer",
-                                                  "minimum": 0,
-                                                  "maximum": 10
-                                                }
-                                              ]
-                                            },
-                                            "outcome": {
-                                              "type": "string",
-                                              "minLength": 1
-                                            }
-                                          },
-                                          "required": ["score_range", "outcome"],
-                                          "additionalProperties": false
-                                        }
-                                      }
-                                    },
-                                    "additionalProperties": false
-                                  }
-                                },
-                                "model": {
-                                  "type": "string"
-                                },
-                                "target": {
-                                  "type": "string"
-                                },
-                                "config": {
-                                  "type": "object",
-                                  "additionalProperties": {}
-                                },
-                                "max_steps": {
-                                  "type": "integer",
-                                  "minimum": 1,
-                                  "maximum": 50
-                                },
-                                "temperature": {
-                                  "type": "number",
-                                  "minimum": 0,
-                                  "maximum": 2
-                                },
-                                "preprocessors": {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "type": {
-                                        "type": "string",
-                                        "minLength": 1
-                                      },
-                                      "command": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string"
-                                          },
-                                          {
-                                            "type": "array",
-                                            "items": {
-                                              "type": "string"
-                                            }
-                                          }
-                                        ]
-                                      }
-                                    },
-                                    "required": ["type", "command"],
-                                    "additionalProperties": false
-                                  }
-                                }
-                              },
-                              "required": ["type"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "include": {
-                                  "type": "string",
-                                  "minLength": 1
-                                }
-                              },
-                              "required": ["include"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "composite"
-                                },
-                                "assertions": {
-                                  "type": "array",
-                                  "items": {}
-                                },
-                                "evaluators": {
-                                  "type": "array",
-                                  "items": {}
-                                },
-                                "aggregator": {
-                                  "anyOf": [
-                                    {
-                                      "type": "object",
-                                      "properties": {
-                                        "type": {
-                                          "type": "string",
-                                          "const": "weighted_average"
-                                        },
-                                        "weights": {
-                                          "type": "object",
-                                          "additionalProperties": {
-                                            "type": "number"
-                                          }
-                                        }
-                                      },
-                                      "required": ["type"],
-                                      "additionalProperties": false
-                                    },
-                                    {
-                                      "type": "object",
-                                      "properties": {
-                                        "type": {
-                                          "type": "string",
-                                          "const": "threshold"
-                                        },
-                                        "threshold": {
-                                          "type": "number",
-                                          "minimum": 0,
-                                          "maximum": 1
-                                        }
-                                      },
-                                      "required": ["type", "threshold"],
-                                      "additionalProperties": false
-                                    },
-                                    {
-                                      "type": "object",
-                                      "properties": {
-                                        "type": {
-                                          "type": "string",
-                                          "const": "code-grader"
-                                        },
-                                        "path": {
-                                          "type": "string"
-                                        },
-                                        "cwd": {
-                                          "type": "string"
-                                        }
-                                      },
-                                      "required": ["type", "path"],
-                                      "additionalProperties": false
-                                    },
-                                    {
-                                      "type": "object",
-                                      "properties": {
-                                        "type": {
-                                          "type": "string",
-                                          "const": "llm-grader"
-                                        },
-                                        "prompt": {
-                                          "type": "string"
-                                        },
-                                        "model": {
-                                          "type": "string"
-                                        }
-                                      },
-                                      "required": ["type"],
-                                      "additionalProperties": false
-                                    }
-                                  ]
-                                }
-                              },
-                              "required": ["type", "aggregator"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["tool-trajectory", "tool_trajectory"]
-                                },
-                                "mode": {
-                                  "type": "string",
-                                  "enum": ["any_order", "in_order", "exact", "subset", "superset"]
-                                },
-                                "minimums": {
-                                  "type": "object",
-                                  "additionalProperties": {
-                                    "type": "integer",
-                                    "minimum": 0
-                                  }
-                                },
-                                "expected": {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "tool": {
-                                        "type": "string"
-                                      },
-                                      "args": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string",
-                                            "const": "any"
-                                          },
-                                          {
-                                            "type": "object",
-                                            "additionalProperties": {}
-                                          }
-                                        ]
-                                      },
-                                      "max_duration_ms": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "maxDurationMs": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "args_match": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string",
-                                            "enum": ["exact", "ignore", "subset", "superset"]
-                                          },
-                                          {
-                                            "type": "array",
-                                            "items": {
-                                              "type": "string"
-                                            }
-                                          }
-                                        ]
-                                      },
-                                      "argsMatch": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string",
-                                            "enum": ["exact", "ignore", "subset", "superset"]
-                                          },
-                                          {
-                                            "type": "array",
-                                            "items": {
-                                              "type": "string"
-                                            }
-                                          }
-                                        ]
-                                      }
-                                    },
-                                    "required": ["tool"],
-                                    "additionalProperties": false
-                                  }
-                                },
-                                "args_match": {
-                                  "anyOf": [
-                                    {
-                                      "type": "string",
-                                      "enum": ["exact", "ignore", "subset", "superset"]
-                                    },
-                                    {
-                                      "type": "array",
-                                      "items": {
-                                        "type": "string"
-                                      }
-                                    }
-                                  ]
-                                },
-                                "argsMatch": {
-                                  "anyOf": [
-                                    {
-                                      "type": "string",
-                                      "enum": ["exact", "ignore", "subset", "superset"]
-                                    },
-                                    {
-                                      "type": "array",
-                                      "items": {
-                                        "type": "string"
-                                      }
-                                    }
-                                  ]
-                                }
-                              },
-                              "required": ["type", "mode"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["field-accuracy", "field_accuracy"]
-                                },
-                                "fields": {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "path": {
-                                        "type": "string"
-                                      },
-                                      "match": {
-                                        "type": "string",
-                                        "enum": ["exact", "numeric_tolerance", "date"]
-                                      },
-                                      "required": {
-                                        "type": "boolean"
-                                      },
-                                      "weight": {
-                                        "type": "number"
-                                      },
-                                      "tolerance": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "relative": {
-                                        "type": "boolean"
-                                      },
-                                      "formats": {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "string"
-                                        }
-                                      }
-                                    },
-                                    "required": ["path", "match"],
-                                    "additionalProperties": false
-                                  },
-                                  "minItems": 1
-                                },
-                                "aggregation": {
-                                  "type": "string",
-                                  "enum": ["weighted_average", "all_or_nothing"]
-                                }
-                              },
-                              "required": ["type", "fields"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "latency"
-                                },
-                                "threshold": {
-                                  "type": "number",
-                                  "minimum": 0
-                                }
-                              },
-                              "required": ["type", "threshold"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "cost"
-                                },
-                                "budget": {
-                                  "type": "number",
-                                  "minimum": 0
-                                }
-                              },
-                              "required": ["type", "budget"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["token-usage", "token_usage"]
-                                },
-                                "max_total": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "max_input": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "max_output": {
-                                  "type": "number",
-                                  "minimum": 0
-                                }
-                              },
-                              "required": ["type"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["execution-metrics", "execution_metrics"]
-                                },
-                                "max_tool_calls": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "max_llm_calls": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "max_tokens": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "max_cost_usd": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "max_duration_ms": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "target_exploration_ratio": {
-                                  "type": "number",
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "exploration_tolerance": {
-                                  "type": "number",
-                                  "minimum": 0
-                                }
-                              },
-                              "required": ["type"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "contains"
-                                },
-                                "value": {
-                                  "type": "string"
-                                }
-                              },
-                              "required": ["type", "value"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "regex"
-                                },
-                                "value": {
-                                  "type": "string"
-                                }
-                              },
-                              "required": ["type", "value"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["is-json", "is_json"]
-                                }
-                              },
-                              "required": ["type"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "equals"
-                                },
-                                "value": {
-                                  "type": "string"
-                                }
-                              },
-                              "required": ["type", "value"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "rubrics"
-                                },
-                                "criteria": {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "id": {
-                                        "type": "string"
-                                      },
-                                      "outcome": {
-                                        "type": "string"
-                                      },
-                                      "operator": {
-                                        "type": "string",
-                                        "enum": ["correctness", "contradiction"]
-                                      },
-                                      "weight": {
-                                        "type": "number"
-                                      },
-                                      "required": {
-                                        "type": "boolean"
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "score_ranges": {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "object",
-                                          "properties": {
-                                            "score_range": {
-                                              "type": "array",
-                                              "minItems": 2,
-                                              "maxItems": 2,
-                                              "items": [
-                                                {
-                                                  "type": "integer",
-                                                  "minimum": 0,
-                                                  "maximum": 10
-                                                },
-                                                {
-                                                  "type": "integer",
-                                                  "minimum": 0,
-                                                  "maximum": 10
-                                                }
-                                              ]
-                                            },
-                                            "outcome": {
-                                              "type": "string",
-                                              "minLength": 1
-                                            }
-                                          },
-                                          "required": ["score_range", "outcome"],
-                                          "additionalProperties": false
-                                        }
-                                      }
-                                    },
-                                    "additionalProperties": false
-                                  },
-                                  "minItems": 1
-                                }
-                              },
-                              "required": ["type", "criteria"],
-                              "additionalProperties": false
-                            }
-                          ]
-                        }
-                      },
-                      "skip_defaults": {
-                        "type": "boolean"
-                      },
-                      "cache": {
-                        "type": "boolean"
-                      },
-                      "trials": {
-                        "type": "object",
-                        "properties": {
-                          "count": {
-                            "type": "integer",
-                            "minimum": 1
-                          },
-                          "strategy": {
-                            "type": "string",
-                            "enum": ["pass_at_k", "mean", "confidence_interval"]
-                          },
-                          "cost_limit_usd": {
-                            "type": "number",
-                            "minimum": 0
-                          },
-                          "costLimitUsd": {
-                            "type": "number",
-                            "minimum": 0
-                          }
-                        },
-                        "required": ["count"],
-                        "additionalProperties": false
-                      },
-                      "budget_usd": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "budgetUsd": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "fail_on_error": {
-                        "type": "boolean"
-                      },
-                      "failOnError": {
-                        "type": "boolean"
-                      },
-                      "threshold": {
-                        "type": "number",
-                        "minimum": 0,
-                        "maximum": 1
-                      }
-                    },
-                    "additionalProperties": false
-                  },
-                  "workspace": {
-                    "type": "object",
-                    "properties": {
-                      "template": {
-                        "type": "string"
-                      },
-                      "isolation": {
-                        "type": "string",
-                        "enum": ["shared", "per_test"]
-                      },
-                      "repos": {
-                        "type": "array",
-                        "items": {
-                          "type": "object",
-                          "properties": {
-                            "path": {
-                              "type": "string"
-                            },
-                            "repo": {
-                              "type": "string",
-                              "minLength": 1
-                            },
-                            "commit": {
-                              "type": "string",
-                              "minLength": 1
-                            },
-                            "base_commit": {
-                              "type": "string",
-                              "minLength": 1
-                            },
-                            "ancestor": {
-                              "type": "integer",
-                              "minimum": 0
-                            },
-                            "sparse": {
-                              "type": "array",
-                              "items": {
-                                "type": "string"
-                              }
-                            }
-                          },
-                          "additionalProperties": false
-                        }
-                      },
-                      "hooks": {
-                        "type": "object",
-                        "properties": {
-                          "enabled": {
-                            "type": "boolean"
-                          },
-                          "before_all": {
-                            "type": "object",
-                            "properties": {
-                              "command": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "script": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "timeout_ms": {
-                                "type": "number"
-                              },
-                              "timeoutMs": {
-                                "type": "number"
-                              },
-                              "cwd": {
-                                "type": "string"
-                              },
-                              "reset": {
-                                "type": "string",
-                                "enum": ["none", "fast", "strict"]
-                              }
-                            },
-                            "additionalProperties": false
-                          },
-                          "before_each": {
-                            "type": "object",
-                            "properties": {
-                              "command": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "script": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "timeout_ms": {
-                                "type": "number"
-                              },
-                              "timeoutMs": {
-                                "type": "number"
-                              },
-                              "cwd": {
-                                "type": "string"
-                              },
-                              "reset": {
-                                "type": "string",
-                                "enum": ["none", "fast", "strict"]
-                              }
-                            },
-                            "additionalProperties": false
-                          },
-                          "after_each": {
-                            "type": "object",
-                            "properties": {
-                              "command": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "script": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "timeout_ms": {
-                                "type": "number"
-                              },
-                              "timeoutMs": {
-                                "type": "number"
-                              },
-                              "cwd": {
-                                "type": "string"
-                              },
-                              "reset": {
-                                "type": "string",
-                                "enum": ["none", "fast", "strict"]
-                              }
-                            },
-                            "additionalProperties": false
-                          },
-                          "after_all": {
-                            "type": "object",
-                            "properties": {
-                              "command": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "script": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "timeout_ms": {
-                                "type": "number"
-                              },
-                              "timeoutMs": {
-                                "type": "number"
-                              },
-                              "cwd": {
-                                "type": "string"
-                              },
-                              "reset": {
-                                "type": "string",
-                                "enum": ["none", "fast", "strict"]
-                              }
-                            },
-                            "additionalProperties": false
-                          }
-                        },
-                        "additionalProperties": false
-                      },
-                      "mode": {
-                        "type": "string",
-                        "enum": ["pooled", "temp", "static"]
-                      },
-                      "path": {
-                        "type": "string"
-                      },
-                      "docker": {
-                        "type": "object",
-                        "properties": {
-                          "image": {
-                            "type": "string"
-                          },
-                          "timeout": {
-                            "type": "integer",
-                            "minimum": 1
-                          },
-                          "memory": {
-                            "type": "string"
-                          },
-                          "cpus": {
-                            "type": "number",
-                            "minimum": 0.1
-                          }
-                        },
-                        "required": ["image"],
-                        "additionalProperties": false
-                      }
-                    },
-                    "additionalProperties": false
-                  },
-                  "metadata": {
-                    "type": "object",
-                    "additionalProperties": {}
-                  },
-                  "conversation_id": {
-                    "type": "string"
-                  },
-                  "suite": {
-                    "type": "string"
-                  },
-                  "depends_on": {
-                    "type": "array",
-                    "items": {
-                      "type": "string"
-                    }
-                  },
-                  "on_dependency_failure": {
-                    "type": "string",
-                    "enum": ["skip", "fail", "run"]
-                  },
-                  "mode": {
-                    "type": "string",
-                    "enum": ["conversation"]
-                  },
-                  "turns": {
-                    "type": "array",
-                    "items": {
-                      "type": "object",
-                      "properties": {
-                        "input": {
-                          "anyOf": [
-                            {
-                              "type": "string"
-                            },
-                            {
-                              "anyOf": [
-                                {
-                                  "type": "string"
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "type": {
-                                        "type": "string",
-                                        "enum": ["text", "file", "image"]
-                                      },
-                                      "value": {
-                                        "type": "string"
-                                      }
-                                    },
-                                    "required": ["type", "value"],
-                                    "additionalProperties": false
-                                  }
-                                }
-                              ]
-                            }
-                          ]
-                        },
-                        "expected_output": {
-                          "anyOf": [
-                            {
-                              "type": "string"
-                            },
-                            {
-                              "anyOf": [
-                                {
-                                  "type": "string"
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "type": {
-                                        "type": "string",
-                                        "enum": ["text", "file", "image"]
-                                      },
-                                      "value": {
-                                        "type": "string"
-                                      }
-                                    },
-                                    "required": ["type", "value"],
-                                    "additionalProperties": false
-                                  }
-                                }
-                              ]
-                            }
-                          ]
-                        },
-                        "assertions": {
-                          "type": "array",
-                          "items": {
-                            "anyOf": [
-                              {
-                                "type": "string"
-                              },
-                              {
-                                "anyOf": [
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "enum": ["code-grader", "code_grader"]
-                                      },
-                                      "command": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string"
-                                          },
-                                          {
-                                            "type": "array",
-                                            "items": {
-                                              "type": "string"
-                                            }
-                                          }
-                                        ]
-                                      },
-                                      "script": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string"
-                                          },
-                                          {
-                                            "type": "array",
-                                            "items": {
-                                              "type": "string"
-                                            }
-                                          }
-                                        ]
-                                      },
-                                      "cwd": {
-                                        "type": "string"
-                                      },
-                                      "target": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "object",
-                                            "properties": {
-                                              "max_calls": {
-                                                "type": "number"
-                                              }
-                                            },
-                                            "additionalProperties": false
-                                          }
-                                        ]
-                                      },
-                                      "config": {
-                                        "type": "object",
-                                        "additionalProperties": {}
-                                      },
-                                      "preprocessors": {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "object",
-                                          "properties": {
-                                            "type": {
-                                              "type": "string",
-                                              "minLength": 1
-                                            },
-                                            "command": {
-                                              "anyOf": [
-                                                {
-                                                  "type": "string"
-                                                },
-                                                {
-                                                  "type": "array",
-                                                  "items": {
-                                                    "type": "string"
-                                                  }
-                                                }
-                                              ]
-                                            }
-                                          },
-                                          "required": ["type", "command"],
-                                          "additionalProperties": false
-                                        }
-                                      }
-                                    },
-                                    "required": ["type", "command"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "enum": ["llm-grader", "llm_grader"]
-                                      },
-                                      "prompt": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string"
-                                          },
-                                          {
-                                            "type": "object",
-                                            "properties": {
-                                              "command": {
-                                                "anyOf": [
-                                                  {
-                                                    "type": "string"
-                                                  },
-                                                  {
-                                                    "type": "array",
-                                                    "items": {
-                                                      "type": "string"
-                                                    }
-                                                  }
-                                                ]
-                                              },
-                                              "script": {
-                                                "anyOf": [
-                                                  {
-                                                    "type": "string"
-                                                  },
-                                                  {
-                                                    "type": "array",
-                                                    "items": {
-                                                      "type": "string"
-                                                    }
-                                                  }
-                                                ]
-                                              },
-                                              "config": {
-                                                "type": "object",
-                                                "additionalProperties": {}
-                                              }
-                                            },
-                                            "additionalProperties": false
-                                          }
-                                        ]
-                                      },
-                                      "rubrics": {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "object",
-                                          "properties": {
-                                            "id": {
-                                              "type": "string"
-                                            },
-                                            "outcome": {
-                                              "type": "string"
-                                            },
-                                            "operator": {
-                                              "type": "string",
-                                              "enum": ["correctness", "contradiction"]
-                                            },
-                                            "weight": {
-                                              "type": "number"
-                                            },
-                                            "required": {
-                                              "type": "boolean"
-                                            },
-                                            "min_score": {
-                                              "type": "number",
-                                              "exclusiveMinimum": true,
-                                              "minimum": 0,
-                                              "maximum": 1
-                                            },
-                                            "score_ranges": {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "object",
-                                                "properties": {
-                                                  "score_range": {
-                                                    "type": "array",
-                                                    "minItems": 2,
-                                                    "maxItems": 2,
-                                                    "items": [
-                                                      {
-                                                        "type": "integer",
-                                                        "minimum": 0,
-                                                        "maximum": 10
-                                                      },
-                                                      {
-                                                        "type": "integer",
-                                                        "minimum": 0,
-                                                        "maximum": 10
-                                                      }
-                                                    ]
-                                                  },
-                                                  "outcome": {
-                                                    "type": "string",
-                                                    "minLength": 1
-                                                  }
-                                                },
-                                                "required": ["score_range", "outcome"],
-                                                "additionalProperties": false
-                                              }
-                                            }
-                                          },
-                                          "additionalProperties": false
-                                        }
-                                      },
-                                      "model": {
-                                        "type": "string"
-                                      },
-                                      "target": {
-                                        "type": "string"
-                                      },
-                                      "config": {
-                                        "type": "object",
-                                        "additionalProperties": {}
-                                      },
-                                      "max_steps": {
-                                        "type": "integer",
-                                        "minimum": 1,
-                                        "maximum": 50
-                                      },
-                                      "temperature": {
-                                        "type": "number",
-                                        "minimum": 0,
-                                        "maximum": 2
-                                      },
-                                      "preprocessors": {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "object",
-                                          "properties": {
-                                            "type": {
-                                              "type": "string",
-                                              "minLength": 1
-                                            },
-                                            "command": {
-                                              "anyOf": [
-                                                {
-                                                  "type": "string"
-                                                },
-                                                {
-                                                  "type": "array",
-                                                  "items": {
-                                                    "type": "string"
-                                                  }
-                                                }
-                                              ]
-                                            }
-                                          },
-                                          "required": ["type", "command"],
-                                          "additionalProperties": false
-                                        }
-                                      }
-                                    },
-                                    "required": ["type"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "include": {
-                                        "type": "string",
-                                        "minLength": 1
-                                      }
-                                    },
-                                    "required": ["include"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "const": "composite"
-                                      },
-                                      "assertions": {
-                                        "type": "array",
-                                        "items": {}
-                                      },
-                                      "evaluators": {
-                                        "type": "array",
-                                        "items": {}
-                                      },
-                                      "aggregator": {
-                                        "anyOf": [
-                                          {
-                                            "type": "object",
-                                            "properties": {
-                                              "type": {
-                                                "type": "string",
-                                                "const": "weighted_average"
-                                              },
-                                              "weights": {
-                                                "type": "object",
-                                                "additionalProperties": {
-                                                  "type": "number"
-                                                }
-                                              }
-                                            },
-                                            "required": ["type"],
-                                            "additionalProperties": false
-                                          },
-                                          {
-                                            "type": "object",
-                                            "properties": {
-                                              "type": {
-                                                "type": "string",
-                                                "const": "threshold"
-                                              },
-                                              "threshold": {
-                                                "type": "number",
-                                                "minimum": 0,
-                                                "maximum": 1
-                                              }
-                                            },
-                                            "required": ["type", "threshold"],
-                                            "additionalProperties": false
-                                          },
-                                          {
-                                            "type": "object",
-                                            "properties": {
-                                              "type": {
-                                                "type": "string",
-                                                "const": "code-grader"
-                                              },
-                                              "path": {
-                                                "type": "string"
-                                              },
-                                              "cwd": {
-                                                "type": "string"
-                                              }
-                                            },
-                                            "required": ["type", "path"],
-                                            "additionalProperties": false
-                                          },
-                                          {
-                                            "type": "object",
-                                            "properties": {
-                                              "type": {
-                                                "type": "string",
-                                                "const": "llm-grader"
-                                              },
-                                              "prompt": {
-                                                "type": "string"
-                                              },
-                                              "model": {
-                                                "type": "string"
-                                              }
-                                            },
-                                            "required": ["type"],
-                                            "additionalProperties": false
-                                          }
-                                        ]
-                                      }
-                                    },
-                                    "required": ["type", "aggregator"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "enum": ["tool-trajectory", "tool_trajectory"]
-                                      },
-                                      "mode": {
-                                        "type": "string",
-                                        "enum": [
-                                          "any_order",
-                                          "in_order",
-                                          "exact",
-                                          "subset",
-                                          "superset"
-                                        ]
-                                      },
-                                      "minimums": {
-                                        "type": "object",
-                                        "additionalProperties": {
-                                          "type": "integer",
-                                          "minimum": 0
-                                        }
-                                      },
-                                      "expected": {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "object",
-                                          "properties": {
-                                            "tool": {
-                                              "type": "string"
-                                            },
-                                            "args": {
-                                              "anyOf": [
-                                                {
-                                                  "type": "string",
-                                                  "const": "any"
-                                                },
-                                                {
-                                                  "type": "object",
-                                                  "additionalProperties": {}
-                                                }
-                                              ]
-                                            },
-                                            "max_duration_ms": {
-                                              "type": "number",
-                                              "minimum": 0
-                                            },
-                                            "maxDurationMs": {
-                                              "type": "number",
-                                              "minimum": 0
-                                            },
-                                            "args_match": {
-                                              "anyOf": [
-                                                {
-                                                  "type": "string",
-                                                  "enum": ["exact", "ignore", "subset", "superset"]
-                                                },
-                                                {
-                                                  "type": "array",
-                                                  "items": {
-                                                    "type": "string"
-                                                  }
-                                                }
-                                              ]
-                                            },
-                                            "argsMatch": {
-                                              "anyOf": [
-                                                {
-                                                  "type": "string",
-                                                  "enum": ["exact", "ignore", "subset", "superset"]
-                                                },
-                                                {
-                                                  "type": "array",
-                                                  "items": {
-                                                    "type": "string"
-                                                  }
-                                                }
-                                              ]
-                                            }
-                                          },
-                                          "required": ["tool"],
-                                          "additionalProperties": false
-                                        }
-                                      },
-                                      "args_match": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string",
-                                            "enum": ["exact", "ignore", "subset", "superset"]
-                                          },
-                                          {
-                                            "type": "array",
-                                            "items": {
-                                              "type": "string"
-                                            }
-                                          }
-                                        ]
-                                      },
-                                      "argsMatch": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string",
-                                            "enum": ["exact", "ignore", "subset", "superset"]
-                                          },
-                                          {
-                                            "type": "array",
-                                            "items": {
-                                              "type": "string"
-                                            }
-                                          }
-                                        ]
-                                      }
-                                    },
-                                    "required": ["type", "mode"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "enum": ["field-accuracy", "field_accuracy"]
-                                      },
-                                      "fields": {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "object",
-                                          "properties": {
-                                            "path": {
-                                              "type": "string"
-                                            },
-                                            "match": {
-                                              "type": "string",
-                                              "enum": ["exact", "numeric_tolerance", "date"]
-                                            },
-                                            "required": {
-                                              "type": "boolean"
-                                            },
-                                            "weight": {
-                                              "type": "number"
-                                            },
-                                            "tolerance": {
-                                              "type": "number",
-                                              "minimum": 0
-                                            },
-                                            "relative": {
-                                              "type": "boolean"
-                                            },
-                                            "formats": {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          },
-                                          "required": ["path", "match"],
-                                          "additionalProperties": false
-                                        },
-                                        "minItems": 1
-                                      },
-                                      "aggregation": {
-                                        "type": "string",
-                                        "enum": ["weighted_average", "all_or_nothing"]
-                                      }
-                                    },
-                                    "required": ["type", "fields"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "const": "latency"
-                                      },
-                                      "threshold": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      }
-                                    },
-                                    "required": ["type", "threshold"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "const": "cost"
-                                      },
-                                      "budget": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      }
-                                    },
-                                    "required": ["type", "budget"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "enum": ["token-usage", "token_usage"]
-                                      },
-                                      "max_total": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "max_input": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "max_output": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      }
-                                    },
-                                    "required": ["type"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "enum": ["execution-metrics", "execution_metrics"]
-                                      },
-                                      "max_tool_calls": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "max_llm_calls": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "max_tokens": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "max_cost_usd": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "max_duration_ms": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "target_exploration_ratio": {
-                                        "type": "number",
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "exploration_tolerance": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      }
-                                    },
-                                    "required": ["type"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "const": "contains"
-                                      },
-                                      "value": {
-                                        "type": "string"
-                                      }
-                                    },
-                                    "required": ["type", "value"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "const": "regex"
-                                      },
-                                      "value": {
-                                        "type": "string"
-                                      }
-                                    },
-                                    "required": ["type", "value"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "enum": ["is-json", "is_json"]
-                                      }
-                                    },
-                                    "required": ["type"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "const": "equals"
-                                      },
-                                      "value": {
-                                        "type": "string"
-                                      }
-                                    },
-                                    "required": ["type", "value"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "const": "rubrics"
-                                      },
-                                      "criteria": {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "object",
-                                          "properties": {
-                                            "id": {
-                                              "type": "string"
-                                            },
-                                            "outcome": {
-                                              "type": "string"
-                                            },
-                                            "operator": {
-                                              "type": "string",
-                                              "enum": ["correctness", "contradiction"]
-                                            },
-                                            "weight": {
-                                              "type": "number"
-                                            },
-                                            "required": {
-                                              "type": "boolean"
-                                            },
-                                            "min_score": {
-                                              "type": "number",
-                                              "exclusiveMinimum": true,
-                                              "minimum": 0,
-                                              "maximum": 1
-                                            },
-                                            "score_ranges": {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "object",
-                                                "properties": {
-                                                  "score_range": {
-                                                    "type": "array",
-                                                    "minItems": 2,
-                                                    "maxItems": 2,
-                                                    "items": [
-                                                      {
-                                                        "type": "integer",
-                                                        "minimum": 0,
-                                                        "maximum": 10
-                                                      },
-                                                      {
-                                                        "type": "integer",
-                                                        "minimum": 0,
-                                                        "maximum": 10
-                                                      }
-                                                    ]
-                                                  },
-                                                  "outcome": {
-                                                    "type": "string",
-                                                    "minLength": 1
-                                                  }
-                                                },
-                                                "required": ["score_range", "outcome"],
-                                                "additionalProperties": false
-                                              }
-                                            }
-                                          },
-                                          "additionalProperties": false
-                                        },
-                                        "minItems": 1
-                                      }
-                                    },
-                                    "required": ["type", "criteria"],
-                                    "additionalProperties": false
-                                  }
-                                ]
-                              }
-                            ]
-                          }
-                        }
-                      },
-                      "required": ["input"],
-                      "additionalProperties": false
-                    },
-                    "minItems": 1
-                  },
-                  "aggregation": {
-                    "type": "string",
-                    "enum": ["mean", "min", "max"]
-                  },
-                  "on_turn_failure": {
-                    "type": "string",
-                    "enum": ["continue", "stop"]
-                  },
-                  "window_size": {
-                    "type": "integer",
-                    "minimum": 1
-                  }
-                },
-                "required": ["id"],
-                "additionalProperties": false
-              }
-            },
-            {
-              "type": "string"
-            }
-          ]
-        },
-        "eval_cases": {
-          "anyOf": [
-            {
-              "type": "array",
-              "items": {
-                "type": "object",
-                "properties": {
-                  "id": {
-                    "type": "string",
-                    "minLength": 1
-                  },
-                  "vars": {
-                    "type": "object",
-                    "properties": {},
-                    "additionalProperties": {}
-                  },
-                  "criteria": {
-                    "type": "string"
-                  },
-                  "input": {
-                    "anyOf": [
-                      {
-                        "type": "string"
-                      },
-                      {
-                        "type": "array",
-                        "items": {
-                          "type": "object",
-                          "properties": {
-                            "role": {
-                              "type": "string",
-                              "enum": ["system", "user", "assistant", "tool"]
-                            },
-                            "content": {
-                              "anyOf": [
-                                {
-                                  "type": "string"
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "type": {
-                                        "type": "string",
-                                        "enum": ["text", "file", "image"]
-                                      },
-                                      "value": {
-                                        "type": "string"
-                                      }
-                                    },
-                                    "required": ["type", "value"],
-                                    "additionalProperties": false
-                                  }
-                                }
-                              ]
-                            }
-                          },
-                          "required": ["role", "content"],
-                          "additionalProperties": false
-                        }
-                      }
-                    ]
-                  },
-                  "input_files": {
-                    "type": "array",
-                    "items": {
-                      "type": "string"
-                    }
-                  },
-                  "expected_output": {
-                    "anyOf": [
-                      {
-                        "type": "string"
-                      },
-                      {
-                        "type": "object",
-                        "properties": {},
-                        "additionalProperties": {}
-                      },
-                      {
-                        "type": "array",
-                        "items": {
-                          "type": "object",
-                          "properties": {
-                            "role": {
-                              "type": "string",
-                              "enum": ["system", "user", "assistant", "tool"]
-                            },
-                            "content": {
-                              "anyOf": [
-                                {
-                                  "type": "string"
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "type": {
-                                        "type": "string",
-                                        "enum": ["text", "file", "image"]
-                                      },
-                                      "value": {
-                                        "type": "string"
-                                      }
-                                    },
-                                    "required": ["type", "value"],
-                                    "additionalProperties": false
-                                  }
-                                }
-                              ]
-                            }
-                          },
-                          "required": ["role", "content"],
-                          "additionalProperties": false
-                        }
-                      }
-                    ]
-                  },
-                  "assertions": {
-                    "type": "array",
-                    "items": {
-                      "anyOf": [
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["code-grader", "code_grader"]
-                            },
-                            "command": {
-                              "anyOf": [
-                                {
-                                  "type": "string"
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "string"
-                                  }
-                                }
-                              ]
-                            },
-                            "script": {
-                              "anyOf": [
-                                {
-                                  "type": "string"
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "string"
-                                  }
-                                }
-                              ]
-                            },
-                            "cwd": {
-                              "type": "string"
-                            },
-                            "target": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "object",
-                                  "properties": {
-                                    "max_calls": {
-                                      "type": "number"
-                                    }
-                                  },
-                                  "additionalProperties": false
-                                }
-                              ]
-                            },
-                            "config": {
-                              "type": "object",
-                              "additionalProperties": {}
-                            },
-                            "preprocessors": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "type": {
-                                    "type": "string",
-                                    "minLength": 1
-                                  },
-                                  "command": {
-                                    "anyOf": [
-                                      {
-                                        "type": "string"
-                                      },
-                                      {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "string"
-                                        }
-                                      }
-                                    ]
-                                  }
-                                },
-                                "required": ["type", "command"],
-                                "additionalProperties": false
-                              }
-                            }
-                          },
-                          "required": ["type", "command"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["llm-grader", "llm_grader"]
-                            },
-                            "prompt": {
-                              "anyOf": [
-                                {
-                                  "type": "string"
-                                },
-                                {
-                                  "type": "object",
-                                  "properties": {
-                                    "command": {
-                                      "anyOf": [
-                                        {
-                                          "type": "string"
-                                        },
-                                        {
-                                          "type": "array",
-                                          "items": {
-                                            "type": "string"
-                                          }
-                                        }
-                                      ]
-                                    },
-                                    "script": {
-                                      "anyOf": [
-                                        {
-                                          "type": "string"
-                                        },
-                                        {
-                                          "type": "array",
-                                          "items": {
-                                            "type": "string"
-                                          }
-                                        }
-                                      ]
-                                    },
-                                    "config": {
-                                      "type": "object",
-                                      "additionalProperties": {}
-                                    }
-                                  },
-                                  "additionalProperties": false
-                                }
-                              ]
-                            },
-                            "rubrics": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "id": {
-                                    "type": "string"
-                                  },
-                                  "outcome": {
-                                    "type": "string"
-                                  },
-                                  "operator": {
-                                    "type": "string",
-                                    "enum": ["correctness", "contradiction"]
-                                  },
-                                  "weight": {
-                                    "type": "number"
-                                  },
-                                  "required": {
-                                    "type": "boolean"
-                                  },
-                                  "min_score": {
-                                    "type": "number",
-                                    "exclusiveMinimum": true,
-                                    "minimum": 0,
-                                    "maximum": 1
-                                  },
-                                  "score_ranges": {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "object",
-                                      "properties": {
-                                        "score_range": {
-                                          "type": "array",
-                                          "minItems": 2,
-                                          "maxItems": 2,
-                                          "items": [
-                                            {
-                                              "type": "integer",
-                                              "minimum": 0,
-                                              "maximum": 10
-                                            },
-                                            {
-                                              "type": "integer",
-                                              "minimum": 0,
-                                              "maximum": 10
-                                            }
-                                          ]
-                                        },
-                                        "outcome": {
-                                          "type": "string",
-                                          "minLength": 1
-                                        }
-                                      },
-                                      "required": ["score_range", "outcome"],
-                                      "additionalProperties": false
-                                    }
-                                  }
-                                },
-                                "additionalProperties": false
-                              }
-                            },
-                            "model": {
-                              "type": "string"
-                            },
-                            "target": {
-                              "type": "string"
-                            },
-                            "config": {
-                              "type": "object",
-                              "additionalProperties": {}
-                            },
-                            "max_steps": {
-                              "type": "integer",
-                              "minimum": 1,
-                              "maximum": 50
-                            },
-                            "temperature": {
-                              "type": "number",
-                              "minimum": 0,
-                              "maximum": 2
-                            },
-                            "preprocessors": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "type": {
-                                    "type": "string",
-                                    "minLength": 1
-                                  },
-                                  "command": {
-                                    "anyOf": [
-                                      {
-                                        "type": "string"
-                                      },
-                                      {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "string"
-                                        }
-                                      }
-                                    ]
-                                  }
-                                },
-                                "required": ["type", "command"],
-                                "additionalProperties": false
-                              }
-                            }
-                          },
-                          "required": ["type"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "include": {
-                              "type": "string",
-                              "minLength": 1
-                            }
-                          },
-                          "required": ["include"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "composite"
-                            },
-                            "assertions": {
-                              "type": "array",
-                              "items": {}
-                            },
-                            "evaluators": {
-                              "type": "array",
-                              "items": {}
-                            },
-                            "aggregator": {
-                              "anyOf": [
-                                {
-                                  "type": "object",
-                                  "properties": {
-                                    "type": {
-                                      "type": "string",
-                                      "const": "weighted_average"
-                                    },
-                                    "weights": {
-                                      "type": "object",
-                                      "additionalProperties": {
-                                        "type": "number"
-                                      }
-                                    }
-                                  },
-                                  "required": ["type"],
-                                  "additionalProperties": false
-                                },
-                                {
-                                  "type": "object",
-                                  "properties": {
-                                    "type": {
-                                      "type": "string",
-                                      "const": "threshold"
-                                    },
-                                    "threshold": {
-                                      "type": "number",
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  },
-                                  "required": ["type", "threshold"],
-                                  "additionalProperties": false
-                                },
-                                {
-                                  "type": "object",
-                                  "properties": {
-                                    "type": {
-                                      "type": "string",
-                                      "const": "code-grader"
-                                    },
-                                    "path": {
-                                      "type": "string"
-                                    },
-                                    "cwd": {
-                                      "type": "string"
-                                    }
-                                  },
-                                  "required": ["type", "path"],
-                                  "additionalProperties": false
-                                },
-                                {
-                                  "type": "object",
-                                  "properties": {
-                                    "type": {
-                                      "type": "string",
-                                      "const": "llm-grader"
-                                    },
-                                    "prompt": {
-                                      "type": "string"
-                                    },
-                                    "model": {
-                                      "type": "string"
-                                    }
-                                  },
-                                  "required": ["type"],
-                                  "additionalProperties": false
-                                }
-                              ]
-                            }
-                          },
-                          "required": ["type", "aggregator"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["tool-trajectory", "tool_trajectory"]
-                            },
-                            "mode": {
-                              "type": "string",
-                              "enum": ["any_order", "in_order", "exact", "subset", "superset"]
-                            },
-                            "minimums": {
-                              "type": "object",
-                              "additionalProperties": {
-                                "type": "integer",
-                                "minimum": 0
-                              }
-                            },
-                            "expected": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "tool": {
-                                    "type": "string"
-                                  },
-                                  "args": {
-                                    "anyOf": [
-                                      {
-                                        "type": "string",
-                                        "const": "any"
-                                      },
-                                      {
-                                        "type": "object",
-                                        "additionalProperties": {}
-                                      }
-                                    ]
-                                  },
-                                  "max_duration_ms": {
-                                    "type": "number",
-                                    "minimum": 0
-                                  },
-                                  "maxDurationMs": {
-                                    "type": "number",
-                                    "minimum": 0
-                                  },
-                                  "args_match": {
-                                    "anyOf": [
-                                      {
-                                        "type": "string",
-                                        "enum": ["exact", "ignore", "subset", "superset"]
-                                      },
-                                      {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "string"
-                                        }
-                                      }
-                                    ]
-                                  },
-                                  "argsMatch": {
-                                    "anyOf": [
-                                      {
-                                        "type": "string",
-                                        "enum": ["exact", "ignore", "subset", "superset"]
-                                      },
-                                      {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "string"
-                                        }
-                                      }
-                                    ]
-                                  }
-                                },
-                                "required": ["tool"],
-                                "additionalProperties": false
-                              }
-                            },
-                            "args_match": {
-                              "anyOf": [
-                                {
-                                  "type": "string",
-                                  "enum": ["exact", "ignore", "subset", "superset"]
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "string"
-                                  }
-                                }
-                              ]
-                            },
-                            "argsMatch": {
-                              "anyOf": [
-                                {
-                                  "type": "string",
-                                  "enum": ["exact", "ignore", "subset", "superset"]
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "string"
-                                  }
-                                }
-                              ]
-                            }
-                          },
-                          "required": ["type", "mode"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["field-accuracy", "field_accuracy"]
-                            },
-                            "fields": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "path": {
-                                    "type": "string"
-                                  },
-                                  "match": {
-                                    "type": "string",
-                                    "enum": ["exact", "numeric_tolerance", "date"]
-                                  },
-                                  "required": {
-                                    "type": "boolean"
-                                  },
-                                  "weight": {
-                                    "type": "number"
-                                  },
-                                  "tolerance": {
-                                    "type": "number",
-                                    "minimum": 0
-                                  },
-                                  "relative": {
-                                    "type": "boolean"
-                                  },
-                                  "formats": {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                },
-                                "required": ["path", "match"],
-                                "additionalProperties": false
-                              },
-                              "minItems": 1
-                            },
-                            "aggregation": {
-                              "type": "string",
-                              "enum": ["weighted_average", "all_or_nothing"]
-                            }
-                          },
-                          "required": ["type", "fields"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "latency"
-                            },
-                            "threshold": {
-                              "type": "number",
-                              "minimum": 0
-                            }
-                          },
-                          "required": ["type", "threshold"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "cost"
-                            },
-                            "budget": {
-                              "type": "number",
-                              "minimum": 0
-                            }
-                          },
-                          "required": ["type", "budget"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["token-usage", "token_usage"]
-                            },
-                            "max_total": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "max_input": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "max_output": {
-                              "type": "number",
-                              "minimum": 0
-                            }
-                          },
-                          "required": ["type"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["execution-metrics", "execution_metrics"]
-                            },
-                            "max_tool_calls": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "max_llm_calls": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "max_tokens": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "max_cost_usd": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "max_duration_ms": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "target_exploration_ratio": {
-                              "type": "number",
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "exploration_tolerance": {
-                              "type": "number",
-                              "minimum": 0
-                            }
-                          },
-                          "required": ["type"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "contains"
-                            },
-                            "value": {
-                              "type": "string"
-                            }
-                          },
-                          "required": ["type", "value"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "regex"
-                            },
-                            "value": {
-                              "type": "string"
-                            }
-                          },
-                          "required": ["type", "value"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["is-json", "is_json"]
-                            }
-                          },
-                          "required": ["type"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "equals"
-                            },
-                            "value": {
-                              "type": "string"
-                            }
-                          },
-                          "required": ["type", "value"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "rubrics"
-                            },
-                            "criteria": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "id": {
-                                    "type": "string"
-                                  },
-                                  "outcome": {
-                                    "type": "string"
-                                  },
-                                  "operator": {
-                                    "type": "string",
-                                    "enum": ["correctness", "contradiction"]
-                                  },
-                                  "weight": {
-                                    "type": "number"
-                                  },
-                                  "required": {
-                                    "type": "boolean"
-                                  },
-                                  "min_score": {
-                                    "type": "number",
-                                    "exclusiveMinimum": true,
-                                    "minimum": 0,
-                                    "maximum": 1
-                                  },
-                                  "score_ranges": {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "object",
-                                      "properties": {
-                                        "score_range": {
-                                          "type": "array",
-                                          "minItems": 2,
-                                          "maxItems": 2,
-                                          "items": [
-                                            {
-                                              "type": "integer",
-                                              "minimum": 0,
-                                              "maximum": 10
-                                            },
-                                            {
-                                              "type": "integer",
-                                              "minimum": 0,
-                                              "maximum": 10
-                                            }
-                                          ]
-                                        },
-                                        "outcome": {
-                                          "type": "string",
-                                          "minLength": 1
-                                        }
-                                      },
-                                      "required": ["score_range", "outcome"],
-                                      "additionalProperties": false
-                                    }
-                                  }
-                                },
-                                "additionalProperties": false
-                              },
-                              "minItems": 1
-                            }
-                          },
-                          "required": ["type", "criteria"],
-                          "additionalProperties": false
-                        }
-                      ]
-                    }
-                  },
-                  "evaluators": {
-                    "type": "array",
-                    "items": {
-                      "anyOf": [
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["code-grader", "code_grader"]
-                            },
-                            "command": {
-                              "anyOf": [
-                                {
-                                  "type": "string"
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "string"
-                                  }
-                                }
-                              ]
-                            },
-                            "script": {
-                              "anyOf": [
-                                {
-                                  "type": "string"
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "string"
-                                  }
-                                }
-                              ]
-                            },
-                            "cwd": {
-                              "type": "string"
-                            },
-                            "target": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "object",
-                                  "properties": {
-                                    "max_calls": {
-                                      "type": "number"
-                                    }
-                                  },
-                                  "additionalProperties": false
-                                }
-                              ]
-                            },
-                            "config": {
-                              "type": "object",
-                              "additionalProperties": {}
-                            },
-                            "preprocessors": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "type": {
-                                    "type": "string",
-                                    "minLength": 1
-                                  },
-                                  "command": {
-                                    "anyOf": [
-                                      {
-                                        "type": "string"
-                                      },
-                                      {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "string"
-                                        }
-                                      }
-                                    ]
-                                  }
-                                },
-                                "required": ["type", "command"],
-                                "additionalProperties": false
-                              }
-                            }
-                          },
-                          "required": ["type", "command"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["llm-grader", "llm_grader"]
-                            },
-                            "prompt": {
-                              "anyOf": [
-                                {
-                                  "type": "string"
-                                },
-                                {
-                                  "type": "object",
-                                  "properties": {
-                                    "command": {
-                                      "anyOf": [
-                                        {
-                                          "type": "string"
-                                        },
-                                        {
-                                          "type": "array",
-                                          "items": {
-                                            "type": "string"
-                                          }
-                                        }
-                                      ]
-                                    },
-                                    "script": {
-                                      "anyOf": [
-                                        {
-                                          "type": "string"
-                                        },
-                                        {
-                                          "type": "array",
-                                          "items": {
-                                            "type": "string"
-                                          }
-                                        }
-                                      ]
-                                    },
-                                    "config": {
-                                      "type": "object",
-                                      "additionalProperties": {}
-                                    }
-                                  },
-                                  "additionalProperties": false
-                                }
-                              ]
-                            },
-                            "rubrics": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "id": {
-                                    "type": "string"
-                                  },
-                                  "outcome": {
-                                    "type": "string"
-                                  },
-                                  "operator": {
-                                    "type": "string",
-                                    "enum": ["correctness", "contradiction"]
-                                  },
-                                  "weight": {
-                                    "type": "number"
-                                  },
-                                  "required": {
-                                    "type": "boolean"
-                                  },
-                                  "min_score": {
-                                    "type": "number",
-                                    "exclusiveMinimum": true,
-                                    "minimum": 0,
-                                    "maximum": 1
-                                  },
-                                  "score_ranges": {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "object",
-                                      "properties": {
-                                        "score_range": {
-                                          "type": "array",
-                                          "minItems": 2,
-                                          "maxItems": 2,
-                                          "items": [
-                                            {
-                                              "type": "integer",
-                                              "minimum": 0,
-                                              "maximum": 10
-                                            },
-                                            {
-                                              "type": "integer",
-                                              "minimum": 0,
-                                              "maximum": 10
-                                            }
-                                          ]
-                                        },
-                                        "outcome": {
-                                          "type": "string",
-                                          "minLength": 1
-                                        }
-                                      },
-                                      "required": ["score_range", "outcome"],
-                                      "additionalProperties": false
-                                    }
-                                  }
-                                },
-                                "additionalProperties": false
-                              }
-                            },
-                            "model": {
-                              "type": "string"
-                            },
-                            "target": {
-                              "type": "string"
-                            },
-                            "config": {
-                              "type": "object",
-                              "additionalProperties": {}
-                            },
-                            "max_steps": {
-                              "type": "integer",
-                              "minimum": 1,
-                              "maximum": 50
-                            },
-                            "temperature": {
-                              "type": "number",
-                              "minimum": 0,
-                              "maximum": 2
-                            },
-                            "preprocessors": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "type": {
-                                    "type": "string",
-                                    "minLength": 1
-                                  },
-                                  "command": {
-                                    "anyOf": [
-                                      {
-                                        "type": "string"
-                                      },
-                                      {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "string"
-                                        }
-                                      }
-                                    ]
-                                  }
-                                },
-                                "required": ["type", "command"],
-                                "additionalProperties": false
-                              }
-                            }
-                          },
-                          "required": ["type"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "include": {
-                              "type": "string",
-                              "minLength": 1
-                            }
-                          },
-                          "required": ["include"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "composite"
-                            },
-                            "assertions": {
-                              "type": "array",
-                              "items": {}
-                            },
-                            "evaluators": {
-                              "type": "array",
-                              "items": {}
-                            },
-                            "aggregator": {
-                              "anyOf": [
-                                {
-                                  "type": "object",
-                                  "properties": {
-                                    "type": {
-                                      "type": "string",
-                                      "const": "weighted_average"
-                                    },
-                                    "weights": {
-                                      "type": "object",
-                                      "additionalProperties": {
-                                        "type": "number"
-                                      }
-                                    }
-                                  },
-                                  "required": ["type"],
-                                  "additionalProperties": false
-                                },
-                                {
-                                  "type": "object",
-                                  "properties": {
-                                    "type": {
-                                      "type": "string",
-                                      "const": "threshold"
-                                    },
-                                    "threshold": {
-                                      "type": "number",
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  },
-                                  "required": ["type", "threshold"],
-                                  "additionalProperties": false
-                                },
-                                {
-                                  "type": "object",
-                                  "properties": {
-                                    "type": {
-                                      "type": "string",
-                                      "const": "code-grader"
-                                    },
-                                    "path": {
-                                      "type": "string"
-                                    },
-                                    "cwd": {
-                                      "type": "string"
-                                    }
-                                  },
-                                  "required": ["type", "path"],
-                                  "additionalProperties": false
-                                },
-                                {
-                                  "type": "object",
-                                  "properties": {
-                                    "type": {
-                                      "type": "string",
-                                      "const": "llm-grader"
-                                    },
-                                    "prompt": {
-                                      "type": "string"
-                                    },
-                                    "model": {
-                                      "type": "string"
-                                    }
-                                  },
-                                  "required": ["type"],
-                                  "additionalProperties": false
-                                }
-                              ]
-                            }
-                          },
-                          "required": ["type", "aggregator"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["tool-trajectory", "tool_trajectory"]
-                            },
-                            "mode": {
-                              "type": "string",
-                              "enum": ["any_order", "in_order", "exact", "subset", "superset"]
-                            },
-                            "minimums": {
-                              "type": "object",
-                              "additionalProperties": {
-                                "type": "integer",
-                                "minimum": 0
-                              }
-                            },
-                            "expected": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "tool": {
-                                    "type": "string"
-                                  },
-                                  "args": {
-                                    "anyOf": [
-                                      {
-                                        "type": "string",
-                                        "const": "any"
-                                      },
-                                      {
-                                        "type": "object",
-                                        "additionalProperties": {}
-                                      }
-                                    ]
-                                  },
-                                  "max_duration_ms": {
-                                    "type": "number",
-                                    "minimum": 0
-                                  },
-                                  "maxDurationMs": {
-                                    "type": "number",
-                                    "minimum": 0
-                                  },
-                                  "args_match": {
-                                    "anyOf": [
-                                      {
-                                        "type": "string",
-                                        "enum": ["exact", "ignore", "subset", "superset"]
-                                      },
-                                      {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "string"
-                                        }
-                                      }
-                                    ]
-                                  },
-                                  "argsMatch": {
-                                    "anyOf": [
-                                      {
-                                        "type": "string",
-                                        "enum": ["exact", "ignore", "subset", "superset"]
-                                      },
-                                      {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "string"
-                                        }
-                                      }
-                                    ]
-                                  }
-                                },
-                                "required": ["tool"],
-                                "additionalProperties": false
-                              }
-                            },
-                            "args_match": {
-                              "anyOf": [
-                                {
-                                  "type": "string",
-                                  "enum": ["exact", "ignore", "subset", "superset"]
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "string"
-                                  }
-                                }
-                              ]
-                            },
-                            "argsMatch": {
-                              "anyOf": [
-                                {
-                                  "type": "string",
-                                  "enum": ["exact", "ignore", "subset", "superset"]
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "string"
-                                  }
-                                }
-                              ]
-                            }
-                          },
-                          "required": ["type", "mode"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["field-accuracy", "field_accuracy"]
-                            },
-                            "fields": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "path": {
-                                    "type": "string"
-                                  },
-                                  "match": {
-                                    "type": "string",
-                                    "enum": ["exact", "numeric_tolerance", "date"]
-                                  },
-                                  "required": {
-                                    "type": "boolean"
-                                  },
-                                  "weight": {
-                                    "type": "number"
-                                  },
-                                  "tolerance": {
-                                    "type": "number",
-                                    "minimum": 0
-                                  },
-                                  "relative": {
-                                    "type": "boolean"
-                                  },
-                                  "formats": {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                },
-                                "required": ["path", "match"],
-                                "additionalProperties": false
-                              },
-                              "minItems": 1
-                            },
-                            "aggregation": {
-                              "type": "string",
-                              "enum": ["weighted_average", "all_or_nothing"]
-                            }
-                          },
-                          "required": ["type", "fields"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "latency"
-                            },
-                            "threshold": {
-                              "type": "number",
-                              "minimum": 0
-                            }
-                          },
-                          "required": ["type", "threshold"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "cost"
-                            },
-                            "budget": {
-                              "type": "number",
-                              "minimum": 0
-                            }
-                          },
-                          "required": ["type", "budget"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["token-usage", "token_usage"]
-                            },
-                            "max_total": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "max_input": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "max_output": {
-                              "type": "number",
-                              "minimum": 0
-                            }
-                          },
-                          "required": ["type"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["execution-metrics", "execution_metrics"]
-                            },
-                            "max_tool_calls": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "max_llm_calls": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "max_tokens": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "max_cost_usd": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "max_duration_ms": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "target_exploration_ratio": {
-                              "type": "number",
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "exploration_tolerance": {
-                              "type": "number",
-                              "minimum": 0
-                            }
-                          },
-                          "required": ["type"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "contains"
-                            },
-                            "value": {
-                              "type": "string"
-                            }
-                          },
-                          "required": ["type", "value"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "regex"
-                            },
-                            "value": {
-                              "type": "string"
-                            }
-                          },
-                          "required": ["type", "value"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "enum": ["is-json", "is_json"]
-                            }
-                          },
-                          "required": ["type"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "equals"
-                            },
-                            "value": {
-                              "type": "string"
-                            }
-                          },
-                          "required": ["type", "value"],
-                          "additionalProperties": false
-                        },
-                        {
-                          "type": "object",
-                          "properties": {
-                            "name": {
-                              "type": "string"
-                            },
-                            "weight": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "required": {
-                              "anyOf": [
-                                {
-                                  "type": "boolean"
-                                },
-                                {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                }
-                              ]
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "negate": {
-                              "type": "boolean"
-                            },
-                            "type": {
-                              "type": "string",
-                              "const": "rubrics"
-                            },
-                            "criteria": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "id": {
-                                    "type": "string"
-                                  },
-                                  "outcome": {
-                                    "type": "string"
-                                  },
-                                  "operator": {
-                                    "type": "string",
-                                    "enum": ["correctness", "contradiction"]
-                                  },
-                                  "weight": {
-                                    "type": "number"
-                                  },
-                                  "required": {
-                                    "type": "boolean"
-                                  },
-                                  "min_score": {
-                                    "type": "number",
-                                    "exclusiveMinimum": true,
-                                    "minimum": 0,
-                                    "maximum": 1
-                                  },
-                                  "score_ranges": {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "object",
-                                      "properties": {
-                                        "score_range": {
-                                          "type": "array",
-                                          "minItems": 2,
-                                          "maxItems": 2,
-                                          "items": [
-                                            {
-                                              "type": "integer",
-                                              "minimum": 0,
-                                              "maximum": 10
-                                            },
-                                            {
-                                              "type": "integer",
-                                              "minimum": 0,
-                                              "maximum": 10
-                                            }
-                                          ]
-                                        },
-                                        "outcome": {
-                                          "type": "string",
-                                          "minLength": 1
-                                        }
-                                      },
-                                      "required": ["score_range", "outcome"],
-                                      "additionalProperties": false
-                                    }
-                                  }
-                                },
-                                "additionalProperties": false
-                              },
-                              "minItems": 1
-                            }
-                          },
-                          "required": ["type", "criteria"],
-                          "additionalProperties": false
-                        }
-                      ]
-                    }
-                  },
-                  "execution": {
-                    "type": "object",
-                    "properties": {
-                      "target": {
-                        "type": "string"
-                      },
-                      "targets": {
-                        "type": "array",
-                        "items": {
-                          "anyOf": [
-                            {
-                              "type": "string"
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string",
-                                  "minLength": 1
-                                },
-                                "use_target": {
-                                  "type": "string"
-                                },
-                                "hooks": {
-                                  "type": "object",
-                                  "properties": {
-                                    "before_all": {
-                                      "type": "object",
-                                      "properties": {
-                                        "command": {
-                                          "anyOf": [
-                                            {
-                                              "type": "string"
-                                            },
-                                            {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          ]
-                                        },
-                                        "script": {
-                                          "anyOf": [
-                                            {
-                                              "type": "string"
-                                            },
-                                            {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          ]
-                                        },
-                                        "timeout_ms": {
-                                          "type": "number"
-                                        },
-                                        "timeoutMs": {
-                                          "type": "number"
-                                        },
-                                        "cwd": {
-                                          "type": "string"
-                                        },
-                                        "reset": {
-                                          "type": "string",
-                                          "enum": ["none", "fast", "strict"]
-                                        }
-                                      },
-                                      "additionalProperties": false
-                                    },
-                                    "before_each": {
-                                      "type": "object",
-                                      "properties": {
-                                        "command": {
-                                          "anyOf": [
-                                            {
-                                              "type": "string"
-                                            },
-                                            {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          ]
-                                        },
-                                        "script": {
-                                          "anyOf": [
-                                            {
-                                              "type": "string"
-                                            },
-                                            {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          ]
-                                        },
-                                        "timeout_ms": {
-                                          "type": "number"
-                                        },
-                                        "timeoutMs": {
-                                          "type": "number"
-                                        },
-                                        "cwd": {
-                                          "type": "string"
-                                        },
-                                        "reset": {
-                                          "type": "string",
-                                          "enum": ["none", "fast", "strict"]
-                                        }
-                                      },
-                                      "additionalProperties": false
-                                    },
-                                    "after_each": {
-                                      "type": "object",
-                                      "properties": {
-                                        "command": {
-                                          "anyOf": [
-                                            {
-                                              "type": "string"
-                                            },
-                                            {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          ]
-                                        },
-                                        "script": {
-                                          "anyOf": [
-                                            {
-                                              "type": "string"
-                                            },
-                                            {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          ]
-                                        },
-                                        "timeout_ms": {
-                                          "type": "number"
-                                        },
-                                        "timeoutMs": {
-                                          "type": "number"
-                                        },
-                                        "cwd": {
-                                          "type": "string"
-                                        },
-                                        "reset": {
-                                          "type": "string",
-                                          "enum": ["none", "fast", "strict"]
-                                        }
-                                      },
-                                      "additionalProperties": false
-                                    },
-                                    "after_all": {
-                                      "type": "object",
-                                      "properties": {
-                                        "command": {
-                                          "anyOf": [
-                                            {
-                                              "type": "string"
-                                            },
-                                            {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          ]
-                                        },
-                                        "script": {
-                                          "anyOf": [
-                                            {
-                                              "type": "string"
-                                            },
-                                            {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          ]
-                                        },
-                                        "timeout_ms": {
-                                          "type": "number"
-                                        },
-                                        "timeoutMs": {
-                                          "type": "number"
-                                        },
-                                        "cwd": {
-                                          "type": "string"
-                                        },
-                                        "reset": {
-                                          "type": "string",
-                                          "enum": ["none", "fast", "strict"]
-                                        }
-                                      },
-                                      "additionalProperties": false
-                                    }
-                                  },
-                                  "additionalProperties": false
-                                }
-                              },
-                              "required": ["name"],
-                              "additionalProperties": false
-                            }
-                          ]
-                        }
-                      },
-                      "workers": {
-                        "type": "integer",
-                        "minimum": 1,
-                        "maximum": 50
-                      },
-                      "assertions": {
-                        "type": "array",
-                        "items": {
-                          "anyOf": [
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["code-grader", "code_grader"]
-                                },
-                                "command": {
-                                  "anyOf": [
-                                    {
-                                      "type": "string"
-                                    },
-                                    {
-                                      "type": "array",
-                                      "items": {
-                                        "type": "string"
-                                      }
-                                    }
-                                  ]
-                                },
-                                "script": {
-                                  "anyOf": [
-                                    {
-                                      "type": "string"
-                                    },
-                                    {
-                                      "type": "array",
-                                      "items": {
-                                        "type": "string"
-                                      }
-                                    }
-                                  ]
-                                },
-                                "cwd": {
-                                  "type": "string"
-                                },
-                                "target": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "object",
-                                      "properties": {
-                                        "max_calls": {
-                                          "type": "number"
-                                        }
-                                      },
-                                      "additionalProperties": false
-                                    }
-                                  ]
-                                },
-                                "config": {
-                                  "type": "object",
-                                  "additionalProperties": {}
-                                },
-                                "preprocessors": {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "type": {
-                                        "type": "string",
-                                        "minLength": 1
-                                      },
-                                      "command": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string"
-                                          },
-                                          {
-                                            "type": "array",
-                                            "items": {
-                                              "type": "string"
-                                            }
-                                          }
-                                        ]
-                                      }
-                                    },
-                                    "required": ["type", "command"],
-                                    "additionalProperties": false
-                                  }
-                                }
-                              },
-                              "required": ["type", "command"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["llm-grader", "llm_grader"]
-                                },
-                                "prompt": {
-                                  "anyOf": [
-                                    {
-                                      "type": "string"
-                                    },
-                                    {
-                                      "type": "object",
-                                      "properties": {
-                                        "command": {
-                                          "anyOf": [
-                                            {
-                                              "type": "string"
-                                            },
-                                            {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          ]
-                                        },
-                                        "script": {
-                                          "anyOf": [
-                                            {
-                                              "type": "string"
-                                            },
-                                            {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          ]
-                                        },
-                                        "config": {
-                                          "type": "object",
-                                          "additionalProperties": {}
-                                        }
-                                      },
-                                      "additionalProperties": false
-                                    }
-                                  ]
-                                },
-                                "rubrics": {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "id": {
-                                        "type": "string"
-                                      },
-                                      "outcome": {
-                                        "type": "string"
-                                      },
-                                      "operator": {
-                                        "type": "string",
-                                        "enum": ["correctness", "contradiction"]
-                                      },
-                                      "weight": {
-                                        "type": "number"
-                                      },
-                                      "required": {
-                                        "type": "boolean"
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "score_ranges": {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "object",
-                                          "properties": {
-                                            "score_range": {
-                                              "type": "array",
-                                              "minItems": 2,
-                                              "maxItems": 2,
-                                              "items": [
-                                                {
-                                                  "type": "integer",
-                                                  "minimum": 0,
-                                                  "maximum": 10
-                                                },
-                                                {
-                                                  "type": "integer",
-                                                  "minimum": 0,
-                                                  "maximum": 10
-                                                }
-                                              ]
-                                            },
-                                            "outcome": {
-                                              "type": "string",
-                                              "minLength": 1
-                                            }
-                                          },
-                                          "required": ["score_range", "outcome"],
-                                          "additionalProperties": false
-                                        }
-                                      }
-                                    },
-                                    "additionalProperties": false
-                                  }
-                                },
-                                "model": {
-                                  "type": "string"
-                                },
-                                "target": {
-                                  "type": "string"
-                                },
-                                "config": {
-                                  "type": "object",
-                                  "additionalProperties": {}
-                                },
-                                "max_steps": {
-                                  "type": "integer",
-                                  "minimum": 1,
-                                  "maximum": 50
-                                },
-                                "temperature": {
-                                  "type": "number",
-                                  "minimum": 0,
-                                  "maximum": 2
-                                },
-                                "preprocessors": {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "type": {
-                                        "type": "string",
-                                        "minLength": 1
-                                      },
-                                      "command": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string"
-                                          },
-                                          {
-                                            "type": "array",
-                                            "items": {
-                                              "type": "string"
-                                            }
-                                          }
-                                        ]
-                                      }
-                                    },
-                                    "required": ["type", "command"],
-                                    "additionalProperties": false
-                                  }
-                                }
-                              },
-                              "required": ["type"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "include": {
-                                  "type": "string",
-                                  "minLength": 1
-                                }
-                              },
-                              "required": ["include"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "composite"
-                                },
-                                "assertions": {
-                                  "type": "array",
-                                  "items": {}
-                                },
-                                "evaluators": {
-                                  "type": "array",
-                                  "items": {}
-                                },
-                                "aggregator": {
-                                  "anyOf": [
-                                    {
-                                      "type": "object",
-                                      "properties": {
-                                        "type": {
-                                          "type": "string",
-                                          "const": "weighted_average"
-                                        },
-                                        "weights": {
-                                          "type": "object",
-                                          "additionalProperties": {
-                                            "type": "number"
-                                          }
-                                        }
-                                      },
-                                      "required": ["type"],
-                                      "additionalProperties": false
-                                    },
-                                    {
-                                      "type": "object",
-                                      "properties": {
-                                        "type": {
-                                          "type": "string",
-                                          "const": "threshold"
-                                        },
-                                        "threshold": {
-                                          "type": "number",
-                                          "minimum": 0,
-                                          "maximum": 1
-                                        }
-                                      },
-                                      "required": ["type", "threshold"],
-                                      "additionalProperties": false
-                                    },
-                                    {
-                                      "type": "object",
-                                      "properties": {
-                                        "type": {
-                                          "type": "string",
-                                          "const": "code-grader"
-                                        },
-                                        "path": {
-                                          "type": "string"
-                                        },
-                                        "cwd": {
-                                          "type": "string"
-                                        }
-                                      },
-                                      "required": ["type", "path"],
-                                      "additionalProperties": false
-                                    },
-                                    {
-                                      "type": "object",
-                                      "properties": {
-                                        "type": {
-                                          "type": "string",
-                                          "const": "llm-grader"
-                                        },
-                                        "prompt": {
-                                          "type": "string"
-                                        },
-                                        "model": {
-                                          "type": "string"
-                                        }
-                                      },
-                                      "required": ["type"],
-                                      "additionalProperties": false
-                                    }
-                                  ]
-                                }
-                              },
-                              "required": ["type", "aggregator"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["tool-trajectory", "tool_trajectory"]
-                                },
-                                "mode": {
-                                  "type": "string",
-                                  "enum": ["any_order", "in_order", "exact", "subset", "superset"]
-                                },
-                                "minimums": {
-                                  "type": "object",
-                                  "additionalProperties": {
-                                    "type": "integer",
-                                    "minimum": 0
-                                  }
-                                },
-                                "expected": {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "tool": {
-                                        "type": "string"
-                                      },
-                                      "args": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string",
-                                            "const": "any"
-                                          },
-                                          {
-                                            "type": "object",
-                                            "additionalProperties": {}
-                                          }
-                                        ]
-                                      },
-                                      "max_duration_ms": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "maxDurationMs": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "args_match": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string",
-                                            "enum": ["exact", "ignore", "subset", "superset"]
-                                          },
-                                          {
-                                            "type": "array",
-                                            "items": {
-                                              "type": "string"
-                                            }
-                                          }
-                                        ]
-                                      },
-                                      "argsMatch": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string",
-                                            "enum": ["exact", "ignore", "subset", "superset"]
-                                          },
-                                          {
-                                            "type": "array",
-                                            "items": {
-                                              "type": "string"
-                                            }
-                                          }
-                                        ]
-                                      }
-                                    },
-                                    "required": ["tool"],
-                                    "additionalProperties": false
-                                  }
-                                },
-                                "args_match": {
-                                  "anyOf": [
-                                    {
-                                      "type": "string",
-                                      "enum": ["exact", "ignore", "subset", "superset"]
-                                    },
-                                    {
-                                      "type": "array",
-                                      "items": {
-                                        "type": "string"
-                                      }
-                                    }
-                                  ]
-                                },
-                                "argsMatch": {
-                                  "anyOf": [
-                                    {
-                                      "type": "string",
-                                      "enum": ["exact", "ignore", "subset", "superset"]
-                                    },
-                                    {
-                                      "type": "array",
-                                      "items": {
-                                        "type": "string"
-                                      }
-                                    }
-                                  ]
-                                }
-                              },
-                              "required": ["type", "mode"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["field-accuracy", "field_accuracy"]
-                                },
-                                "fields": {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "path": {
-                                        "type": "string"
-                                      },
-                                      "match": {
-                                        "type": "string",
-                                        "enum": ["exact", "numeric_tolerance", "date"]
-                                      },
-                                      "required": {
-                                        "type": "boolean"
-                                      },
-                                      "weight": {
-                                        "type": "number"
-                                      },
-                                      "tolerance": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "relative": {
-                                        "type": "boolean"
-                                      },
-                                      "formats": {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "string"
-                                        }
-                                      }
-                                    },
-                                    "required": ["path", "match"],
-                                    "additionalProperties": false
-                                  },
-                                  "minItems": 1
-                                },
-                                "aggregation": {
-                                  "type": "string",
-                                  "enum": ["weighted_average", "all_or_nothing"]
-                                }
-                              },
-                              "required": ["type", "fields"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "latency"
-                                },
-                                "threshold": {
-                                  "type": "number",
-                                  "minimum": 0
-                                }
-                              },
-                              "required": ["type", "threshold"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "cost"
-                                },
-                                "budget": {
-                                  "type": "number",
-                                  "minimum": 0
-                                }
-                              },
-                              "required": ["type", "budget"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["token-usage", "token_usage"]
-                                },
-                                "max_total": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "max_input": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "max_output": {
-                                  "type": "number",
-                                  "minimum": 0
-                                }
-                              },
-                              "required": ["type"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["execution-metrics", "execution_metrics"]
-                                },
-                                "max_tool_calls": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "max_llm_calls": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "max_tokens": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "max_cost_usd": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "max_duration_ms": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "target_exploration_ratio": {
-                                  "type": "number",
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "exploration_tolerance": {
-                                  "type": "number",
-                                  "minimum": 0
-                                }
-                              },
-                              "required": ["type"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "contains"
-                                },
-                                "value": {
-                                  "type": "string"
-                                }
-                              },
-                              "required": ["type", "value"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "regex"
-                                },
-                                "value": {
-                                  "type": "string"
-                                }
-                              },
-                              "required": ["type", "value"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["is-json", "is_json"]
-                                }
-                              },
-                              "required": ["type"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "equals"
-                                },
-                                "value": {
-                                  "type": "string"
-                                }
-                              },
-                              "required": ["type", "value"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "rubrics"
-                                },
-                                "criteria": {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "id": {
-                                        "type": "string"
-                                      },
-                                      "outcome": {
-                                        "type": "string"
-                                      },
-                                      "operator": {
-                                        "type": "string",
-                                        "enum": ["correctness", "contradiction"]
-                                      },
-                                      "weight": {
-                                        "type": "number"
-                                      },
-                                      "required": {
-                                        "type": "boolean"
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "score_ranges": {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "object",
-                                          "properties": {
-                                            "score_range": {
-                                              "type": "array",
-                                              "minItems": 2,
-                                              "maxItems": 2,
-                                              "items": [
-                                                {
-                                                  "type": "integer",
-                                                  "minimum": 0,
-                                                  "maximum": 10
-                                                },
-                                                {
-                                                  "type": "integer",
-                                                  "minimum": 0,
-                                                  "maximum": 10
-                                                }
-                                              ]
-                                            },
-                                            "outcome": {
-                                              "type": "string",
-                                              "minLength": 1
-                                            }
-                                          },
-                                          "required": ["score_range", "outcome"],
-                                          "additionalProperties": false
-                                        }
-                                      }
-                                    },
-                                    "additionalProperties": false
-                                  },
-                                  "minItems": 1
-                                }
-                              },
-                              "required": ["type", "criteria"],
-                              "additionalProperties": false
-                            }
-                          ]
-                        }
-                      },
-                      "evaluators": {
-                        "type": "array",
-                        "items": {
-                          "anyOf": [
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["code-grader", "code_grader"]
-                                },
-                                "command": {
-                                  "anyOf": [
-                                    {
-                                      "type": "string"
-                                    },
-                                    {
-                                      "type": "array",
-                                      "items": {
-                                        "type": "string"
-                                      }
-                                    }
-                                  ]
-                                },
-                                "script": {
-                                  "anyOf": [
-                                    {
-                                      "type": "string"
-                                    },
-                                    {
-                                      "type": "array",
-                                      "items": {
-                                        "type": "string"
-                                      }
-                                    }
-                                  ]
-                                },
-                                "cwd": {
-                                  "type": "string"
-                                },
-                                "target": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "object",
-                                      "properties": {
-                                        "max_calls": {
-                                          "type": "number"
-                                        }
-                                      },
-                                      "additionalProperties": false
-                                    }
-                                  ]
-                                },
-                                "config": {
-                                  "type": "object",
-                                  "additionalProperties": {}
-                                },
-                                "preprocessors": {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "type": {
-                                        "type": "string",
-                                        "minLength": 1
-                                      },
-                                      "command": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string"
-                                          },
-                                          {
-                                            "type": "array",
-                                            "items": {
-                                              "type": "string"
-                                            }
-                                          }
-                                        ]
-                                      }
-                                    },
-                                    "required": ["type", "command"],
-                                    "additionalProperties": false
-                                  }
-                                }
-                              },
-                              "required": ["type", "command"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["llm-grader", "llm_grader"]
-                                },
-                                "prompt": {
-                                  "anyOf": [
-                                    {
-                                      "type": "string"
-                                    },
-                                    {
-                                      "type": "object",
-                                      "properties": {
-                                        "command": {
-                                          "anyOf": [
-                                            {
-                                              "type": "string"
-                                            },
-                                            {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          ]
-                                        },
-                                        "script": {
-                                          "anyOf": [
-                                            {
-                                              "type": "string"
-                                            },
-                                            {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          ]
-                                        },
-                                        "config": {
-                                          "type": "object",
-                                          "additionalProperties": {}
-                                        }
-                                      },
-                                      "additionalProperties": false
-                                    }
-                                  ]
-                                },
-                                "rubrics": {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "id": {
-                                        "type": "string"
-                                      },
-                                      "outcome": {
-                                        "type": "string"
-                                      },
-                                      "operator": {
-                                        "type": "string",
-                                        "enum": ["correctness", "contradiction"]
-                                      },
-                                      "weight": {
-                                        "type": "number"
-                                      },
-                                      "required": {
-                                        "type": "boolean"
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "score_ranges": {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "object",
-                                          "properties": {
-                                            "score_range": {
-                                              "type": "array",
-                                              "minItems": 2,
-                                              "maxItems": 2,
-                                              "items": [
-                                                {
-                                                  "type": "integer",
-                                                  "minimum": 0,
-                                                  "maximum": 10
-                                                },
-                                                {
-                                                  "type": "integer",
-                                                  "minimum": 0,
-                                                  "maximum": 10
-                                                }
-                                              ]
-                                            },
-                                            "outcome": {
-                                              "type": "string",
-                                              "minLength": 1
-                                            }
-                                          },
-                                          "required": ["score_range", "outcome"],
-                                          "additionalProperties": false
-                                        }
-                                      }
-                                    },
-                                    "additionalProperties": false
-                                  }
-                                },
-                                "model": {
-                                  "type": "string"
-                                },
-                                "target": {
-                                  "type": "string"
-                                },
-                                "config": {
-                                  "type": "object",
-                                  "additionalProperties": {}
-                                },
-                                "max_steps": {
-                                  "type": "integer",
-                                  "minimum": 1,
-                                  "maximum": 50
-                                },
-                                "temperature": {
-                                  "type": "number",
-                                  "minimum": 0,
-                                  "maximum": 2
-                                },
-                                "preprocessors": {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "type": {
-                                        "type": "string",
-                                        "minLength": 1
-                                      },
-                                      "command": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string"
-                                          },
-                                          {
-                                            "type": "array",
-                                            "items": {
-                                              "type": "string"
-                                            }
-                                          }
-                                        ]
-                                      }
-                                    },
-                                    "required": ["type", "command"],
-                                    "additionalProperties": false
-                                  }
-                                }
-                              },
-                              "required": ["type"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "include": {
-                                  "type": "string",
-                                  "minLength": 1
-                                }
-                              },
-                              "required": ["include"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "composite"
-                                },
-                                "assertions": {
-                                  "type": "array",
-                                  "items": {}
-                                },
-                                "evaluators": {
-                                  "type": "array",
-                                  "items": {}
-                                },
-                                "aggregator": {
-                                  "anyOf": [
-                                    {
-                                      "type": "object",
-                                      "properties": {
-                                        "type": {
-                                          "type": "string",
-                                          "const": "weighted_average"
-                                        },
-                                        "weights": {
-                                          "type": "object",
-                                          "additionalProperties": {
-                                            "type": "number"
-                                          }
-                                        }
-                                      },
-                                      "required": ["type"],
-                                      "additionalProperties": false
-                                    },
-                                    {
-                                      "type": "object",
-                                      "properties": {
-                                        "type": {
-                                          "type": "string",
-                                          "const": "threshold"
-                                        },
-                                        "threshold": {
-                                          "type": "number",
-                                          "minimum": 0,
-                                          "maximum": 1
-                                        }
-                                      },
-                                      "required": ["type", "threshold"],
-                                      "additionalProperties": false
-                                    },
-                                    {
-                                      "type": "object",
-                                      "properties": {
-                                        "type": {
-                                          "type": "string",
-                                          "const": "code-grader"
-                                        },
-                                        "path": {
-                                          "type": "string"
-                                        },
-                                        "cwd": {
-                                          "type": "string"
-                                        }
-                                      },
-                                      "required": ["type", "path"],
-                                      "additionalProperties": false
-                                    },
-                                    {
-                                      "type": "object",
-                                      "properties": {
-                                        "type": {
-                                          "type": "string",
-                                          "const": "llm-grader"
-                                        },
-                                        "prompt": {
-                                          "type": "string"
-                                        },
-                                        "model": {
-                                          "type": "string"
-                                        }
-                                      },
-                                      "required": ["type"],
-                                      "additionalProperties": false
-                                    }
-                                  ]
-                                }
-                              },
-                              "required": ["type", "aggregator"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["tool-trajectory", "tool_trajectory"]
-                                },
-                                "mode": {
-                                  "type": "string",
-                                  "enum": ["any_order", "in_order", "exact", "subset", "superset"]
-                                },
-                                "minimums": {
-                                  "type": "object",
-                                  "additionalProperties": {
-                                    "type": "integer",
-                                    "minimum": 0
-                                  }
-                                },
-                                "expected": {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "tool": {
-                                        "type": "string"
-                                      },
-                                      "args": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string",
-                                            "const": "any"
-                                          },
-                                          {
-                                            "type": "object",
-                                            "additionalProperties": {}
-                                          }
-                                        ]
-                                      },
-                                      "max_duration_ms": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "maxDurationMs": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "args_match": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string",
-                                            "enum": ["exact", "ignore", "subset", "superset"]
-                                          },
-                                          {
-                                            "type": "array",
-                                            "items": {
-                                              "type": "string"
-                                            }
-                                          }
-                                        ]
-                                      },
-                                      "argsMatch": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string",
-                                            "enum": ["exact", "ignore", "subset", "superset"]
-                                          },
-                                          {
-                                            "type": "array",
-                                            "items": {
-                                              "type": "string"
-                                            }
-                                          }
-                                        ]
-                                      }
-                                    },
-                                    "required": ["tool"],
-                                    "additionalProperties": false
-                                  }
-                                },
-                                "args_match": {
-                                  "anyOf": [
-                                    {
-                                      "type": "string",
-                                      "enum": ["exact", "ignore", "subset", "superset"]
-                                    },
-                                    {
-                                      "type": "array",
-                                      "items": {
-                                        "type": "string"
-                                      }
-                                    }
-                                  ]
-                                },
-                                "argsMatch": {
-                                  "anyOf": [
-                                    {
-                                      "type": "string",
-                                      "enum": ["exact", "ignore", "subset", "superset"]
-                                    },
-                                    {
-                                      "type": "array",
-                                      "items": {
-                                        "type": "string"
-                                      }
-                                    }
-                                  ]
-                                }
-                              },
-                              "required": ["type", "mode"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["field-accuracy", "field_accuracy"]
-                                },
-                                "fields": {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "path": {
-                                        "type": "string"
-                                      },
-                                      "match": {
-                                        "type": "string",
-                                        "enum": ["exact", "numeric_tolerance", "date"]
-                                      },
-                                      "required": {
-                                        "type": "boolean"
-                                      },
-                                      "weight": {
-                                        "type": "number"
-                                      },
-                                      "tolerance": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "relative": {
-                                        "type": "boolean"
-                                      },
-                                      "formats": {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "string"
-                                        }
-                                      }
-                                    },
-                                    "required": ["path", "match"],
-                                    "additionalProperties": false
-                                  },
-                                  "minItems": 1
-                                },
-                                "aggregation": {
-                                  "type": "string",
-                                  "enum": ["weighted_average", "all_or_nothing"]
-                                }
-                              },
-                              "required": ["type", "fields"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "latency"
-                                },
-                                "threshold": {
-                                  "type": "number",
-                                  "minimum": 0
-                                }
-                              },
-                              "required": ["type", "threshold"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "cost"
-                                },
-                                "budget": {
-                                  "type": "number",
-                                  "minimum": 0
-                                }
-                              },
-                              "required": ["type", "budget"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["token-usage", "token_usage"]
-                                },
-                                "max_total": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "max_input": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "max_output": {
-                                  "type": "number",
-                                  "minimum": 0
-                                }
-                              },
-                              "required": ["type"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["execution-metrics", "execution_metrics"]
-                                },
-                                "max_tool_calls": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "max_llm_calls": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "max_tokens": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "max_cost_usd": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "max_duration_ms": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "target_exploration_ratio": {
-                                  "type": "number",
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "exploration_tolerance": {
-                                  "type": "number",
-                                  "minimum": 0
-                                }
-                              },
-                              "required": ["type"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "contains"
-                                },
-                                "value": {
-                                  "type": "string"
-                                }
-                              },
-                              "required": ["type", "value"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "regex"
-                                },
-                                "value": {
-                                  "type": "string"
-                                }
-                              },
-                              "required": ["type", "value"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "enum": ["is-json", "is_json"]
-                                }
-                              },
-                              "required": ["type"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "equals"
-                                },
-                                "value": {
-                                  "type": "string"
-                                }
-                              },
-                              "required": ["type", "value"],
-                              "additionalProperties": false
-                            },
-                            {
-                              "type": "object",
-                              "properties": {
-                                "name": {
-                                  "type": "string"
-                                },
-                                "weight": {
-                                  "type": "number",
-                                  "minimum": 0
-                                },
-                                "required": {
-                                  "anyOf": [
-                                    {
-                                      "type": "boolean"
-                                    },
-                                    {
-                                      "type": "number",
-                                      "exclusiveMinimum": true,
-                                      "minimum": 0,
-                                      "maximum": 1
-                                    }
-                                  ]
-                                },
-                                "min_score": {
-                                  "type": "number",
-                                  "exclusiveMinimum": true,
-                                  "minimum": 0,
-                                  "maximum": 1
-                                },
-                                "negate": {
-                                  "type": "boolean"
-                                },
-                                "type": {
-                                  "type": "string",
-                                  "const": "rubrics"
-                                },
-                                "criteria": {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "id": {
-                                        "type": "string"
-                                      },
-                                      "outcome": {
-                                        "type": "string"
-                                      },
-                                      "operator": {
-                                        "type": "string",
-                                        "enum": ["correctness", "contradiction"]
-                                      },
-                                      "weight": {
-                                        "type": "number"
-                                      },
-                                      "required": {
-                                        "type": "boolean"
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "score_ranges": {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "object",
-                                          "properties": {
-                                            "score_range": {
-                                              "type": "array",
-                                              "minItems": 2,
-                                              "maxItems": 2,
-                                              "items": [
-                                                {
-                                                  "type": "integer",
-                                                  "minimum": 0,
-                                                  "maximum": 10
-                                                },
-                                                {
-                                                  "type": "integer",
-                                                  "minimum": 0,
-                                                  "maximum": 10
-                                                }
-                                              ]
-                                            },
-                                            "outcome": {
-                                              "type": "string",
-                                              "minLength": 1
-                                            }
-                                          },
-                                          "required": ["score_range", "outcome"],
-                                          "additionalProperties": false
-                                        }
-                                      }
-                                    },
-                                    "additionalProperties": false
-                                  },
-                                  "minItems": 1
-                                }
-                              },
-                              "required": ["type", "criteria"],
-                              "additionalProperties": false
-                            }
-                          ]
-                        }
-                      },
-                      "skip_defaults": {
-                        "type": "boolean"
-                      },
-                      "cache": {
-                        "type": "boolean"
-                      },
-                      "trials": {
-                        "type": "object",
-                        "properties": {
-                          "count": {
-                            "type": "integer",
-                            "minimum": 1
-                          },
-                          "strategy": {
-                            "type": "string",
-                            "enum": ["pass_at_k", "mean", "confidence_interval"]
-                          },
-                          "cost_limit_usd": {
-                            "type": "number",
-                            "minimum": 0
-                          },
-                          "costLimitUsd": {
-                            "type": "number",
-                            "minimum": 0
-                          }
-                        },
-                        "required": ["count"],
-                        "additionalProperties": false
-                      },
-                      "budget_usd": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "budgetUsd": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "fail_on_error": {
-                        "type": "boolean"
-                      },
-                      "failOnError": {
-                        "type": "boolean"
-                      },
-                      "threshold": {
-                        "type": "number",
-                        "minimum": 0,
-                        "maximum": 1
-                      }
-                    },
-                    "additionalProperties": false
-                  },
-                  "workspace": {
-                    "type": "object",
-                    "properties": {
-                      "template": {
-                        "type": "string"
-                      },
-                      "isolation": {
-                        "type": "string",
-                        "enum": ["shared", "per_test"]
-                      },
-                      "repos": {
-                        "type": "array",
-                        "items": {
-                          "type": "object",
-                          "properties": {
-                            "path": {
-                              "type": "string"
-                            },
-                            "repo": {
-                              "type": "string",
-                              "minLength": 1
-                            },
-                            "commit": {
-                              "type": "string",
-                              "minLength": 1
-                            },
-                            "base_commit": {
-                              "type": "string",
-                              "minLength": 1
-                            },
-                            "ancestor": {
-                              "type": "integer",
-                              "minimum": 0
-                            },
-                            "sparse": {
-                              "type": "array",
-                              "items": {
-                                "type": "string"
-                              }
-                            }
-                          },
-                          "additionalProperties": false
-                        }
-                      },
-                      "hooks": {
-                        "type": "object",
-                        "properties": {
-                          "enabled": {
-                            "type": "boolean"
-                          },
-                          "before_all": {
-                            "type": "object",
-                            "properties": {
-                              "command": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "script": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "timeout_ms": {
-                                "type": "number"
-                              },
-                              "timeoutMs": {
-                                "type": "number"
-                              },
-                              "cwd": {
-                                "type": "string"
-                              },
-                              "reset": {
-                                "type": "string",
-                                "enum": ["none", "fast", "strict"]
-                              }
-                            },
-                            "additionalProperties": false
-                          },
-                          "before_each": {
-                            "type": "object",
-                            "properties": {
-                              "command": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "script": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "timeout_ms": {
-                                "type": "number"
-                              },
-                              "timeoutMs": {
-                                "type": "number"
-                              },
-                              "cwd": {
-                                "type": "string"
-                              },
-                              "reset": {
-                                "type": "string",
-                                "enum": ["none", "fast", "strict"]
-                              }
-                            },
-                            "additionalProperties": false
-                          },
-                          "after_each": {
-                            "type": "object",
-                            "properties": {
-                              "command": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "script": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "timeout_ms": {
-                                "type": "number"
-                              },
-                              "timeoutMs": {
-                                "type": "number"
-                              },
-                              "cwd": {
-                                "type": "string"
-                              },
-                              "reset": {
-                                "type": "string",
-                                "enum": ["none", "fast", "strict"]
-                              }
-                            },
-                            "additionalProperties": false
-                          },
-                          "after_all": {
-                            "type": "object",
-                            "properties": {
-                              "command": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "script": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "timeout_ms": {
-                                "type": "number"
-                              },
-                              "timeoutMs": {
-                                "type": "number"
-                              },
-                              "cwd": {
-                                "type": "string"
-                              },
-                              "reset": {
-                                "type": "string",
-                                "enum": ["none", "fast", "strict"]
-                              }
-                            },
-                            "additionalProperties": false
-                          }
-                        },
-                        "additionalProperties": false
-                      },
-                      "mode": {
-                        "type": "string",
-                        "enum": ["pooled", "temp", "static"]
-                      },
-                      "path": {
-                        "type": "string"
-                      },
-                      "docker": {
-                        "type": "object",
-                        "properties": {
-                          "image": {
-                            "type": "string"
-                          },
-                          "timeout": {
-                            "type": "integer",
-                            "minimum": 1
-                          },
-                          "memory": {
-                            "type": "string"
-                          },
-                          "cpus": {
-                            "type": "number",
-                            "minimum": 0.1
-                          }
-                        },
-                        "required": ["image"],
-                        "additionalProperties": false
-                      }
-                    },
-                    "additionalProperties": false
-                  },
-                  "metadata": {
-                    "type": "object",
-                    "additionalProperties": {}
-                  },
-                  "conversation_id": {
-                    "type": "string"
-                  },
-                  "suite": {
-                    "type": "string"
-                  },
-                  "depends_on": {
-                    "type": "array",
-                    "items": {
-                      "type": "string"
-                    }
-                  },
-                  "on_dependency_failure": {
-                    "type": "string",
-                    "enum": ["skip", "fail", "run"]
-                  },
-                  "mode": {
-                    "type": "string",
-                    "enum": ["conversation"]
-                  },
-                  "turns": {
-                    "type": "array",
-                    "items": {
-                      "type": "object",
-                      "properties": {
-                        "input": {
-                          "anyOf": [
-                            {
-                              "type": "string"
-                            },
-                            {
-                              "anyOf": [
-                                {
-                                  "type": "string"
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "type": {
-                                        "type": "string",
-                                        "enum": ["text", "file", "image"]
-                                      },
-                                      "value": {
-                                        "type": "string"
-                                      }
-                                    },
-                                    "required": ["type", "value"],
-                                    "additionalProperties": false
-                                  }
-                                }
-                              ]
-                            }
-                          ]
-                        },
-                        "expected_output": {
-                          "anyOf": [
-                            {
-                              "type": "string"
-                            },
-                            {
-                              "anyOf": [
-                                {
-                                  "type": "string"
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "object",
-                                    "properties": {
-                                      "type": {
-                                        "type": "string",
-                                        "enum": ["text", "file", "image"]
-                                      },
-                                      "value": {
-                                        "type": "string"
-                                      }
-                                    },
-                                    "required": ["type", "value"],
-                                    "additionalProperties": false
-                                  }
-                                }
-                              ]
-                            }
-                          ]
-                        },
-                        "assertions": {
-                          "type": "array",
-                          "items": {
-                            "anyOf": [
-                              {
-                                "type": "string"
-                              },
-                              {
-                                "anyOf": [
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "enum": ["code-grader", "code_grader"]
-                                      },
-                                      "command": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string"
-                                          },
-                                          {
-                                            "type": "array",
-                                            "items": {
-                                              "type": "string"
-                                            }
-                                          }
-                                        ]
-                                      },
-                                      "script": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string"
-                                          },
-                                          {
-                                            "type": "array",
-                                            "items": {
-                                              "type": "string"
-                                            }
-                                          }
-                                        ]
-                                      },
-                                      "cwd": {
-                                        "type": "string"
-                                      },
-                                      "target": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "object",
-                                            "properties": {
-                                              "max_calls": {
-                                                "type": "number"
-                                              }
-                                            },
-                                            "additionalProperties": false
-                                          }
-                                        ]
-                                      },
-                                      "config": {
-                                        "type": "object",
-                                        "additionalProperties": {}
-                                      },
-                                      "preprocessors": {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "object",
-                                          "properties": {
-                                            "type": {
-                                              "type": "string",
-                                              "minLength": 1
-                                            },
-                                            "command": {
-                                              "anyOf": [
-                                                {
-                                                  "type": "string"
-                                                },
-                                                {
-                                                  "type": "array",
-                                                  "items": {
-                                                    "type": "string"
-                                                  }
-                                                }
-                                              ]
-                                            }
-                                          },
-                                          "required": ["type", "command"],
-                                          "additionalProperties": false
-                                        }
-                                      }
-                                    },
-                                    "required": ["type", "command"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "enum": ["llm-grader", "llm_grader"]
-                                      },
-                                      "prompt": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string"
-                                          },
-                                          {
-                                            "type": "object",
-                                            "properties": {
-                                              "command": {
-                                                "anyOf": [
-                                                  {
-                                                    "type": "string"
-                                                  },
-                                                  {
-                                                    "type": "array",
-                                                    "items": {
-                                                      "type": "string"
-                                                    }
-                                                  }
-                                                ]
-                                              },
-                                              "script": {
-                                                "anyOf": [
-                                                  {
-                                                    "type": "string"
-                                                  },
-                                                  {
-                                                    "type": "array",
-                                                    "items": {
-                                                      "type": "string"
-                                                    }
-                                                  }
-                                                ]
-                                              },
-                                              "config": {
-                                                "type": "object",
-                                                "additionalProperties": {}
-                                              }
-                                            },
-                                            "additionalProperties": false
-                                          }
-                                        ]
-                                      },
-                                      "rubrics": {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "object",
-                                          "properties": {
-                                            "id": {
-                                              "type": "string"
-                                            },
-                                            "outcome": {
-                                              "type": "string"
-                                            },
-                                            "operator": {
-                                              "type": "string",
-                                              "enum": ["correctness", "contradiction"]
-                                            },
-                                            "weight": {
-                                              "type": "number"
-                                            },
-                                            "required": {
-                                              "type": "boolean"
-                                            },
-                                            "min_score": {
-                                              "type": "number",
-                                              "exclusiveMinimum": true,
-                                              "minimum": 0,
-                                              "maximum": 1
-                                            },
-                                            "score_ranges": {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "object",
-                                                "properties": {
-                                                  "score_range": {
-                                                    "type": "array",
-                                                    "minItems": 2,
-                                                    "maxItems": 2,
-                                                    "items": [
-                                                      {
-                                                        "type": "integer",
-                                                        "minimum": 0,
-                                                        "maximum": 10
-                                                      },
-                                                      {
-                                                        "type": "integer",
-                                                        "minimum": 0,
-                                                        "maximum": 10
-                                                      }
-                                                    ]
-                                                  },
-                                                  "outcome": {
-                                                    "type": "string",
-                                                    "minLength": 1
-                                                  }
-                                                },
-                                                "required": ["score_range", "outcome"],
-                                                "additionalProperties": false
-                                              }
-                                            }
-                                          },
-                                          "additionalProperties": false
-                                        }
-                                      },
-                                      "model": {
-                                        "type": "string"
-                                      },
-                                      "target": {
-                                        "type": "string"
-                                      },
-                                      "config": {
-                                        "type": "object",
-                                        "additionalProperties": {}
-                                      },
-                                      "max_steps": {
-                                        "type": "integer",
-                                        "minimum": 1,
-                                        "maximum": 50
-                                      },
-                                      "temperature": {
-                                        "type": "number",
-                                        "minimum": 0,
-                                        "maximum": 2
-                                      },
-                                      "preprocessors": {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "object",
-                                          "properties": {
-                                            "type": {
-                                              "type": "string",
-                                              "minLength": 1
-                                            },
-                                            "command": {
-                                              "anyOf": [
-                                                {
-                                                  "type": "string"
-                                                },
-                                                {
-                                                  "type": "array",
-                                                  "items": {
-                                                    "type": "string"
-                                                  }
-                                                }
-                                              ]
-                                            }
-                                          },
-                                          "required": ["type", "command"],
-                                          "additionalProperties": false
-                                        }
-                                      }
-                                    },
-                                    "required": ["type"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "include": {
-                                        "type": "string",
-                                        "minLength": 1
-                                      }
-                                    },
-                                    "required": ["include"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "const": "composite"
-                                      },
-                                      "assertions": {
-                                        "type": "array",
-                                        "items": {}
-                                      },
-                                      "evaluators": {
-                                        "type": "array",
-                                        "items": {}
-                                      },
-                                      "aggregator": {
-                                        "anyOf": [
-                                          {
-                                            "type": "object",
-                                            "properties": {
-                                              "type": {
-                                                "type": "string",
-                                                "const": "weighted_average"
-                                              },
-                                              "weights": {
-                                                "type": "object",
-                                                "additionalProperties": {
-                                                  "type": "number"
-                                                }
-                                              }
-                                            },
-                                            "required": ["type"],
-                                            "additionalProperties": false
-                                          },
-                                          {
-                                            "type": "object",
-                                            "properties": {
-                                              "type": {
-                                                "type": "string",
-                                                "const": "threshold"
-                                              },
-                                              "threshold": {
-                                                "type": "number",
-                                                "minimum": 0,
-                                                "maximum": 1
-                                              }
-                                            },
-                                            "required": ["type", "threshold"],
-                                            "additionalProperties": false
-                                          },
-                                          {
-                                            "type": "object",
-                                            "properties": {
-                                              "type": {
-                                                "type": "string",
-                                                "const": "code-grader"
-                                              },
-                                              "path": {
-                                                "type": "string"
-                                              },
-                                              "cwd": {
-                                                "type": "string"
-                                              }
-                                            },
-                                            "required": ["type", "path"],
-                                            "additionalProperties": false
-                                          },
-                                          {
-                                            "type": "object",
-                                            "properties": {
-                                              "type": {
-                                                "type": "string",
-                                                "const": "llm-grader"
-                                              },
-                                              "prompt": {
-                                                "type": "string"
-                                              },
-                                              "model": {
-                                                "type": "string"
-                                              }
-                                            },
-                                            "required": ["type"],
-                                            "additionalProperties": false
-                                          }
-                                        ]
-                                      }
-                                    },
-                                    "required": ["type", "aggregator"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "enum": ["tool-trajectory", "tool_trajectory"]
-                                      },
-                                      "mode": {
-                                        "type": "string",
-                                        "enum": [
-                                          "any_order",
-                                          "in_order",
-                                          "exact",
-                                          "subset",
-                                          "superset"
-                                        ]
-                                      },
-                                      "minimums": {
-                                        "type": "object",
-                                        "additionalProperties": {
-                                          "type": "integer",
-                                          "minimum": 0
-                                        }
-                                      },
-                                      "expected": {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "object",
-                                          "properties": {
-                                            "tool": {
-                                              "type": "string"
-                                            },
-                                            "args": {
-                                              "anyOf": [
-                                                {
-                                                  "type": "string",
-                                                  "const": "any"
-                                                },
-                                                {
-                                                  "type": "object",
-                                                  "additionalProperties": {}
-                                                }
-                                              ]
-                                            },
-                                            "max_duration_ms": {
-                                              "type": "number",
-                                              "minimum": 0
-                                            },
-                                            "maxDurationMs": {
-                                              "type": "number",
-                                              "minimum": 0
-                                            },
-                                            "args_match": {
-                                              "anyOf": [
-                                                {
-                                                  "type": "string",
-                                                  "enum": ["exact", "ignore", "subset", "superset"]
-                                                },
-                                                {
-                                                  "type": "array",
-                                                  "items": {
-                                                    "type": "string"
-                                                  }
-                                                }
-                                              ]
-                                            },
-                                            "argsMatch": {
-                                              "anyOf": [
-                                                {
-                                                  "type": "string",
-                                                  "enum": ["exact", "ignore", "subset", "superset"]
-                                                },
-                                                {
-                                                  "type": "array",
-                                                  "items": {
-                                                    "type": "string"
-                                                  }
-                                                }
-                                              ]
-                                            }
-                                          },
-                                          "required": ["tool"],
-                                          "additionalProperties": false
-                                        }
-                                      },
-                                      "args_match": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string",
-                                            "enum": ["exact", "ignore", "subset", "superset"]
-                                          },
-                                          {
-                                            "type": "array",
-                                            "items": {
-                                              "type": "string"
-                                            }
-                                          }
-                                        ]
-                                      },
-                                      "argsMatch": {
-                                        "anyOf": [
-                                          {
-                                            "type": "string",
-                                            "enum": ["exact", "ignore", "subset", "superset"]
-                                          },
-                                          {
-                                            "type": "array",
-                                            "items": {
-                                              "type": "string"
-                                            }
-                                          }
-                                        ]
-                                      }
-                                    },
-                                    "required": ["type", "mode"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "enum": ["field-accuracy", "field_accuracy"]
-                                      },
-                                      "fields": {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "object",
-                                          "properties": {
-                                            "path": {
-                                              "type": "string"
-                                            },
-                                            "match": {
-                                              "type": "string",
-                                              "enum": ["exact", "numeric_tolerance", "date"]
-                                            },
-                                            "required": {
-                                              "type": "boolean"
-                                            },
-                                            "weight": {
-                                              "type": "number"
-                                            },
-                                            "tolerance": {
-                                              "type": "number",
-                                              "minimum": 0
-                                            },
-                                            "relative": {
-                                              "type": "boolean"
-                                            },
-                                            "formats": {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "string"
-                                              }
-                                            }
-                                          },
-                                          "required": ["path", "match"],
-                                          "additionalProperties": false
-                                        },
-                                        "minItems": 1
-                                      },
-                                      "aggregation": {
-                                        "type": "string",
-                                        "enum": ["weighted_average", "all_or_nothing"]
-                                      }
-                                    },
-                                    "required": ["type", "fields"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "const": "latency"
-                                      },
-                                      "threshold": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      }
-                                    },
-                                    "required": ["type", "threshold"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "const": "cost"
-                                      },
-                                      "budget": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      }
-                                    },
-                                    "required": ["type", "budget"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "enum": ["token-usage", "token_usage"]
-                                      },
-                                      "max_total": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "max_input": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "max_output": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      }
-                                    },
-                                    "required": ["type"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "enum": ["execution-metrics", "execution_metrics"]
-                                      },
-                                      "max_tool_calls": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "max_llm_calls": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "max_tokens": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "max_cost_usd": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "max_duration_ms": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "target_exploration_ratio": {
-                                        "type": "number",
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "exploration_tolerance": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      }
-                                    },
-                                    "required": ["type"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "const": "contains"
-                                      },
-                                      "value": {
-                                        "type": "string"
-                                      }
-                                    },
-                                    "required": ["type", "value"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "const": "regex"
-                                      },
-                                      "value": {
-                                        "type": "string"
-                                      }
-                                    },
-                                    "required": ["type", "value"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "enum": ["is-json", "is_json"]
-                                      }
-                                    },
-                                    "required": ["type"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "const": "equals"
-                                      },
-                                      "value": {
-                                        "type": "string"
-                                      }
-                                    },
-                                    "required": ["type", "value"],
-                                    "additionalProperties": false
-                                  },
-                                  {
-                                    "type": "object",
-                                    "properties": {
-                                      "name": {
-                                        "type": "string"
-                                      },
-                                      "weight": {
-                                        "type": "number",
-                                        "minimum": 0
-                                      },
-                                      "required": {
-                                        "anyOf": [
-                                          {
-                                            "type": "boolean"
-                                          },
-                                          {
-                                            "type": "number",
-                                            "exclusiveMinimum": true,
-                                            "minimum": 0,
-                                            "maximum": 1
-                                          }
-                                        ]
-                                      },
-                                      "min_score": {
-                                        "type": "number",
-                                        "exclusiveMinimum": true,
-                                        "minimum": 0,
-                                        "maximum": 1
-                                      },
-                                      "negate": {
-                                        "type": "boolean"
-                                      },
-                                      "type": {
-                                        "type": "string",
-                                        "const": "rubrics"
-                                      },
-                                      "criteria": {
-                                        "type": "array",
-                                        "items": {
-                                          "type": "object",
-                                          "properties": {
-                                            "id": {
-                                              "type": "string"
-                                            },
-                                            "outcome": {
-                                              "type": "string"
-                                            },
-                                            "operator": {
-                                              "type": "string",
-                                              "enum": ["correctness", "contradiction"]
-                                            },
-                                            "weight": {
-                                              "type": "number"
-                                            },
-                                            "required": {
-                                              "type": "boolean"
-                                            },
-                                            "min_score": {
-                                              "type": "number",
-                                              "exclusiveMinimum": true,
-                                              "minimum": 0,
-                                              "maximum": 1
-                                            },
-                                            "score_ranges": {
-                                              "type": "array",
-                                              "items": {
-                                                "type": "object",
-                                                "properties": {
-                                                  "score_range": {
-                                                    "type": "array",
-                                                    "minItems": 2,
-                                                    "maxItems": 2,
-                                                    "items": [
-                                                      {
-                                                        "type": "integer",
-                                                        "minimum": 0,
-                                                        "maximum": 10
-                                                      },
-                                                      {
-                                                        "type": "integer",
-                                                        "minimum": 0,
-                                                        "maximum": 10
-                                                      }
-                                                    ]
-                                                  },
-                                                  "outcome": {
-                                                    "type": "string",
-                                                    "minLength": 1
-                                                  }
-                                                },
-                                                "required": ["score_range", "outcome"],
-                                                "additionalProperties": false
-                                              }
-                                            }
-                                          },
-                                          "additionalProperties": false
-                                        },
-                                        "minItems": 1
-                                      }
-                                    },
-                                    "required": ["type", "criteria"],
-                                    "additionalProperties": false
-                                  }
-                                ]
-                              }
-                            ]
-                          }
-                        }
-                      },
-                      "required": ["input"],
-                      "additionalProperties": false
-                    },
-                    "minItems": 1
-                  },
-                  "aggregation": {
-                    "type": "string",
-                    "enum": ["mean", "min", "max"]
-                  },
-                  "on_turn_failure": {
-                    "type": "string",
-                    "enum": ["continue", "stop"]
-                  },
-                  "window_size": {
-                    "type": "integer",
-                    "minimum": 1
-                  }
-                },
-                "required": ["id"],
-                "additionalProperties": false
-              }
-            },
-            {
-              "type": "string"
-            }
-          ]
-        },
-        "target": {
-          "type": "string"
-        },
-        "execution": {
-          "type": "object",
-          "properties": {
-            "target": {
-              "type": "string"
-            },
-            "targets": {
-              "type": "array",
-              "items": {
-                "anyOf": [
-                  {
-                    "type": "string"
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string",
-                        "minLength": 1
-                      },
-                      "use_target": {
-                        "type": "string"
-                      },
-                      "hooks": {
-                        "type": "object",
-                        "properties": {
-                          "before_all": {
-                            "type": "object",
-                            "properties": {
-                              "command": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "script": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "timeout_ms": {
-                                "type": "number"
-                              },
-                              "timeoutMs": {
-                                "type": "number"
-                              },
-                              "cwd": {
-                                "type": "string"
-                              },
-                              "reset": {
-                                "type": "string",
-                                "enum": ["none", "fast", "strict"]
-                              }
-                            },
-                            "additionalProperties": false
-                          },
-                          "before_each": {
-                            "type": "object",
-                            "properties": {
-                              "command": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "script": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "timeout_ms": {
-                                "type": "number"
-                              },
-                              "timeoutMs": {
-                                "type": "number"
-                              },
-                              "cwd": {
-                                "type": "string"
-                              },
-                              "reset": {
-                                "type": "string",
-                                "enum": ["none", "fast", "strict"]
-                              }
-                            },
-                            "additionalProperties": false
-                          },
-                          "after_each": {
-                            "type": "object",
-                            "properties": {
-                              "command": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "script": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "timeout_ms": {
-                                "type": "number"
-                              },
-                              "timeoutMs": {
-                                "type": "number"
-                              },
-                              "cwd": {
-                                "type": "string"
-                              },
-                              "reset": {
-                                "type": "string",
-                                "enum": ["none", "fast", "strict"]
-                              }
-                            },
-                            "additionalProperties": false
-                          },
-                          "after_all": {
-                            "type": "object",
-                            "properties": {
-                              "command": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "script": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "timeout_ms": {
-                                "type": "number"
-                              },
-                              "timeoutMs": {
-                                "type": "number"
-                              },
-                              "cwd": {
-                                "type": "string"
-                              },
-                              "reset": {
-                                "type": "string",
-                                "enum": ["none", "fast", "strict"]
-                              }
-                            },
-                            "additionalProperties": false
-                          }
-                        },
-                        "additionalProperties": false
-                      }
-                    },
-                    "required": ["name"],
-                    "additionalProperties": false
-                  }
-                ]
-              }
-            },
-            "workers": {
-              "type": "integer",
-              "minimum": 1,
-              "maximum": 50
-            },
-            "assertions": {
-              "type": "array",
-              "items": {
-                "anyOf": [
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "enum": ["code-grader", "code_grader"]
-                      },
-                      "command": {
-                        "anyOf": [
-                          {
-                            "type": "string"
-                          },
-                          {
-                            "type": "array",
-                            "items": {
-                              "type": "string"
-                            }
-                          }
-                        ]
-                      },
-                      "script": {
-                        "anyOf": [
-                          {
-                            "type": "string"
-                          },
-                          {
-                            "type": "array",
-                            "items": {
-                              "type": "string"
-                            }
-                          }
-                        ]
-                      },
-                      "cwd": {
-                        "type": "string"
-                      },
-                      "target": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "object",
-                            "properties": {
-                              "max_calls": {
-                                "type": "number"
-                              }
-                            },
-                            "additionalProperties": false
-                          }
-                        ]
-                      },
-                      "config": {
-                        "type": "object",
-                        "additionalProperties": {}
-                      },
-                      "preprocessors": {
-                        "type": "array",
-                        "items": {
-                          "type": "object",
-                          "properties": {
-                            "type": {
-                              "type": "string",
-                              "minLength": 1
-                            },
-                            "command": {
-                              "anyOf": [
-                                {
-                                  "type": "string"
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "string"
-                                  }
-                                }
-                              ]
-                            }
-                          },
-                          "required": ["type", "command"],
-                          "additionalProperties": false
-                        }
-                      }
-                    },
-                    "required": ["type", "command"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "enum": ["llm-grader", "llm_grader"]
-                      },
-                      "prompt": {
-                        "anyOf": [
-                          {
-                            "type": "string"
-                          },
-                          {
-                            "type": "object",
-                            "properties": {
-                              "command": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "script": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "config": {
-                                "type": "object",
-                                "additionalProperties": {}
-                              }
-                            },
-                            "additionalProperties": false
-                          }
-                        ]
-                      },
-                      "rubrics": {
-                        "type": "array",
-                        "items": {
-                          "type": "object",
-                          "properties": {
-                            "id": {
-                              "type": "string"
-                            },
-                            "outcome": {
-                              "type": "string"
-                            },
-                            "operator": {
-                              "type": "string",
-                              "enum": ["correctness", "contradiction"]
-                            },
-                            "weight": {
-                              "type": "number"
-                            },
-                            "required": {
-                              "type": "boolean"
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "score_ranges": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "score_range": {
-                                    "type": "array",
-                                    "minItems": 2,
-                                    "maxItems": 2,
-                                    "items": [
-                                      {
-                                        "type": "integer",
-                                        "minimum": 0,
-                                        "maximum": 10
-                                      },
-                                      {
-                                        "type": "integer",
-                                        "minimum": 0,
-                                        "maximum": 10
-                                      }
-                                    ]
-                                  },
-                                  "outcome": {
-                                    "type": "string",
-                                    "minLength": 1
-                                  }
-                                },
-                                "required": ["score_range", "outcome"],
-                                "additionalProperties": false
-                              }
-                            }
-                          },
-                          "additionalProperties": false
-                        }
-                      },
-                      "model": {
-                        "type": "string"
-                      },
-                      "target": {
-                        "type": "string"
-                      },
-                      "config": {
-                        "type": "object",
-                        "additionalProperties": {}
-                      },
-                      "max_steps": {
-                        "type": "integer",
-                        "minimum": 1,
-                        "maximum": 50
-                      },
-                      "temperature": {
-                        "type": "number",
-                        "minimum": 0,
-                        "maximum": 2
-                      },
-                      "preprocessors": {
-                        "type": "array",
-                        "items": {
-                          "type": "object",
-                          "properties": {
-                            "type": {
-                              "type": "string",
-                              "minLength": 1
-                            },
-                            "command": {
-                              "anyOf": [
-                                {
-                                  "type": "string"
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "string"
-                                  }
-                                }
-                              ]
-                            }
-                          },
-                          "required": ["type", "command"],
-                          "additionalProperties": false
-                        }
-                      }
-                    },
-                    "required": ["type"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "include": {
-                        "type": "string",
-                        "minLength": 1
-                      }
-                    },
-                    "required": ["include"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "const": "composite"
-                      },
-                      "assertions": {
-                        "type": "array",
-                        "items": {}
-                      },
-                      "evaluators": {
-                        "type": "array",
-                        "items": {}
-                      },
-                      "aggregator": {
-                        "anyOf": [
-                          {
-                            "type": "object",
-                            "properties": {
-                              "type": {
-                                "type": "string",
-                                "const": "weighted_average"
-                              },
-                              "weights": {
-                                "type": "object",
-                                "additionalProperties": {
-                                  "type": "number"
-                                }
-                              }
-                            },
-                            "required": ["type"],
-                            "additionalProperties": false
-                          },
-                          {
-                            "type": "object",
-                            "properties": {
-                              "type": {
-                                "type": "string",
-                                "const": "threshold"
-                              },
-                              "threshold": {
-                                "type": "number",
-                                "minimum": 0,
-                                "maximum": 1
-                              }
-                            },
-                            "required": ["type", "threshold"],
-                            "additionalProperties": false
-                          },
-                          {
-                            "type": "object",
-                            "properties": {
-                              "type": {
-                                "type": "string",
-                                "const": "code-grader"
-                              },
-                              "path": {
-                                "type": "string"
-                              },
-                              "cwd": {
-                                "type": "string"
-                              }
-                            },
-                            "required": ["type", "path"],
-                            "additionalProperties": false
-                          },
-                          {
-                            "type": "object",
-                            "properties": {
-                              "type": {
-                                "type": "string",
-                                "const": "llm-grader"
-                              },
-                              "prompt": {
-                                "type": "string"
-                              },
-                              "model": {
-                                "type": "string"
-                              }
-                            },
-                            "required": ["type"],
-                            "additionalProperties": false
-                          }
-                        ]
-                      }
-                    },
-                    "required": ["type", "aggregator"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "enum": ["tool-trajectory", "tool_trajectory"]
-                      },
-                      "mode": {
-                        "type": "string",
-                        "enum": ["any_order", "in_order", "exact", "subset", "superset"]
-                      },
-                      "minimums": {
-                        "type": "object",
-                        "additionalProperties": {
-                          "type": "integer",
-                          "minimum": 0
-                        }
-                      },
-                      "expected": {
-                        "type": "array",
-                        "items": {
-                          "type": "object",
-                          "properties": {
-                            "tool": {
-                              "type": "string"
-                            },
-                            "args": {
-                              "anyOf": [
-                                {
-                                  "type": "string",
-                                  "const": "any"
-                                },
-                                {
-                                  "type": "object",
-                                  "additionalProperties": {}
-                                }
-                              ]
-                            },
-                            "max_duration_ms": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "maxDurationMs": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "args_match": {
-                              "anyOf": [
-                                {
-                                  "type": "string",
-                                  "enum": ["exact", "ignore", "subset", "superset"]
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "string"
-                                  }
-                                }
-                              ]
-                            },
-                            "argsMatch": {
-                              "anyOf": [
-                                {
-                                  "type": "string",
-                                  "enum": ["exact", "ignore", "subset", "superset"]
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "string"
-                                  }
-                                }
-                              ]
-                            }
-                          },
-                          "required": ["tool"],
-                          "additionalProperties": false
-                        }
-                      },
-                      "args_match": {
-                        "anyOf": [
-                          {
-                            "type": "string",
-                            "enum": ["exact", "ignore", "subset", "superset"]
-                          },
-                          {
-                            "type": "array",
-                            "items": {
-                              "type": "string"
-                            }
-                          }
-                        ]
-                      },
-                      "argsMatch": {
-                        "anyOf": [
-                          {
-                            "type": "string",
-                            "enum": ["exact", "ignore", "subset", "superset"]
-                          },
-                          {
-                            "type": "array",
-                            "items": {
-                              "type": "string"
-                            }
-                          }
-                        ]
-                      }
-                    },
-                    "required": ["type", "mode"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "enum": ["field-accuracy", "field_accuracy"]
-                      },
-                      "fields": {
-                        "type": "array",
-                        "items": {
-                          "type": "object",
-                          "properties": {
-                            "path": {
-                              "type": "string"
-                            },
-                            "match": {
-                              "type": "string",
-                              "enum": ["exact", "numeric_tolerance", "date"]
-                            },
-                            "required": {
-                              "type": "boolean"
-                            },
-                            "weight": {
-                              "type": "number"
-                            },
-                            "tolerance": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "relative": {
-                              "type": "boolean"
-                            },
-                            "formats": {
-                              "type": "array",
-                              "items": {
-                                "type": "string"
-                              }
-                            }
-                          },
-                          "required": ["path", "match"],
-                          "additionalProperties": false
-                        },
-                        "minItems": 1
-                      },
-                      "aggregation": {
-                        "type": "string",
-                        "enum": ["weighted_average", "all_or_nothing"]
-                      }
-                    },
-                    "required": ["type", "fields"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "const": "latency"
-                      },
-                      "threshold": {
-                        "type": "number",
-                        "minimum": 0
-                      }
-                    },
-                    "required": ["type", "threshold"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "const": "cost"
-                      },
-                      "budget": {
-                        "type": "number",
-                        "minimum": 0
-                      }
-                    },
-                    "required": ["type", "budget"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "enum": ["token-usage", "token_usage"]
-                      },
-                      "max_total": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "max_input": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "max_output": {
-                        "type": "number",
-                        "minimum": 0
-                      }
-                    },
-                    "required": ["type"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "enum": ["execution-metrics", "execution_metrics"]
-                      },
-                      "max_tool_calls": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "max_llm_calls": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "max_tokens": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "max_cost_usd": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "max_duration_ms": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "target_exploration_ratio": {
-                        "type": "number",
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "exploration_tolerance": {
-                        "type": "number",
-                        "minimum": 0
-                      }
-                    },
-                    "required": ["type"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "const": "contains"
-                      },
-                      "value": {
-                        "type": "string"
-                      }
-                    },
-                    "required": ["type", "value"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "const": "regex"
-                      },
-                      "value": {
-                        "type": "string"
-                      }
-                    },
-                    "required": ["type", "value"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "enum": ["is-json", "is_json"]
-                      }
-                    },
-                    "required": ["type"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "const": "equals"
-                      },
-                      "value": {
-                        "type": "string"
-                      }
-                    },
-                    "required": ["type", "value"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "const": "rubrics"
-                      },
-                      "criteria": {
-                        "type": "array",
-                        "items": {
-                          "type": "object",
-                          "properties": {
-                            "id": {
-                              "type": "string"
-                            },
-                            "outcome": {
-                              "type": "string"
-                            },
-                            "operator": {
-                              "type": "string",
-                              "enum": ["correctness", "contradiction"]
-                            },
-                            "weight": {
-                              "type": "number"
-                            },
-                            "required": {
-                              "type": "boolean"
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "score_ranges": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "score_range": {
-                                    "type": "array",
-                                    "minItems": 2,
-                                    "maxItems": 2,
-                                    "items": [
-                                      {
-                                        "type": "integer",
-                                        "minimum": 0,
-                                        "maximum": 10
-                                      },
-                                      {
-                                        "type": "integer",
-                                        "minimum": 0,
-                                        "maximum": 10
-                                      }
-                                    ]
-                                  },
-                                  "outcome": {
-                                    "type": "string",
-                                    "minLength": 1
-                                  }
-                                },
-                                "required": ["score_range", "outcome"],
-                                "additionalProperties": false
-                              }
-                            }
-                          },
-                          "additionalProperties": false
-                        },
-                        "minItems": 1
-                      }
-                    },
-                    "required": ["type", "criteria"],
-                    "additionalProperties": false
-                  }
-                ]
-              }
-            },
-            "evaluators": {
-              "type": "array",
-              "items": {
-                "anyOf": [
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "enum": ["code-grader", "code_grader"]
-                      },
-                      "command": {
-                        "anyOf": [
-                          {
-                            "type": "string"
-                          },
-                          {
-                            "type": "array",
-                            "items": {
-                              "type": "string"
-                            }
-                          }
-                        ]
-                      },
-                      "script": {
-                        "anyOf": [
-                          {
-                            "type": "string"
-                          },
-                          {
-                            "type": "array",
-                            "items": {
-                              "type": "string"
-                            }
-                          }
-                        ]
-                      },
-                      "cwd": {
-                        "type": "string"
-                      },
-                      "target": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "object",
-                            "properties": {
-                              "max_calls": {
-                                "type": "number"
-                              }
-                            },
-                            "additionalProperties": false
-                          }
-                        ]
-                      },
-                      "config": {
-                        "type": "object",
-                        "additionalProperties": {}
-                      },
-                      "preprocessors": {
-                        "type": "array",
-                        "items": {
-                          "type": "object",
-                          "properties": {
-                            "type": {
-                              "type": "string",
-                              "minLength": 1
-                            },
-                            "command": {
-                              "anyOf": [
-                                {
-                                  "type": "string"
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "string"
-                                  }
-                                }
-                              ]
-                            }
-                          },
-                          "required": ["type", "command"],
-                          "additionalProperties": false
-                        }
-                      }
-                    },
-                    "required": ["type", "command"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "enum": ["llm-grader", "llm_grader"]
-                      },
-                      "prompt": {
-                        "anyOf": [
-                          {
-                            "type": "string"
-                          },
-                          {
-                            "type": "object",
-                            "properties": {
-                              "command": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "script": {
-                                "anyOf": [
-                                  {
-                                    "type": "string"
-                                  },
-                                  {
-                                    "type": "array",
-                                    "items": {
-                                      "type": "string"
-                                    }
-                                  }
-                                ]
-                              },
-                              "config": {
-                                "type": "object",
-                                "additionalProperties": {}
-                              }
-                            },
-                            "additionalProperties": false
-                          }
-                        ]
-                      },
-                      "rubrics": {
-                        "type": "array",
-                        "items": {
-                          "type": "object",
-                          "properties": {
-                            "id": {
-                              "type": "string"
-                            },
-                            "outcome": {
-                              "type": "string"
-                            },
-                            "operator": {
-                              "type": "string",
-                              "enum": ["correctness", "contradiction"]
-                            },
-                            "weight": {
-                              "type": "number"
-                            },
-                            "required": {
-                              "type": "boolean"
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "score_ranges": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "score_range": {
-                                    "type": "array",
-                                    "minItems": 2,
-                                    "maxItems": 2,
-                                    "items": [
-                                      {
-                                        "type": "integer",
-                                        "minimum": 0,
-                                        "maximum": 10
-                                      },
-                                      {
-                                        "type": "integer",
-                                        "minimum": 0,
-                                        "maximum": 10
-                                      }
-                                    ]
-                                  },
-                                  "outcome": {
-                                    "type": "string",
-                                    "minLength": 1
-                                  }
-                                },
-                                "required": ["score_range", "outcome"],
-                                "additionalProperties": false
-                              }
-                            }
-                          },
-                          "additionalProperties": false
-                        }
-                      },
-                      "model": {
-                        "type": "string"
-                      },
-                      "target": {
-                        "type": "string"
-                      },
-                      "config": {
-                        "type": "object",
-                        "additionalProperties": {}
-                      },
-                      "max_steps": {
-                        "type": "integer",
-                        "minimum": 1,
-                        "maximum": 50
-                      },
-                      "temperature": {
-                        "type": "number",
-                        "minimum": 0,
-                        "maximum": 2
-                      },
-                      "preprocessors": {
-                        "type": "array",
-                        "items": {
-                          "type": "object",
-                          "properties": {
-                            "type": {
-                              "type": "string",
-                              "minLength": 1
-                            },
-                            "command": {
-                              "anyOf": [
-                                {
-                                  "type": "string"
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "string"
-                                  }
-                                }
-                              ]
-                            }
-                          },
-                          "required": ["type", "command"],
-                          "additionalProperties": false
-                        }
-                      }
-                    },
-                    "required": ["type"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "include": {
-                        "type": "string",
-                        "minLength": 1
-                      }
-                    },
-                    "required": ["include"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "const": "composite"
-                      },
-                      "assertions": {
-                        "type": "array",
-                        "items": {}
-                      },
-                      "evaluators": {
-                        "type": "array",
-                        "items": {}
-                      },
-                      "aggregator": {
-                        "anyOf": [
-                          {
-                            "type": "object",
-                            "properties": {
-                              "type": {
-                                "type": "string",
-                                "const": "weighted_average"
-                              },
-                              "weights": {
-                                "type": "object",
-                                "additionalProperties": {
-                                  "type": "number"
-                                }
-                              }
-                            },
-                            "required": ["type"],
-                            "additionalProperties": false
-                          },
-                          {
-                            "type": "object",
-                            "properties": {
-                              "type": {
-                                "type": "string",
-                                "const": "threshold"
-                              },
-                              "threshold": {
-                                "type": "number",
-                                "minimum": 0,
-                                "maximum": 1
-                              }
-                            },
-                            "required": ["type", "threshold"],
-                            "additionalProperties": false
-                          },
-                          {
-                            "type": "object",
-                            "properties": {
-                              "type": {
-                                "type": "string",
-                                "const": "code-grader"
-                              },
-                              "path": {
-                                "type": "string"
-                              },
-                              "cwd": {
-                                "type": "string"
-                              }
-                            },
-                            "required": ["type", "path"],
-                            "additionalProperties": false
-                          },
-                          {
-                            "type": "object",
-                            "properties": {
-                              "type": {
-                                "type": "string",
-                                "const": "llm-grader"
-                              },
-                              "prompt": {
-                                "type": "string"
-                              },
-                              "model": {
-                                "type": "string"
-                              }
-                            },
-                            "required": ["type"],
-                            "additionalProperties": false
-                          }
-                        ]
-                      }
-                    },
-                    "required": ["type", "aggregator"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "enum": ["tool-trajectory", "tool_trajectory"]
-                      },
-                      "mode": {
-                        "type": "string",
-                        "enum": ["any_order", "in_order", "exact", "subset", "superset"]
-                      },
-                      "minimums": {
-                        "type": "object",
-                        "additionalProperties": {
-                          "type": "integer",
-                          "minimum": 0
-                        }
-                      },
-                      "expected": {
-                        "type": "array",
-                        "items": {
-                          "type": "object",
-                          "properties": {
-                            "tool": {
-                              "type": "string"
-                            },
-                            "args": {
-                              "anyOf": [
-                                {
-                                  "type": "string",
-                                  "const": "any"
-                                },
-                                {
-                                  "type": "object",
-                                  "additionalProperties": {}
-                                }
-                              ]
-                            },
-                            "max_duration_ms": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "maxDurationMs": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "args_match": {
-                              "anyOf": [
-                                {
-                                  "type": "string",
-                                  "enum": ["exact", "ignore", "subset", "superset"]
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "string"
-                                  }
-                                }
-                              ]
-                            },
-                            "argsMatch": {
-                              "anyOf": [
-                                {
-                                  "type": "string",
-                                  "enum": ["exact", "ignore", "subset", "superset"]
-                                },
-                                {
-                                  "type": "array",
-                                  "items": {
-                                    "type": "string"
-                                  }
-                                }
-                              ]
-                            }
-                          },
-                          "required": ["tool"],
-                          "additionalProperties": false
-                        }
-                      },
-                      "args_match": {
-                        "anyOf": [
-                          {
-                            "type": "string",
-                            "enum": ["exact", "ignore", "subset", "superset"]
-                          },
-                          {
-                            "type": "array",
-                            "items": {
-                              "type": "string"
-                            }
-                          }
-                        ]
-                      },
-                      "argsMatch": {
-                        "anyOf": [
-                          {
-                            "type": "string",
-                            "enum": ["exact", "ignore", "subset", "superset"]
-                          },
-                          {
-                            "type": "array",
-                            "items": {
-                              "type": "string"
-                            }
-                          }
-                        ]
-                      }
-                    },
-                    "required": ["type", "mode"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "enum": ["field-accuracy", "field_accuracy"]
-                      },
-                      "fields": {
-                        "type": "array",
-                        "items": {
-                          "type": "object",
-                          "properties": {
-                            "path": {
-                              "type": "string"
-                            },
-                            "match": {
-                              "type": "string",
-                              "enum": ["exact", "numeric_tolerance", "date"]
-                            },
-                            "required": {
-                              "type": "boolean"
-                            },
-                            "weight": {
-                              "type": "number"
-                            },
-                            "tolerance": {
-                              "type": "number",
-                              "minimum": 0
-                            },
-                            "relative": {
-                              "type": "boolean"
-                            },
-                            "formats": {
-                              "type": "array",
-                              "items": {
-                                "type": "string"
-                              }
-                            }
-                          },
-                          "required": ["path", "match"],
-                          "additionalProperties": false
-                        },
-                        "minItems": 1
-                      },
-                      "aggregation": {
-                        "type": "string",
-                        "enum": ["weighted_average", "all_or_nothing"]
-                      }
-                    },
-                    "required": ["type", "fields"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "const": "latency"
-                      },
-                      "threshold": {
-                        "type": "number",
-                        "minimum": 0
-                      }
-                    },
-                    "required": ["type", "threshold"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "const": "cost"
-                      },
-                      "budget": {
-                        "type": "number",
-                        "minimum": 0
-                      }
-                    },
-                    "required": ["type", "budget"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "enum": ["token-usage", "token_usage"]
-                      },
-                      "max_total": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "max_input": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "max_output": {
-                        "type": "number",
-                        "minimum": 0
-                      }
-                    },
-                    "required": ["type"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "enum": ["execution-metrics", "execution_metrics"]
-                      },
-                      "max_tool_calls": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "max_llm_calls": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "max_tokens": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "max_cost_usd": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "max_duration_ms": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "target_exploration_ratio": {
-                        "type": "number",
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "exploration_tolerance": {
-                        "type": "number",
-                        "minimum": 0
-                      }
-                    },
-                    "required": ["type"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "const": "contains"
-                      },
-                      "value": {
-                        "type": "string"
-                      }
-                    },
-                    "required": ["type", "value"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "const": "regex"
-                      },
-                      "value": {
-                        "type": "string"
-                      }
-                    },
-                    "required": ["type", "value"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "enum": ["is-json", "is_json"]
-                      }
-                    },
-                    "required": ["type"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "const": "equals"
-                      },
-                      "value": {
-                        "type": "string"
-                      }
-                    },
-                    "required": ["type", "value"],
-                    "additionalProperties": false
-                  },
-                  {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "weight": {
-                        "type": "number",
-                        "minimum": 0
-                      },
-                      "required": {
-                        "anyOf": [
-                          {
-                            "type": "boolean"
-                          },
-                          {
-                            "type": "number",
-                            "exclusiveMinimum": true,
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        ]
-                      },
-                      "min_score": {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      },
-                      "negate": {
-                        "type": "boolean"
-                      },
-                      "type": {
-                        "type": "string",
-                        "const": "rubrics"
-                      },
-                      "criteria": {
-                        "type": "array",
-                        "items": {
-                          "type": "object",
-                          "properties": {
-                            "id": {
-                              "type": "string"
-                            },
-                            "outcome": {
-                              "type": "string"
-                            },
-                            "operator": {
-                              "type": "string",
-                              "enum": ["correctness", "contradiction"]
-                            },
-                            "weight": {
-                              "type": "number"
-                            },
-                            "required": {
-                              "type": "boolean"
-                            },
-                            "min_score": {
-                              "type": "number",
-                              "exclusiveMinimum": true,
-                              "minimum": 0,
-                              "maximum": 1
-                            },
-                            "score_ranges": {
-                              "type": "array",
-                              "items": {
-                                "type": "object",
-                                "properties": {
-                                  "score_range": {
-                                    "type": "array",
-                                    "minItems": 2,
-                                    "maxItems": 2,
-                                    "items": [
-                                      {
-                                        "type": "integer",
-                                        "minimum": 0,
-                                        "maximum": 10
-                                      },
-                                      {
-                                        "type": "integer",
-                                        "minimum": 0,
-                                        "maximum": 10
-                                      }
-                                    ]
-                                  },
-                                  "outcome": {
-                                    "type": "string",
-                                    "minLength": 1
-                                  }
-                                },
-                                "required": ["score_range", "outcome"],
-                                "additionalProperties": false
-                              }
-                            }
-                          },
-                          "additionalProperties": false
-                        },
-                        "minItems": 1
-                      }
-                    },
-                    "required": ["type", "criteria"],
-                    "additionalProperties": false
-                  }
-                ]
-              }
-            },
-            "skip_defaults": {
-              "type": "boolean"
-            },
-            "cache": {
-              "type": "boolean"
-            },
-            "trials": {
-              "type": "object",
-              "properties": {
-                "count": {
-                  "type": "integer",
-                  "minimum": 1
-                },
-                "strategy": {
-                  "type": "string",
-                  "enum": ["pass_at_k", "mean", "confidence_interval"]
-                },
-                "cost_limit_usd": {
-                  "type": "number",
-                  "minimum": 0
-                },
-                "costLimitUsd": {
-                  "type": "number",
-                  "minimum": 0
-                }
-              },
-              "required": ["count"],
-              "additionalProperties": false
-            },
-            "budget_usd": {
-              "type": "number",
-              "minimum": 0
-            },
-            "budgetUsd": {
-              "type": "number",
-              "minimum": 0
-            },
-            "fail_on_error": {
-              "type": "boolean"
-            },
-            "failOnError": {
-              "type": "boolean"
-            },
-            "threshold": {
-              "type": "number",
-              "minimum": 0,
-              "maximum": 1
-            }
-          },
-          "additionalProperties": false
-        },
-        "assertions": {
-          "type": "array",
-          "items": {
-            "anyOf": [
-              {
-                "type": "object",
-                "properties": {
-                  "name": {
-                    "type": "string"
-                  },
-                  "weight": {
-                    "type": "number",
-                    "minimum": 0
-                  },
-                  "required": {
-                    "anyOf": [
-                      {
-                        "type": "boolean"
-                      },
-                      {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      }
-                    ]
-                  },
-                  "min_score": {
-                    "type": "number",
-                    "exclusiveMinimum": true,
-                    "minimum": 0,
-                    "maximum": 1
-                  },
-                  "negate": {
-                    "type": "boolean"
-                  },
-                  "type": {
-                    "type": "string",
-                    "enum": ["code-grader", "code_grader"]
-                  },
-                  "command": {
-                    "anyOf": [
-                      {
-                        "type": "string"
-                      },
-                      {
-                        "type": "array",
-                        "items": {
-                          "type": "string"
-                        }
-                      }
-                    ]
-                  },
-                  "script": {
-                    "anyOf": [
-                      {
-                        "type": "string"
-                      },
-                      {
-                        "type": "array",
-                        "items": {
-                          "type": "string"
-                        }
-                      }
-                    ]
-                  },
-                  "cwd": {
-                    "type": "string"
-                  },
-                  "target": {
-                    "anyOf": [
-                      {
-                        "type": "boolean"
-                      },
-                      {
-                        "type": "object",
-                        "properties": {
-                          "max_calls": {
-                            "type": "number"
-                          }
-                        },
-                        "additionalProperties": false
-                      }
-                    ]
-                  },
-                  "config": {
-                    "type": "object",
-                    "additionalProperties": {}
-                  },
-                  "preprocessors": {
-                    "type": "array",
-                    "items": {
-                      "type": "object",
-                      "properties": {
-                        "type": {
-                          "type": "string",
-                          "minLength": 1
-                        },
-                        "command": {
-                          "anyOf": [
-                            {
-                              "type": "string"
-                            },
-                            {
-                              "type": "array",
-                              "items": {
-                                "type": "string"
-                              }
-                            }
-                          ]
-                        }
-                      },
-                      "required": ["type", "command"],
-                      "additionalProperties": false
-                    }
-                  }
-                },
-                "required": ["type", "command"],
-                "additionalProperties": false
-              },
-              {
-                "type": "object",
-                "properties": {
-                  "name": {
-                    "type": "string"
-                  },
-                  "weight": {
-                    "type": "number",
-                    "minimum": 0
-                  },
-                  "required": {
-                    "anyOf": [
-                      {
-                        "type": "boolean"
-                      },
-                      {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      }
-                    ]
-                  },
-                  "min_score": {
-                    "type": "number",
-                    "exclusiveMinimum": true,
-                    "minimum": 0,
-                    "maximum": 1
-                  },
-                  "negate": {
-                    "type": "boolean"
-                  },
-                  "type": {
-                    "type": "string",
-                    "enum": ["llm-grader", "llm_grader"]
-                  },
-                  "prompt": {
-                    "anyOf": [
-                      {
-                        "type": "string"
-                      },
-                      {
-                        "type": "object",
-                        "properties": {
-                          "command": {
-                            "anyOf": [
-                              {
-                                "type": "string"
-                              },
-                              {
-                                "type": "array",
-                                "items": {
-                                  "type": "string"
-                                }
-                              }
-                            ]
-                          },
-                          "script": {
-                            "anyOf": [
-                              {
-                                "type": "string"
-                              },
-                              {
-                                "type": "array",
-                                "items": {
-                                  "type": "string"
-                                }
-                              }
-                            ]
-                          },
-                          "config": {
-                            "type": "object",
-                            "additionalProperties": {}
-                          }
-                        },
-                        "additionalProperties": false
-                      }
-                    ]
-                  },
-                  "rubrics": {
-                    "type": "array",
-                    "items": {
-                      "type": "object",
-                      "properties": {
-                        "id": {
-                          "type": "string"
-                        },
-                        "outcome": {
-                          "type": "string"
-                        },
-                        "operator": {
-                          "type": "string",
-                          "enum": ["correctness", "contradiction"]
-                        },
-                        "weight": {
-                          "type": "number"
-                        },
-                        "required": {
-                          "type": "boolean"
-                        },
-                        "min_score": {
-                          "type": "number",
-                          "exclusiveMinimum": true,
-                          "minimum": 0,
-                          "maximum": 1
-                        },
-                        "score_ranges": {
-                          "type": "array",
-                          "items": {
-                            "type": "object",
-                            "properties": {
-                              "score_range": {
-                                "type": "array",
-                                "minItems": 2,
-                                "maxItems": 2,
-                                "items": [
-                                  {
-                                    "type": "integer",
-                                    "minimum": 0,
-                                    "maximum": 10
-                                  },
-                                  {
-                                    "type": "integer",
-                                    "minimum": 0,
-                                    "maximum": 10
-                                  }
-                                ]
-                              },
-                              "outcome": {
-                                "type": "string",
-                                "minLength": 1
-                              }
-                            },
-                            "required": ["score_range", "outcome"],
-                            "additionalProperties": false
-                          }
-                        }
-                      },
-                      "additionalProperties": false
-                    }
-                  },
-                  "model": {
-                    "type": "string"
-                  },
-                  "target": {
-                    "type": "string"
-                  },
-                  "config": {
-                    "type": "object",
-                    "additionalProperties": {}
-                  },
-                  "max_steps": {
-                    "type": "integer",
-                    "minimum": 1,
-                    "maximum": 50
-                  },
-                  "temperature": {
-                    "type": "number",
-                    "minimum": 0,
-                    "maximum": 2
-                  },
-                  "preprocessors": {
-                    "type": "array",
-                    "items": {
-                      "type": "object",
-                      "properties": {
-                        "type": {
-                          "type": "string",
-                          "minLength": 1
-                        },
-                        "command": {
-                          "anyOf": [
-                            {
-                              "type": "string"
-                            },
-                            {
-                              "type": "array",
-                              "items": {
-                                "type": "string"
-                              }
-                            }
-                          ]
-                        }
-                      },
-                      "required": ["type", "command"],
-                      "additionalProperties": false
-                    }
-                  }
-                },
-                "required": ["type"],
-                "additionalProperties": false
-              },
-              {
-                "type": "object",
-                "properties": {
-                  "include": {
-                    "type": "string",
-                    "minLength": 1
-                  }
-                },
-                "required": ["include"],
-                "additionalProperties": false
-              },
-              {
-                "type": "object",
-                "properties": {
-                  "name": {
-                    "type": "string"
-                  },
-                  "weight": {
-                    "type": "number",
-                    "minimum": 0
-                  },
-                  "required": {
-                    "anyOf": [
-                      {
-                        "type": "boolean"
-                      },
-                      {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      }
-                    ]
-                  },
-                  "min_score": {
-                    "type": "number",
-                    "exclusiveMinimum": true,
-                    "minimum": 0,
-                    "maximum": 1
-                  },
-                  "negate": {
-                    "type": "boolean"
-                  },
-                  "type": {
-                    "type": "string",
-                    "const": "composite"
-                  },
-                  "assertions": {
-                    "type": "array",
-                    "items": {}
-                  },
-                  "evaluators": {
-                    "type": "array",
-                    "items": {}
-                  },
-                  "aggregator": {
-                    "anyOf": [
-                      {
-                        "type": "object",
-                        "properties": {
-                          "type": {
-                            "type": "string",
-                            "const": "weighted_average"
-                          },
-                          "weights": {
-                            "type": "object",
-                            "additionalProperties": {
-                              "type": "number"
-                            }
-                          }
-                        },
-                        "required": ["type"],
-                        "additionalProperties": false
-                      },
-                      {
-                        "type": "object",
-                        "properties": {
-                          "type": {
-                            "type": "string",
-                            "const": "threshold"
-                          },
-                          "threshold": {
-                            "type": "number",
-                            "minimum": 0,
-                            "maximum": 1
-                          }
-                        },
-                        "required": ["type", "threshold"],
-                        "additionalProperties": false
-                      },
-                      {
-                        "type": "object",
-                        "properties": {
-                          "type": {
-                            "type": "string",
-                            "const": "code-grader"
-                          },
-                          "path": {
-                            "type": "string"
-                          },
-                          "cwd": {
-                            "type": "string"
-                          }
-                        },
-                        "required": ["type", "path"],
-                        "additionalProperties": false
-                      },
-                      {
-                        "type": "object",
-                        "properties": {
-                          "type": {
-                            "type": "string",
-                            "const": "llm-grader"
-                          },
-                          "prompt": {
-                            "type": "string"
-                          },
-                          "model": {
-                            "type": "string"
-                          }
-                        },
-                        "required": ["type"],
-                        "additionalProperties": false
-                      }
-                    ]
-                  }
-                },
-                "required": ["type", "aggregator"],
-                "additionalProperties": false
-              },
-              {
-                "type": "object",
-                "properties": {
-                  "name": {
-                    "type": "string"
-                  },
-                  "weight": {
-                    "type": "number",
-                    "minimum": 0
-                  },
-                  "required": {
-                    "anyOf": [
-                      {
-                        "type": "boolean"
-                      },
-                      {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      }
-                    ]
-                  },
-                  "min_score": {
-                    "type": "number",
-                    "exclusiveMinimum": true,
-                    "minimum": 0,
-                    "maximum": 1
-                  },
-                  "negate": {
-                    "type": "boolean"
-                  },
-                  "type": {
-                    "type": "string",
-                    "enum": ["tool-trajectory", "tool_trajectory"]
-                  },
-                  "mode": {
-                    "type": "string",
-                    "enum": ["any_order", "in_order", "exact", "subset", "superset"]
-                  },
-                  "minimums": {
-                    "type": "object",
-                    "additionalProperties": {
-                      "type": "integer",
-                      "minimum": 0
-                    }
-                  },
-                  "expected": {
-                    "type": "array",
-                    "items": {
-                      "type": "object",
-                      "properties": {
-                        "tool": {
-                          "type": "string"
-                        },
-                        "args": {
-                          "anyOf": [
-                            {
-                              "type": "string",
-                              "const": "any"
-                            },
-                            {
-                              "type": "object",
-                              "additionalProperties": {}
-                            }
-                          ]
-                        },
-                        "max_duration_ms": {
-                          "type": "number",
-                          "minimum": 0
-                        },
-                        "maxDurationMs": {
-                          "type": "number",
-                          "minimum": 0
-                        },
-                        "args_match": {
-                          "anyOf": [
-                            {
-                              "type": "string",
-                              "enum": ["exact", "ignore", "subset", "superset"]
-                            },
-                            {
-                              "type": "array",
-                              "items": {
-                                "type": "string"
-                              }
-                            }
-                          ]
-                        },
-                        "argsMatch": {
-                          "anyOf": [
-                            {
-                              "type": "string",
-                              "enum": ["exact", "ignore", "subset", "superset"]
-                            },
-                            {
-                              "type": "array",
-                              "items": {
-                                "type": "string"
-                              }
-                            }
-                          ]
-                        }
-                      },
-                      "required": ["tool"],
-                      "additionalProperties": false
-                    }
-                  },
-                  "args_match": {
-                    "anyOf": [
-                      {
-                        "type": "string",
-                        "enum": ["exact", "ignore", "subset", "superset"]
-                      },
-                      {
-                        "type": "array",
-                        "items": {
-                          "type": "string"
-                        }
-                      }
-                    ]
-                  },
-                  "argsMatch": {
-                    "anyOf": [
-                      {
-                        "type": "string",
-                        "enum": ["exact", "ignore", "subset", "superset"]
-                      },
-                      {
-                        "type": "array",
-                        "items": {
-                          "type": "string"
-                        }
-                      }
-                    ]
-                  }
-                },
-                "required": ["type", "mode"],
-                "additionalProperties": false
-              },
-              {
-                "type": "object",
-                "properties": {
-                  "name": {
-                    "type": "string"
-                  },
-                  "weight": {
-                    "type": "number",
-                    "minimum": 0
-                  },
-                  "required": {
-                    "anyOf": [
-                      {
-                        "type": "boolean"
-                      },
-                      {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      }
-                    ]
-                  },
-                  "min_score": {
-                    "type": "number",
-                    "exclusiveMinimum": true,
-                    "minimum": 0,
-                    "maximum": 1
-                  },
-                  "negate": {
-                    "type": "boolean"
-                  },
-                  "type": {
-                    "type": "string",
-                    "enum": ["field-accuracy", "field_accuracy"]
-                  },
-                  "fields": {
-                    "type": "array",
-                    "items": {
-                      "type": "object",
-                      "properties": {
-                        "path": {
-                          "type": "string"
-                        },
-                        "match": {
-                          "type": "string",
-                          "enum": ["exact", "numeric_tolerance", "date"]
-                        },
-                        "required": {
-                          "type": "boolean"
-                        },
-                        "weight": {
-                          "type": "number"
-                        },
-                        "tolerance": {
-                          "type": "number",
-                          "minimum": 0
-                        },
-                        "relative": {
-                          "type": "boolean"
-                        },
-                        "formats": {
-                          "type": "array",
-                          "items": {
-                            "type": "string"
-                          }
-                        }
-                      },
-                      "required": ["path", "match"],
-                      "additionalProperties": false
-                    },
-                    "minItems": 1
-                  },
-                  "aggregation": {
-                    "type": "string",
-                    "enum": ["weighted_average", "all_or_nothing"]
-                  }
-                },
-                "required": ["type", "fields"],
-                "additionalProperties": false
-              },
-              {
-                "type": "object",
-                "properties": {
-                  "name": {
-                    "type": "string"
-                  },
-                  "weight": {
-                    "type": "number",
-                    "minimum": 0
-                  },
-                  "required": {
-                    "anyOf": [
-                      {
-                        "type": "boolean"
-                      },
-                      {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      }
-                    ]
-                  },
-                  "min_score": {
-                    "type": "number",
-                    "exclusiveMinimum": true,
-                    "minimum": 0,
-                    "maximum": 1
-                  },
-                  "negate": {
-                    "type": "boolean"
-                  },
-                  "type": {
-                    "type": "string",
-                    "const": "latency"
-                  },
-                  "threshold": {
-                    "type": "number",
-                    "minimum": 0
-                  }
-                },
-                "required": ["type", "threshold"],
-                "additionalProperties": false
-              },
-              {
-                "type": "object",
-                "properties": {
-                  "name": {
-                    "type": "string"
-                  },
-                  "weight": {
-                    "type": "number",
-                    "minimum": 0
-                  },
-                  "required": {
-                    "anyOf": [
-                      {
-                        "type": "boolean"
-                      },
-                      {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      }
-                    ]
-                  },
-                  "min_score": {
-                    "type": "number",
-                    "exclusiveMinimum": true,
-                    "minimum": 0,
-                    "maximum": 1
-                  },
-                  "negate": {
-                    "type": "boolean"
-                  },
-                  "type": {
-                    "type": "string",
-                    "const": "cost"
-                  },
-                  "budget": {
-                    "type": "number",
-                    "minimum": 0
-                  }
-                },
-                "required": ["type", "budget"],
-                "additionalProperties": false
-              },
-              {
-                "type": "object",
-                "properties": {
-                  "name": {
-                    "type": "string"
-                  },
-                  "weight": {
-                    "type": "number",
-                    "minimum": 0
-                  },
-                  "required": {
-                    "anyOf": [
-                      {
-                        "type": "boolean"
-                      },
-                      {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      }
-                    ]
-                  },
-                  "min_score": {
-                    "type": "number",
-                    "exclusiveMinimum": true,
-                    "minimum": 0,
-                    "maximum": 1
-                  },
-                  "negate": {
-                    "type": "boolean"
-                  },
-                  "type": {
-                    "type": "string",
-                    "enum": ["token-usage", "token_usage"]
-                  },
-                  "max_total": {
-                    "type": "number",
-                    "minimum": 0
-                  },
-                  "max_input": {
-                    "type": "number",
-                    "minimum": 0
-                  },
-                  "max_output": {
-                    "type": "number",
-                    "minimum": 0
-                  }
-                },
-                "required": ["type"],
-                "additionalProperties": false
-              },
-              {
-                "type": "object",
-                "properties": {
-                  "name": {
-                    "type": "string"
-                  },
-                  "weight": {
-                    "type": "number",
-                    "minimum": 0
-                  },
-                  "required": {
-                    "anyOf": [
-                      {
-                        "type": "boolean"
-                      },
-                      {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      }
-                    ]
-                  },
-                  "min_score": {
-                    "type": "number",
-                    "exclusiveMinimum": true,
-                    "minimum": 0,
-                    "maximum": 1
-                  },
-                  "negate": {
-                    "type": "boolean"
-                  },
-                  "type": {
-                    "type": "string",
-                    "enum": ["execution-metrics", "execution_metrics"]
-                  },
-                  "max_tool_calls": {
-                    "type": "number",
-                    "minimum": 0
-                  },
-                  "max_llm_calls": {
-                    "type": "number",
-                    "minimum": 0
-                  },
-                  "max_tokens": {
-                    "type": "number",
-                    "minimum": 0
-                  },
-                  "max_cost_usd": {
-                    "type": "number",
-                    "minimum": 0
-                  },
-                  "max_duration_ms": {
-                    "type": "number",
-                    "minimum": 0
-                  },
-                  "target_exploration_ratio": {
-                    "type": "number",
-                    "minimum": 0,
-                    "maximum": 1
-                  },
-                  "exploration_tolerance": {
-                    "type": "number",
-                    "minimum": 0
-                  }
-                },
-                "required": ["type"],
-                "additionalProperties": false
-              },
-              {
-                "type": "object",
-                "properties": {
-                  "name": {
-                    "type": "string"
-                  },
-                  "weight": {
-                    "type": "number",
-                    "minimum": 0
-                  },
-                  "required": {
-                    "anyOf": [
-                      {
-                        "type": "boolean"
-                      },
-                      {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      }
-                    ]
-                  },
-                  "min_score": {
-                    "type": "number",
-                    "exclusiveMinimum": true,
-                    "minimum": 0,
-                    "maximum": 1
-                  },
-                  "negate": {
-                    "type": "boolean"
-                  },
-                  "type": {
-                    "type": "string",
-                    "const": "contains"
-                  },
-                  "value": {
-                    "type": "string"
-                  }
-                },
-                "required": ["type", "value"],
-                "additionalProperties": false
-              },
-              {
-                "type": "object",
-                "properties": {
-                  "name": {
-                    "type": "string"
-                  },
-                  "weight": {
-                    "type": "number",
-                    "minimum": 0
-                  },
-                  "required": {
-                    "anyOf": [
-                      {
-                        "type": "boolean"
-                      },
-                      {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      }
-                    ]
-                  },
-                  "min_score": {
-                    "type": "number",
-                    "exclusiveMinimum": true,
-                    "minimum": 0,
-                    "maximum": 1
-                  },
-                  "negate": {
-                    "type": "boolean"
-                  },
-                  "type": {
-                    "type": "string",
-                    "const": "regex"
-                  },
-                  "value": {
-                    "type": "string"
-                  }
-                },
-                "required": ["type", "value"],
-                "additionalProperties": false
-              },
-              {
-                "type": "object",
-                "properties": {
-                  "name": {
-                    "type": "string"
-                  },
-                  "weight": {
-                    "type": "number",
-                    "minimum": 0
-                  },
-                  "required": {
-                    "anyOf": [
-                      {
-                        "type": "boolean"
-                      },
-                      {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      }
-                    ]
-                  },
-                  "min_score": {
-                    "type": "number",
-                    "exclusiveMinimum": true,
-                    "minimum": 0,
-                    "maximum": 1
-                  },
-                  "negate": {
-                    "type": "boolean"
-                  },
-                  "type": {
-                    "type": "string",
-                    "enum": ["is-json", "is_json"]
-                  }
-                },
-                "required": ["type"],
-                "additionalProperties": false
-              },
-              {
-                "type": "object",
-                "properties": {
-                  "name": {
-                    "type": "string"
-                  },
-                  "weight": {
-                    "type": "number",
-                    "minimum": 0
-                  },
-                  "required": {
-                    "anyOf": [
-                      {
-                        "type": "boolean"
-                      },
-                      {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      }
-                    ]
-                  },
-                  "min_score": {
-                    "type": "number",
-                    "exclusiveMinimum": true,
-                    "minimum": 0,
-                    "maximum": 1
-                  },
-                  "negate": {
-                    "type": "boolean"
-                  },
-                  "type": {
-                    "type": "string",
-                    "const": "equals"
-                  },
-                  "value": {
-                    "type": "string"
-                  }
-                },
-                "required": ["type", "value"],
-                "additionalProperties": false
-              },
-              {
-                "type": "object",
-                "properties": {
-                  "name": {
-                    "type": "string"
-                  },
-                  "weight": {
-                    "type": "number",
-                    "minimum": 0
-                  },
-                  "required": {
-                    "anyOf": [
-                      {
-                        "type": "boolean"
-                      },
-                      {
-                        "type": "number",
-                        "exclusiveMinimum": true,
-                        "minimum": 0,
-                        "maximum": 1
-                      }
-                    ]
-                  },
-                  "min_score": {
-                    "type": "number",
-                    "exclusiveMinimum": true,
-                    "minimum": 0,
-                    "maximum": 1
-                  },
-                  "negate": {
-                    "type": "boolean"
-                  },
-                  "type": {
-                    "type": "string",
-                    "const": "rubrics"
-                  },
-                  "criteria": {
-                    "type": "array",
-                    "items": {
-                      "type": "object",
-                      "properties": {
-                        "id": {
-                          "type": "string"
-                        },
-                        "outcome": {
-                          "type": "string"
-                        },
-                        "operator": {
-                          "type": "string",
-                          "enum": ["correctness", "contradiction"]
-                        },
-                        "weight": {
-                          "type": "number"
-                        },
-                        "required": {
-                          "type": "boolean"
-                        },
-                        "min_score": {
-                          "type": "number",
-                          "exclusiveMinimum": true,
-                          "minimum": 0,
-                          "maximum": 1
-                        },
-                        "score_ranges": {
-                          "type": "array",
-                          "items": {
-                            "type": "object",
-                            "properties": {
-                              "score_range": {
-                                "type": "array",
-                                "minItems": 2,
-                                "maxItems": 2,
-                                "items": [
-                                  {
-                                    "type": "integer",
-                                    "minimum": 0,
-                                    "maximum": 10
-                                  },
-                                  {
-                                    "type": "integer",
-                                    "minimum": 0,
-                                    "maximum": 10
-                                  }
-                                ]
-                              },
-                              "outcome": {
-                                "type": "string",
-                                "minLength": 1
-                              }
-                            },
-                            "required": ["score_range", "outcome"],
-                            "additionalProperties": false
-                          }
-                        }
-                      },
-                      "additionalProperties": false
-                    },
-                    "minItems": 1
-                  }
-                },
-                "required": ["type", "criteria"],
-                "additionalProperties": false
-              }
-            ]
-          }
-        },
-        "preprocessors": {
-          "type": "array",
-          "items": {
-            "type": "object",
-            "properties": {
-              "type": {
-                "type": "string",
-                "minLength": 1
-              },
-              "command": {
-                "anyOf": [
-                  {
-                    "type": "string"
-                  },
-                  {
-                    "type": "array",
-                    "items": {
-                      "type": "string"
-                    }
-                  }
-                ]
-              }
-            },
-            "required": ["type", "command"],
-            "additionalProperties": false
-          }
-        },
-        "workspace": {
-          "anyOf": [
-            {
-              "type": "object",
-              "properties": {
-                "template": {
-                  "type": "string"
-                },
-                "isolation": {
-                  "type": "string",
-                  "enum": ["shared", "per_test"]
-                },
-                "repos": {
-                  "type": "array",
-                  "items": {
-                    "type": "object",
-                    "properties": {
-                      "path": {
-                        "type": "string"
-                      },
-                      "repo": {
-                        "type": "string",
-                        "minLength": 1
-                      },
-                      "commit": {
-                        "type": "string",
-                        "minLength": 1
-                      },
-                      "base_commit": {
-                        "type": "string",
-                        "minLength": 1
-                      },
-                      "ancestor": {
-                        "type": "integer",
-                        "minimum": 0
-                      },
-                      "sparse": {
-                        "type": "array",
-                        "items": {
-                          "type": "string"
-                        }
-                      }
-                    },
-                    "additionalProperties": false
-                  }
-                },
-                "hooks": {
-                  "type": "object",
-                  "properties": {
-                    "enabled": {
-                      "type": "boolean"
-                    },
-                    "before_all": {
-                      "type": "object",
-                      "properties": {
-                        "command": {
-                          "anyOf": [
-                            {
-                              "type": "string"
-                            },
-                            {
-                              "type": "array",
-                              "items": {
-                                "type": "string"
-                              }
-                            }
-                          ]
-                        },
-                        "script": {
-                          "anyOf": [
-                            {
-                              "type": "string"
-                            },
-                            {
-                              "type": "array",
-                              "items": {
-                                "type": "string"
-                              }
-                            }
-                          ]
-                        },
-                        "timeout_ms": {
-                          "type": "number"
-                        },
-                        "timeoutMs": {
-                          "type": "number"
-                        },
-                        "cwd": {
-                          "type": "string"
-                        },
-                        "reset": {
-                          "type": "string",
-                          "enum": ["none", "fast", "strict"]
-                        }
-                      },
-                      "additionalProperties": false
-                    },
-                    "before_each": {
-                      "type": "object",
-                      "properties": {
-                        "command": {
-                          "anyOf": [
-                            {
-                              "type": "string"
-                            },
-                            {
-                              "type": "array",
-                              "items": {
-                                "type": "string"
-                              }
-                            }
-                          ]
-                        },
-                        "script": {
-                          "anyOf": [
-                            {
-                              "type": "string"
-                            },
-                            {
-                              "type": "array",
-                              "items": {
-                                "type": "string"
-                              }
-                            }
-                          ]
-                        },
-                        "timeout_ms": {
-                          "type": "number"
-                        },
-                        "timeoutMs": {
-                          "type": "number"
-                        },
-                        "cwd": {
-                          "type": "string"
-                        },
-                        "reset": {
-                          "type": "string",
-                          "enum": ["none", "fast", "strict"]
-                        }
-                      },
-                      "additionalProperties": false
-                    },
-                    "after_each": {
-                      "type": "object",
-                      "properties": {
-                        "command": {
-                          "anyOf": [
-                            {
-                              "type": "string"
-                            },
-                            {
-                              "type": "array",
-                              "items": {
-                                "type": "string"
-                              }
-                            }
-                          ]
-                        },
-                        "script": {
-                          "anyOf": [
-                            {
-                              "type": "string"
-                            },
-                            {
-                              "type": "array",
-                              "items": {
-                                "type": "string"
-                              }
-                            }
-                          ]
-                        },
-                        "timeout_ms": {
-                          "type": "number"
-                        },
-                        "timeoutMs": {
-                          "type": "number"
-                        },
-                        "cwd": {
-                          "type": "string"
-                        },
-                        "reset": {
-                          "type": "string",
-                          "enum": ["none", "fast", "strict"]
-                        }
-                      },
-                      "additionalProperties": false
-                    },
-                    "after_all": {
-                      "type": "object",
-                      "properties": {
-                        "command": {
-                          "anyOf": [
-                            {
-                              "type": "string"
-                            },
-                            {
-                              "type": "array",
-                              "items": {
-                                "type": "string"
-                              }
-                            }
-                          ]
-                        },
-                        "script": {
-                          "anyOf": [
-                            {
-                              "type": "string"
-                            },
-                            {
-                              "type": "array",
-                              "items": {
-                                "type": "string"
-                              }
-                            }
-                          ]
-                        },
-                        "timeout_ms": {
-                          "type": "number"
-                        },
-                        "timeoutMs": {
-                          "type": "number"
-                        },
-                        "cwd": {
-                          "type": "string"
-                        },
-                        "reset": {
-                          "type": "string",
-                          "enum": ["none", "fast", "strict"]
-                        }
-                      },
-                      "additionalProperties": false
-                    }
-                  },
-                  "additionalProperties": false
-                },
-                "mode": {
-                  "type": "string",
-                  "enum": ["pooled", "temp", "static"]
-                },
-                "path": {
-                  "type": "string"
-                },
-                "docker": {
-                  "type": "object",
-                  "properties": {
-                    "image": {
-                      "type": "string"
-                    },
-                    "timeout": {
-                      "type": "integer",
-                      "minimum": 1
-                    },
-                    "memory": {
-                      "type": "string"
-                    },
-                    "cpus": {
-                      "type": "number",
-                      "minimum": 0.1
-                    }
-                  },
-                  "required": ["image"],
-                  "additionalProperties": false
-                }
-              },
-              "additionalProperties": false
-            },
-            {
-              "type": "string"
-            }
-          ]
-        }
-      },
-      "required": ["tests"],
-      "additionalProperties": false
-    }
-  }
-}
-
---- references/rubric-evaluator.md ---
-# Rubric Grader
-
-Rubrics are defined as `assertions` entries with `type: rubrics`. They support binary checklist grading and score-range analytic grading.
-
-## Field Reference
-
-| Field | Type | Default | Description |
-|-------|------|---------|-------------|
-| `type` | string | required | Must be `rubrics` |
-| `criteria` | array | required | List of criterion strings or objects |
-| `required` | boolean or number | - | Gate: `true` requires score >= 0.8; a number (0–1) sets a custom threshold |
-
-### Criterion Object Fields
-
-| Field | Type | Default | Description |
-|-------|------|---------|-------------|
-| `id` | string | auto-generated | Unique identifier |
-| `outcome` | string | required* | Criterion being evaluated (*optional if `score_ranges` used) |
-| `operator` | string | - | Optional intent: `correctness` or `contradiction` |
-| `weight` | number | 1.0 | Relative importance |
-| `required` | boolean | true | Failing forces verdict to 'fail' (checklist mode) |
-| `min_score` | number | - | Minimum score (0–1) to pass this criterion |
-| `required_min_score` | integer | - | **Deprecated.** Use `min_score` instead. Legacy 0–10 scale. |
-| `score_ranges` | map or array | - | Score range definitions for analytic scoring |
-
-## String Shorthand (Recommended)
-
-Plain strings in `assertions` are automatically treated as rubric criteria:
-
-```yaml
-assertions:
-  - Mentions divide-and-conquer approach
-  - Explains partition step
-  - States time complexity
-```
-
-Equivalent to the full form with `type: rubrics`. Use the full form only when you need weights, `required: false`, or `score_ranges`.
-
-Mixed strings and objects are supported in `assertions` — strings are grouped into a single rubrics grader at the position of the first string:
-
-```yaml
-assertions:
-  - Mentions divide-and-conquer approach  # grouped into rubrics
-  - type: code-grader
-    command: [check_syntax.py]
-  - States time complexity                # grouped into rubrics
-```
-
-## Checklist Mode
-
-```yaml
-assertions:
-  - type: rubrics
-    criteria:
-      - Mentions divide-and-conquer approach
-      - id: complexity
-        outcome: States time complexity correctly
-        weight: 2.0
-        required: true
-      - id: examples
-        outcome: Includes code examples
-        weight: 1.0
-        required: false
-```
-
-### Criterion Operators
-
-Use `operator` when outcome text should carry grading intent without embedding words like "must not contradict" in the outcome itself:
-
-```yaml
-assertions:
-  - type: rubrics
-    criteria:
-      - id: supported-fact
-        operator: correctness
-        outcome: States revenue increased to $10M
-      - id: no-conflicting-fact
-        operator: contradiction
-        outcome: Revenue increased to $10M
-```
-
-- `correctness`: answer must positively support or fulfill the outcome.
-- `contradiction`: answer may omit the outcome, but must not make an incompatible claim.
-
-## Score-Range Mode
-
-Shorthand map format (recommended):
-
-```yaml
-assertions:
-  - type: rubrics
-    criteria:
-      - id: correctness
-        weight: 2.0
-        min_score: 0.7
-        score_ranges:
-          0: Critical bugs
-          3: Minor bugs
-          6: Correct with minor issues
-          9: Fully correct
-```
-
-Map keys are lower bounds (0-10). Each range extends from its key to (next key - 1), with the last extending to 10. Must start at 0.
-
-Array format is also accepted:
-
-```yaml
-    score_ranges:
-      - score_range: [0, 2]
-        outcome: Critical bugs
-      - score_range: [3, 5]
-        outcome: Minor bugs
-      - score_range: [6, 8]
-        outcome: Correct with minor issues
-      - score_range: [9, 10]
-        outcome: Fully correct
-```
-
-Ranges must be integers 0-10, non-overlapping, covering all values 0-10.
-
-## Scoring
-
-**Checklist:** `score = sum(satisfied weights) / sum(all weights)`
-
-**Score-range:** `score = weighted_average(raw_score / 10)` per criterion
-
-## Verdicts
-
-| Verdict | Condition |
-|---------|-----------|
-| `pass` | score >= 0.8 AND all gating criteria satisfied |
-| `fail` | score < 0.8 OR any gating criterion failed |
-
-Gating: checklist uses `required: true`, score-range uses `min_score: N` (0–1 scale).
-
---- references/config-schema.json ---
-{
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "title": "AgentV Config Schema",
-  "description": "Schema for .agentv/config.yaml configuration files",
-  "type": "object",
-  "properties": {
-    "$schema": {
-      "type": "string",
-      "description": "Schema identifier",
-      "enum": ["agentv-config-v2"]
-    },
-    "required_version": {
-      "type": "string",
-      "description": "Minimum AgentV version required to run this project's evals. Uses semver range syntax (e.g., '>=2.11.0', '^2.11.0'). When the installed version is below the range, AgentV warns and prompts to update.",
-      "examples": [">=2.11.0", "^2.12.0", ">=2.11.0 <3.0.0"]
-    },
-    "eval_patterns": {
-      "type": "array",
-      "description": "Glob patterns for discovering eval files during interactive mode (`agentv eval` with no args). Defaults to ['**/evals/**/dataset*.yaml', '**/evals/**/eval.yaml'] if not specified.",
-      "items": {
-        "type": "string",
-        "description": "Glob pattern (e.g., '**/evals/**/dataset*.yaml', '**/evals/**/eval.yaml')"
-      },
-      "examples": [["**/evals/**/dataset*.yaml", "**/evals/**/eval.yaml"], ["**/evals/**/*.yaml"]]
-    },
-    "execution": {
-      "type": "object",
-      "description": "Default execution options. CLI flags take precedence over these values.",
-      "properties": {
-        "verbose": {
-          "type": "boolean",
-          "description": "Enable verbose logging (equivalent to --verbose)",
-          "default": false
-        },
-        "keep_workspaces": {
-          "type": "boolean",
-          "description": "Always keep temp workspaces after eval (equivalent to --keep-workspaces)",
-          "default": false
-        },
-        "otel_file": {
-          "type": "string",
-          "description": "Write OTLP JSON trace to this path (equivalent to --otel-file). Supports {timestamp} placeholder.",
-          "examples": [".agentv/results/otel-{timestamp}.json"]
-        }
-      },
-      "additionalProperties": false
-    },
-    "hooks": {
-      "type": "object",
-      "description": "Lifecycle hooks that run at specific points during agentv execution.",
-      "properties": {
-        "before_session": {
-          "type": "string",
-          "description": "Shell command to run once at agentv startup, before any command executes. stdout is parsed for env var exports (KEY=value or export KEY=\"value\") and injected into process.env. Keys already set in the environment are not overwritten. stderr is forwarded to the user. Non-zero exit aborts with an error.",
-          "examples": ["bun scripts/load-secrets.ts", "eval $(aws ssm get-parameters-by-path ...)"]
-        }
-      },
-      "additionalProperties": false
-    }
-  },
-  "required": ["$schema"],
-  "additionalProperties": false
-}
-
---- references/custom-evaluators.md ---
-# Custom Graders
-
-## Wire Format
-
-### Input (stdin JSON)
-
-```json
-{
-  "criteria": "string",
-  "input_files": ["path"],
-  "input": [{"role": "user", "content": "..."}],
-  "expected_output": [{"role": "assistant", "content": "..."}],
-  "output": "final answer text",
-  "messages": [{"role": "assistant", "content": "final answer text"}],
-  "trace": {
-    "schema_version": "agentv.trace.v1",
-    "event_count": 5,
-    "tool_calls": {"fetch": 1},
-    "error_count": 0,
-    "llm_call_count": 2,
-    "messages": [],
-    "events": []
-  },
-  "trace_summary": {
-    "event_count": 5,
-    "tool_calls": {"fetch": 1},
-    "error_count": 0,
-    "llm_call_count": 2
-  },
-  "token_usage": {"input": 1000, "output": 500},
-  "cost_usd": 0.0015,
-  "duration_ms": 3500,
-  "start_time": "2026-02-13T10:00:00.000Z",
-  "end_time": "2026-02-13T10:00:03.500Z"
-}
-```
-
-### Output (stdout JSON)
-
-```json
-{
-  "score": 0.85,
-  "assertions": [
-    { "text": "passed check", "passed": true },
-    { "text": "failed check", "passed": false }
-  ]
-}
-```
-
-`score` (0.0-1.0) required. `assertions` and `details` optional.
-
-## SDK Functions
-
-```typescript
-import { defineCodeGrader, createTargetClient, definePromptTemplate } from '@agentv/sdk';
-```
-
-- `defineCodeGrader(fn)` - Wraps evaluation function with stdin/stdout handling
-- `createTargetClient()` - Returns LLM proxy client (when `target: {}` configured)
-  - `.invoke({question, systemPrompt})` - Single LLM call
-  - `.invokeBatch(requests)` - Batch LLM calls
-- `definePromptTemplate(fn)` - Wraps prompt generation function
-  - Raw stdin uses `snake_case`; SDK handlers receive `camelCase`
-  - Context fields: `input`, `expectedOutput`, `output`, `messages`, `criteria`, `config`, `trace`, `traceSummary`, `tokenUsage`, `costUsd`, `durationMs`, `startTime`, `endTime`
-
-For Python, the repo-local helper example in `examples/features/sdk-python/` keeps canonical `snake_case` fields and rejects deprecated wire aliases like `output_text`, `input_text`, and `reference_answer`. It is not a separate Python runner; generated evals still run through the AgentV CLI.
-
-## Python Example
-
-```python
-#!/usr/bin/env python3
-from agentv_py.grader import Assertion, CodeGraderResult, define_code_grader
-
-
-def evaluate(context):
-    candidate = context.output or ""
-    assertions = []
-    for kw in ["async", "await"]:
-        assertions.append(Assertion(text=f"Keyword '{kw}'", passed=kw in candidate))
-    passed = sum(1 for item in assertions if item.passed)
-    return CodeGraderResult(
-        score=passed / max(len(assertions), 1),
-        assertions=assertions,
-    )
-
-
-if __name__ == "__main__":
-    define_code_grader(evaluate)
-```
-
-## TypeScript Example
-
-```typescript
-#!/usr/bin/env bun
-import { defineCodeGrader } from '@agentv/sdk';
-
-export default defineCodeGrader(({ output, criteria }) => {
-  const candidate = output ?? '';
-  const assertions: Array<{ text: string; passed: boolean }> = [];
-  if (candidate.includes(criteria)) {
-    assertions.push({ text: 'Matches expected outcome', passed: true });
-  } else {
-    assertions.push({ text: 'Does not match expected outcome', passed: false });
-  }
-  const passed = assertions.filter(a => a.passed).length;
-  return {
-    score: passed / Math.max(assertions.length, 1),
-    assertions,
-  };
-});
-```
-
-## Template Variables
-
-Derived from test fields (users never author these directly):
-
-| Variable | Source |
-|----------|--------|
-| `criteria` | Test `criteria` field |
-| `input` | Full resolved input array (JSON) |
-| `expected_output` | Full resolved expected array (JSON) |
-| `output` | Final answer / scored result string |
-| `messages` | Transcript messages from target execution |
-
-Markdown templates use `{{variable}}` syntax. TypeScript templates receive context object.
diff --git a/evals/agentv-dev/skills/fixtures/agentv-eval-writer.txt b/evals/agentv-dev/skills/fixtures/agentv-eval-writer.txt
deleted file mode 100644
index 79a4928b3..000000000
--- a/evals/agentv-dev/skills/fixtures/agentv-eval-writer.txt
+++ /dev/null
@@ -1,731 +0,0 @@
----
-name: agentv-eval-writer
-description: >-
-  Write, edit, review, and validate AgentV EVAL.yaml / .eval.yaml evaluation files.
-  Use when asked to create new eval files, update or fix existing ones, add or remove test cases,
-  configure graders (`llm-grader`, `code-grader`, `rubrics`), review whether an eval is correct or complete,
-  convert between EVAL.yaml and evals.json using `agentv convert`, or generate eval test cases
-  from chat transcripts (markdown conversation or JSON messages).
-  Do NOT use for creating SKILL.md files, writing skill definitions, or running evals —
-  running and benchmarking belongs to agentv-bench.
----
-
-# AgentV Eval Writer
-
-Comprehensive docs: https://agentv.dev
-
-## Authoring Principle
-
-Treat YAML as the canonical portable model. Prefer authoring `.eval.yaml` / `EVAL.yaml` first, then use TypeScript helpers, Python scripts, or executable graders only when they lower to the same fields or when the evaluation logic must actually run code.
-
-## Evaluation Types
-
-AgentV evaluations measure **execution quality** — whether your agent or skill produces correct output when invoked.
-
-For **trigger quality** (whether the right skill is triggered for the right prompts), see the [Evaluation Types guide](https://agentv.dev/guides/evaluation-types/). Do not use execution eval configs (`EVAL.yaml`, `evals.json`) for trigger evaluation — these are distinct concerns requiring different tooling and methodologies.
-
-## Starting from evals.json?
-
-If the project already has an Agent Skills `evals.json` file, use it as a starting point instead of writing YAML from scratch:
-
-```bash
-# Convert evals.json to AgentV EVAL YAML
-agentv convert evals.json
-
-# Run directly without converting (all commands accept evals.json)
-agentv eval evals.json
-```
-
-The converter maps `prompt` → `input`, `expected_output` → `expected_output`, `assertions` → `assertions` (`llm-grader`), and resolves `files[]` paths. The generated YAML includes TODO comments for AgentV features to add (workspace setup, code graders, rubrics, required gates).
-
-After converting, enhance the YAML with AgentV-specific capabilities shown below.
-
-## From Chat Transcript
-
-Convert a chat conversation into eval test cases without starting from scratch.
-
-**Input formats:**
-
-Markdown conversation:
-```
-User: How do I reset my password?
-Assistant: Go to Settings > Security > Reset Password...
-```
-
-JSON messages:
-```json
-[{"role": "user", "content": "How do I reset my password?"},
- {"role": "assistant", "content": "Go to Settings > Security > Reset Password..."}]
-```
-
-**Select exchanges that make good test cases:**
-- Factual Q&A — verifiable answers
-- Task completion — user requests an action, agent performs it
-- Edge cases — unusual inputs, error handling, boundary conditions
-- Multi-turn reasoning — exchanges where earlier context matters
-
-**Skip:** greetings, one-word acknowledgments, repeated exchanges
-
-**Multi-turn format** (when context from prior turns matters):
-```yaml
-tests:
-  - id: multi-turn-context
-    criteria: "Agent remembers prior context"
-    input:
-      - role: user
-        content: "My name is Alice"
-      - role: assistant
-        content: "Nice to meet you, Alice!"
-      - role: user
-        content: "What's my name?"
-    expected_output: "Your name is Alice."
-    assertions:
-      - type: rubrics
-        criteria:
-          - Correctly recalls the user's name from earlier in the conversation
-```
-
-**Guidelines:** preserve exact wording in `expected_output`; aim for 5–15 tests per transcript; pick exchanges that test different capabilities.
-
-## Quick Start
-
-```yaml
-description: Example eval
-execution:
-  target: default
-
-tests:
-  - id: greeting
-    criteria: Friendly greeting
-    input: "Say hello"
-    expected_output: "Hello! How can I help you?"
-    assertions:
-      - type: rubrics
-        criteria:
-          - Greeting is friendly and warm
-          - Offers to help
-```
-
-## Eval File Structure
-
-**Required:** `tests` (array or string path)
-**Optional:** `name`, `description`, `version`, `author`, `tags`, `license`, `requires`, `execution`, `suite`, `workspace`, `assertions`, `input`
-
-**Test fields:**
-
-| Field | Required | Description |
-|-------|----------|-------------|
-| `id` | yes | Unique identifier |
-| `criteria` | yes | What the response should accomplish |
-| `input` | yes | Input to the agent (string shorthand or full message array) |
-| `expected_output` | no | Gold-standard reference answer (string shorthand or full message array) |
-| `assertions` | no | Graders: deterministic checks, rubrics, and LLM/code graders |
-| `execution` | no | Per-case execution overrides |
-| `workspace` | no | Per-case workspace config (overrides suite-level) |
-| `metadata` | no | Arbitrary key-value pairs passed to setup/teardown scripts |
-| `conversation_id` | no | Thread grouping |
-
-**Shorthand forms:**
-- `input` (string) expands to `[{role: "user", content: "..."}]`
-- `expected_output` (string/object) expands to `[{role: "assistant", content: ...}]`
-- Use these canonical field names on disk; keep the wire format `snake_case`
-
-**Message format:** `{role, content}` where role is `system`, `user`, `assistant`, or `tool`
-**Content types:** inline text, `{type: "file", value: "./path.md"}`
-**File paths:** relative from eval file dir, or absolute with `/` prefix from repo root
-**File handling by provider type:** LLM providers receive file content inlined in XML tags. Agent providers receive a preread block with `file://` URIs and must read files themselves. See [Coding Agents > Prompt format](https://agentv.dev/targets/coding-agents#prompt-format).
-
-**JSONL format:** One test per line as JSON. Optional `.yaml` sidecar for shared defaults. See `examples/features/basic-jsonl/`.
-
-**Environment variables:** All string fields support `${{ VAR }}` interpolation. Missing vars resolve to empty string. Works in eval files, external case files, and workspace configs. `.env` files are loaded automatically.
-
-## Metadata
-
-When `name` is present, the suite is parsed as a metadata-bearing eval:
-
-```yaml
-name: export-screening        # required, lowercase/hyphens, max 64 chars
-description: Evaluates export control screening accuracy
-version: "1.0"
-author: acme-compliance
-tags: [compliance, agents]
-license: Apache-2.0
-requires:
-  agentv: ">=0.30.0"
-```
-
-## Suite-level Input
-
-Prepend shared input messages to every test (like suite-level `assertions`). Avoids repeating the same prompt file in each test:
-
-```yaml
-input:
-  - role: user
-    content:
-      - type: file
-        value: ./system-prompt.md
-
-tests: ./cases.yaml
-
-# cases.yaml — each test only needs its own query
-# - id: test-1
-#   criteria: ...
-#   input: "User question here"
-```
-
-Effective input: `[...suite input, ...test input]`. Skipped when `execution.skip_defaults: true`.
-Accepts same formats as test `input` (string or message array).
-
-## Tests as String Path
-
-Point `tests` to an external file instead of inlining:
-
-```yaml
-name: my-eval
-description: My evaluation suite
-tests: ./cases.yaml           # relative to eval file dir
-```
-
-The external file can be YAML (array of test objects) or JSONL.
-
-## Assertions Field
-
-`assertions` defines graders at the suite level or per-test level. It is the canonical field for all graders:
-
-```yaml
-# Suite-level (appended to every test)
-assertions:
-  - type: is-json
-    required: true
-  - type: contains
-    value: "status"
-
-tests:
-  - id: test-1
-    criteria: Returns JSON
-    input: Get status
-    # Per-test assertions (runs before suite-level)
-    assertions:
-      - type: equals
-        value: '{"status": "ok"}'
-```
-
-## How `criteria` and `assertions` Interact
-
-`criteria` is a **data field** — it describes what the response should accomplish. It is **not** a grader. How it gets evaluated depends on whether `assertions` is present:
-
-| Scenario | What happens | Warning? |
-|----------|-------------|----------|
-| `criteria` + **no `assertions`** | Implicit `llm-grader` runs automatically against `criteria` | No |
-| `criteria` + **`assertions` with only deterministic graders** (contains, regex, etc.) | Only declared graders run. `criteria` is **not evaluated**. | Yes — warns that no grader will consume criteria |
-| `criteria` + **`assertions` with a grader** (`llm-grader`, `code-grader`, `rubrics`) | Declared graders run. Graders receive `criteria` as input. | No |
-
-### No assertions → implicit llm-grader
-
-The simplest path. `criteria` is automatically evaluated by the default `llm-grader`:
-
-```yaml
-tests:
-  - id: simple-eval
-    criteria: Assistant correctly explains the bug and proposes a fix
-    input: "Debug this function..."
-    # No assertions → default llm-grader evaluates against criteria
-```
-
-### assertions present → no implicit grader
-
-When `assertions` is defined, **only the declared graders run**. If you want an LLM grader alongside deterministic checks, declare it explicitly:
-
-```yaml
-tests:
-  - id: mixed-eval
-    criteria: Response is helpful and mentions the fix
-    input: "Debug this function..."
-    assertions:
-      - type: llm-grader       # must be explicit when assertions is present
-      - type: contains
-        value: "fix"
-```
-
-**Common mistake:** defining `criteria` with only deterministic graders. The criteria will be ignored and a warning is emitted:
-
-```yaml
-tests:
-  - id: bad-example
-    criteria: Gives a thoughtful answer    # ⚠ NOT evaluated — no grader in assertions
-    input: "What is 2+2?"
-    assertions:
-      - type: contains
-        value: "4"
-    # Warning: criteria is defined but no grader in assertions will evaluate it.
-```
-
-## Required Gates
-
-Any grader can be marked `required` to enforce a minimum score:
-
-```yaml
-assertions:
-  - type: contains
-    value: "DENIED"
-    required: true          # must score >= 0.8 (default)
-  - type: rubrics
-    required: 0.6           # must score >= 0.6 (custom threshold)
-    criteria:
-      - id: accuracy
-        outcome: Identifies the denied party
-        weight: 5.0
-```
-
-If a required grader scores below its threshold, the overall verdict is forced to `fail`.
-
-## Workspace Setup/Teardown
-
-Run scripts before/after each test. Define at suite level or override per case:
-
-```yaml
-workspace:
-  template: ./workspace-templates/my-project
-  repos:
-    - path: ./repo
-      repo: sympy/sympy
-      base_commit: "abc123"
-  hooks:
-    before_all:
-      command: ["bun", "run", "setup.ts"]
-      timeout_ms: 120000
-    after_each:
-      reset: fast
-    after_all:
-      command: ["bun", "run", "teardown.ts"]
-
-tests:
-  - id: case-1
-    input: Fix the bug
-    criteria: Bug is fixed
-    metadata:
-      source_repo: sympy/sympy
-      source_commit: "abc123"
-```
-
-**Lifecycle:** template copy → repo materialization → workspace before_all → target before_all → git baseline → before_each hooks → agent → file changes → after_each hooks → after_all hooks → cleanup
-**Merge:** Case-level fields replace suite-level fields.
-**Commands receive stdin JSON:** `{workspace_path, test_id, eval_run_id, case_input, case_metadata}`
-**Setup failure:** aborts case. **Teardown failure:** non-fatal (warning).
-For SWE-bench-style evals, keep operational checkout state under `workspace.repos[].base_commit`; treat `metadata.source_commit` as informational only.
-
-### Repository Lifecycle
-
-Materialize repos into the eval workspace automatically. Repo entries declare identity and checkout pins only; AgentV resolves acquisition from registered projects, `git_cache.mirrors`, its mirror cache, then remote clone. For shared repo workspaces, pooling is the default:
-
-```yaml
-workspace:
-  repos:
-    - path: ./repo
-      repo: https://github.com/org/repo.git
-      commit: main
-      ancestor: 1       # parent commit
-  hooks:
-    after_each:
-      reset: fast          # none | fast | strict
-  isolation: shared        # shared | per_test
-  mode: pooled             # pooled | temp | static
-```
-
-- `repo`: full clone URL or GitHub `org/name` shorthand
-- `commit`: branch, tag, or SHA to check out
-- `base_commit`: alias for `commit` for SWE-bench-style datasets
-- `ancestor`: walk N commits back from the checked-out ref
-- `sparse`: sparse checkout paths array
-- Do not use legacy `source`, `type`, `checkout`, `resolve`, or `clone` fields under `workspace.repos[]`
-- `mode`: `pooled` (default for shared repos), `temp`, or `static`
-- `path`: workspace path used when `mode: static`; when empty/missing the workspace is auto-materialised (template copied + repos cloned); populated dirs are reused as-is
-- `hooks.enabled`: boolean (default `true`); set `false` to skip all lifecycle hooks
-- Pool reset defaults to `fast` (`git clean -fd`); use `--workspace-clean full` for strict reset (`git clean -fdx`)
-- Pool entries are managed separately via `agentv workspace list` and `agentv workspace clean`
-- `agentv workspace deps <eval-paths>` scans eval files and outputs a JSON manifest of required git repos (useful for CI pre-cloning)
-
-See https://agentv.dev/targets/configuration/#repository-lifecycle
-
-## Grader Types
-
-Configure via `assertions` array. Multiple graders produce a weighted average score.
-
-### code-grader
-```yaml
-- name: format_check
-  type: code-grader
-  command: [uv, run, validate.py]
-  cwd: ./scripts          # optional working directory
-  target: {}              # optional: enable LLM target proxy (max_calls: 50)
-```
-Contract: stdin JSON -> stdout JSON `{score, assertions: [{text, passed, evidence?}], reasoning}`
-Raw stdin uses snake_case and includes: `criteria`, `input`, `expected_output`, `output` (final answer string), `messages`, `trace`, `trace_summary`, `token_usage`, `cost_usd`, `duration_ms`, `start_time`, `end_time`, `file_changes`, `workspace_path`, `config`
-SDK handlers receive the same payload in camelCase: `expectedOutput`, `traceSummary`, `tokenUsage`, `costUsd`, `durationMs`, `startTime`, `endTime`, `fileChanges`, `workspacePath`.
-When a workspace is configured, `workspace_path` is the absolute path to the workspace dir (also available as `AGENTV_WORKSPACE_PATH` env var). Use this for functional grading (e.g., running `npm test` in the workspace).
-See docs at https://agentv.dev/graders/code-graders/
-
-### llm-grader
-```yaml
-- name: quality
-  type: llm-grader
-  prompt: ./prompts/eval.md     # markdown template or command config
-  target: grader_gpt_5_mini     # optional: override the grader target for this grader
-  model: gpt-5-chat            # optional model override
-  config:                       # passed to prompt templates as context.config
-    strictness: high
-```
-Variables: `{{criteria}}`, `{{input}}`, `{{expected_output}}`, `{{output}}`, `{{metadata}}`, `{{metadata_json}}`, `{{rubrics}}`, `{{rubrics_json}}`, `{{file_changes}}`, `{{tool_calls}}`
-- Markdown templates: use `{{variable}}` syntax
-- TypeScript templates: use `definePromptTemplate(fn)` from `@agentv/sdk`, receives context object with all variables + `config`
-- Use `target:` to run different `llm-grader` graders against different named LLM targets in the same eval (useful for grader panels / ensembles)
-
-### composite
-```yaml
-- name: gate
-  type: composite
-  assertions:
-    - name: safety
-      type: llm-grader
-      prompt: ./safety.md
-    - name: quality
-      type: llm-grader
-  aggregator:
-    type: weighted_average
-    weights: { safety: 0.3, quality: 0.7 }
-```
-Aggregator types: `weighted_average`, `all_or_nothing`, `minimum`, `maximum`, `safety_gate`
-- `safety_gate`: fails immediately if the named gate grader scores below threshold (default 1.0)
-
-### tool_trajectory
-```yaml
-- name: tool_check
-  type: tool-trajectory
-  mode: any_order            # any_order | in_order | exact
-  minimums:                  # for any_order
-    knowledgeSearch: 2
-  expected:                  # for in_order/exact
-    - tool: knowledgeSearch
-      args: { query: "search term" }   # partial deep equality match
-    - tool: documentRetrieve
-      args: any                        # any arguments accepted
-      max_duration_ms: 5000            # per-tool latency assertion
-    - tool: summarize                  # omit args to skip argument checking
-```
-
-### field_accuracy
-```yaml
-- name: fields
-  type: field-accuracy
-  match_type: exact          # exact | date | numeric_tolerance
-  numeric_tolerance: 0.01    # for numeric_tolerance match_type
-  aggregation: weighted_average  # weighted_average | all_or_nothing
-```
-Compares `output` fields against `expected_output` fields.
-
-### latency
-```yaml
-- name: speed
-  type: latency
-  max_ms: 5000
-```
-
-### cost
-```yaml
-- name: budget
-  type: cost
-  max_usd: 0.10
-```
-
-### token_usage
-```yaml
-- name: tokens
-  type: token-usage
-  max_total_tokens: 4000
-```
-
-### execution_metrics
-```yaml
-- name: efficiency
-  type: execution-metrics
-  max_tool_calls: 10        # Maximum tool invocations
-  max_llm_calls: 5          # Maximum LLM calls (assistant messages)
-  max_tokens: 5000          # Maximum total tokens (input + output)
-  max_cost_usd: 0.05        # Maximum cost in USD
-  max_duration_ms: 30000    # Maximum execution duration
-  target_exploration_ratio: 0.6   # Target ratio of read-only tool calls
-  exploration_tolerance: 0.2      # Tolerance for ratio check (default: 0.2)
-```
-Declarative threshold-based checks on execution metrics. Only specified thresholds are checked.
-Score is proportional: `passed / total` assertions. Missing data counts as a failed assertion.
-
-### contains
-```yaml
-- type: contains
-  value: "DENIED"
-  required: true
-```
-Binary check: does output contain the substring? Name auto-generated if omitted.
-
-### regex
-```yaml
-- type: regex
-  value: "\\d{3}-\\d{2}-\\d{4}"
-```
-Binary check: does output match the regex pattern?
-
-### equals
-```yaml
-- type: equals
-  value: "42"
-```
-Binary check: does output exactly equal the value (both trimmed)?
-
-### is_json
-```yaml
-- type: is-json
-  required: true
-```
-Binary check: is the output valid JSON?
-
-### rubrics
-```yaml
-- type: rubrics
-  criteria:
-    - id: accuracy
-      outcome: Correctly identifies the denied party
-      weight: 5.0
-    - id: reasoning
-      outcome: Provides clear reasoning
-      weight: 3.0
-```
-LLM-judged structured evaluation with weighted criteria. Criteria items support `id`, `outcome`, `weight`, and `required` fields.
-Use optional `operator: correctness` for positive support checks or `operator: contradiction` for guard criteria where omission is acceptable but incompatible claims fail.
-
-See `references/rubric-grader.md` for score-range mode and scoring formula.
-
-## Execution Error Tolerance
-
-Control how the runner handles execution errors (infrastructure failures, not quality failures):
-
-```yaml
-execution:
-  fail_on_error: false    # never halt (default)
-  # fail_on_error: true   # halt on first execution error
-```
-
-When halted, remaining tests get `executionStatus: 'execution_error'` with `failureReasonCode: 'error_threshold_exceeded'`.
-
-## Suite-Level Quality Threshold
-
-Set a minimum mean score for the eval suite. If the mean quality score falls below the threshold, the CLI exits with code 1 — useful for CI/CD quality gates.
-
-```yaml
-execution:
-  threshold: 0.8
-```
-
-CLI flag `--threshold 0.8` overrides the YAML value. Must be a number between 0 and 1. Mean score is computed from quality results only (execution errors excluded).
-
-The threshold also controls JUnit XML pass/fail: tests with scores below the threshold are marked as `<failure>`. When no threshold is set, JUnit defaults to 0.5.
-
-## CLI Commands
-
-```bash
-# Run evaluation (requires API keys)
-agentv eval <file.yaml> [--test-id <id>] [--target <name>] [--dry-run] [--threshold <0-1>]
-
-# Run with OTLP JSON file (importable by OTel backends)
-agentv eval <file.yaml> --otel-file traces/eval.otlp.json
-
-# Record live target output for later target substitution
-agentv eval <file.yaml> --target live_agent --record-replay fixtures/target-output.jsonl
-agentv eval <file.yaml> --target replay_agent
-
-# Run a single assertion in isolation (no API keys needed)
-agentv eval assert <grader-name> --agent-output "..." --agent-input "..."
-
-# Import agent transcripts for offline grading
-agentv import claude --session-id <uuid>
-
-# Re-run only execution errors from a previous run
-agentv eval <file.yaml> --retry-errors .agentv/results/runs/<timestamp>/index.jsonl
-
-# Validate eval file
-agentv validate <file.yaml>
-
-# Compare results — N-way matrix from a canonical run manifest
-agentv compare .agentv/results/runs/<timestamp>/index.jsonl
-agentv compare .agentv/results/runs/<timestamp>/index.jsonl --baseline <target>                   # CI regression gate
-agentv compare .agentv/results/runs/<timestamp>/index.jsonl --baseline <target> --candidate <target>  # pairwise
-agentv compare .agentv/results/runs/<baseline-timestamp>/index.jsonl .agentv/results/runs/<candidate-timestamp>/index.jsonl
-
-# Author assertions directly in the eval file
-# Prefer simple assertions when they fit the criteria; use deterministic or LLM-based graders when needed
-agentv validate <file.yaml>
-```
-
-**Replay targets:** Add `provider: replay`, `fixtures: <jsonl>`, and `source_target: <live target name>` in `.agentv/targets.yaml`. Optional `suite`, `eval_path`, and `variant` tighten lookup. The eval YAML and graders stay unchanged; replay only substitutes recorded target output, and graders run fresh.
-
-## Code Judge SDK
-
-Use `@agentv/sdk` to build custom graders in TypeScript/JavaScript:
-
-### defineAssertion (recommended for custom checks)
-```typescript
-#!/usr/bin/env bun
-import { defineAssertion } from '@agentv/sdk';
-
-export default defineAssertion(({ output, trace }) => {
-  const finalOutput = output ?? '';
-  return {
-    pass: finalOutput.length > 0 && (trace?.eventCount ?? 0) <= 10,
-    reasoning: 'Checks content exists and is efficient',
-  };
-});
-```
-
-Assertions support both `pass: boolean` and `score: number` (0-1). If only `pass` is given, score is 1 (pass) or 0 (fail).
-
-### defineCodeGrader (full control)
-```typescript
-#!/usr/bin/env bun
-import { defineCodeGrader } from '@agentv/sdk';
-
-export default defineCodeGrader(({ output, trace }) => {
-  const finalOutput = output ?? '';
-  return {
-    score: finalOutput.length > 0 && (trace?.eventCount ?? 0) <= 5 ? 1.0 : 0.5,
-    assertions: [
-      { text: 'Output is not empty', passed: finalOutput.length > 0 },
-      { text: 'Efficient tool usage', passed: (trace?.eventCount ?? 0) <= 5 },
-    ],
-  };
-});
-```
-
-Both are used via `type: code-grader` in YAML with `command: [bun, run, grader.ts]`.
-
-### Convention-Based Discovery
-
-Place assertion files in `.agentv/assertions/` — they auto-register by filename:
-
-```
-.agentv/assertions/word-count.ts  →  type: word-count
-.agentv/assertions/sentiment.ts   →  type: sentiment
-```
-
-No `command:` needed in YAML — just use `type: <filename>`.
-
-## Programmatic API
-
-Use `evaluate()` from `@agentv/core` to run evals as a library when you need application-level control. Keep YAML as the default portable surface; use `specFile` to point at existing evals and inline `tests` only when the definition belongs in code.
-
-```typescript
-import { evaluate } from '@agentv/core';
-
-const { results, summary } = await evaluate({
-  tests: [
-    {
-      id: 'greeting',
-      input: 'Say hello',
-      expectedOutput: 'Hello there!',
-      assert: [{ type: 'contains', value: 'hello' }],
-    },
-  ],
-  target: { provider: 'mock_agent' },
-});
-console.log(`${summary.passed}/${summary.total} passed`);
-```
-
-Programmatic API notes:
-
-- Inline programmatic tests use `assert`, not `assertions`.
-- Use camelCase in TypeScript (`expectedOutput`, `beforeAll`, `budgetUsd`).
-- When bridging from Python, generate canonical YAML/JSONL or call the CLI; there is no separate first-party Python authoring SDK.
-
-Supports inline tests or file-based usage via `specFile`.
-
-## defineConfig
-
-Type-safe project configuration in `agentv.config.ts`:
-
-```typescript
-import { defineConfig } from '@agentv/core';
-
-export default defineConfig({
-  execution: { workers: 5, maxRetries: 2 },
-  output: { dir: './results' },
-  limits: { maxCostUsd: 10.0 },
-});
-```
-
-Auto-discovered from project root. Validated with Zod.
-
-## Scaffold Commands
-
-```bash
-agentv create assertion <name>  # → .agentv/assertions/<name>.ts
-agentv create eval <name>       # → evals/<name>.eval.yaml + .cases.jsonl
-```
-
-## Skill Improvement Workflow
-
-For a complete guide to iterating on skills using evaluations — writing scenarios, running baselines, comparing results, and improving — see the [Skill Improvement Workflow](https://agentv.dev/guides/skill-improvement-workflow/) guide.
-## Human Review Checkpoint
-
-After running evals, perform a human review before iterating. Create `feedback.json` in the results directory:
-
-```json
-{
-  "run_id": "2026-03-14T10-32-00_claude",
-  "reviewer": "engineer-name",
-  "timestamp": "2026-03-14T12:00:00Z",
-  "overall_notes": "Summary of observations",
-  "per_case": [
-    {
-      "test_id": "test-id",
-      "verdict": "acceptable | needs_improvement | incorrect | flaky",
-      "notes": "Why this verdict",
-      "evaluator_overrides": { "code-grader:name": "Override note" },
-      "workspace_notes": "Workspace state observations"
-    }
-  ]
-}
-```
-
-Use `evaluator_overrides` for workspace evaluations to annotate specific grader results (e.g., "code-grader was too strict"). Use `workspace_notes` for observations about workspace state.
-
-Review workflow: run evals → inspect results (`agentv inspect show`) → write feedback → tune prompts/graders → re-run.
-
-Full guide: https://agentv.dev/guides/human-review/
-
-## Observability Export
-
-AgentV exports observability data via OpenTelemetry:
-
-- `agentv eval <file.yaml> --otel-file traces/eval.otlp.json` writes a post-run OTLP JSON file that external systems such as Opik can ingest.
-- `agentv eval <file.yaml> --export-otel --otel-backend <name>` streams live traces when a built-in or local resolver exists.
-
-Do not invent a separate Opik-specific eval surface. Keep the eval definition in YAML and route observability through OTLP export.
-
-## Schemas
-
-- Eval file: `references/eval-schema.json`
-- Config: `references/config-schema.json`
-
-## Accessing reference files
-
-To load a specific reference without pulling the entire skill into context:
-
-```bash
-agentv skills get agentv-eval-writer --ref eval-schema.json
-```
-
-Or resolve the skill directory and read files directly:
-
-```bash
-cat $(agentv skills path agentv-eval-writer)/references/eval-schema.json
-```
-
-Use `--full` to retrieve every file in the skill at once.
diff --git a/evals/agentv-dev/skills/fixtures/agentv-governance-full.txt b/evals/agentv-dev/skills/fixtures/agentv-governance-full.txt
deleted file mode 100644
index 001223eb6..000000000
--- a/evals/agentv-dev/skills/fixtures/agentv-governance-full.txt
+++ /dev/null
@@ -1,561 +0,0 @@
----
-name: agentv-governance
-description: >-
-  Author, edit, and lint `governance:` blocks in `*.eval.yaml` files.
-  Use when creating or updating evaluation suites that carry AI-governance metadata
-  (OWASP LLM Top 10, OWASP Agentic Top 10, MITRE ATLAS, EU AI Act, ISO 42001).
-  Also use non-interactively (e.g., from a GitHub Action) to lint changed eval files
-  and report violations against the rules in `references/lint-rules.md`.
-  Do NOT use for running evals or benchmarking — that belongs to agentv-bench.
----
-
-# AgentV Compliance Skill
-
-Teaches AI agents how to author syntactically correct `governance:` blocks in AgentV
-eval files, and how to lint them against known vocabulary rules.
-
-## Dual mode
-
-**Authoring (interactive):** When a human or AI agent is editing a `*.eval.yaml` file
-that contains or should contain a `governance:` block, this skill provides vocabulary,
-valid values, and example shapes. Load it alongside `agentv-eval-writer` when building
-red-team or compliance suites.
-
-**Linting (non-interactive / CI):** When invoked from a GitHub Action (see
-`examples/governance/compliance-lint/`), this skill lints each changed `*.eval.yaml` file
-against the rules in `references/lint-rules.md` and returns a structured JSON report.
-The expected output format is:
-```json
-{
-  "pass": true,
-  "violations": [
-    {
-      "rule": "known_key",
-      "key": "risk_level",
-      "value": "high",
-      "message": "Unknown governance key 'risk_level'. Did you mean 'risk_tier'?",
-      "suggestion": "Replace 'risk_level' with 'risk_tier'."
-    }
-  ]
-}
-```
-`pass` is `true` when `violations` is empty.
-
-## Reference files
-
-| File | Purpose |
-|------|---------|
-| `references/governance-yaml-shape.md` | YAML shape, merge semantics, worked examples |
-| `references/lint-rules.md` | Machine-readable rules applied during lint |
-| `references/owasp-llm-top-10-2025.md` | LLM01–LLM10 canonical IDs and descriptions |
-| `references/owasp-agentic-top-10-2025.md` | T01–T10 agentic-AI categories |
-| `references/mitre-atlas.md` | Common AML.Txxxx technique IDs |
-| `references/eu-ai-act-risk-tiers.md` | Four risk tiers + article references |
-| `references/iso-42001-controls.md` | Curated ISO/IEC 42001:2023 controls for AI eval |
-
-## Quick authoring guide
-
-1. Check which risks this eval exercises using the reference files above.
-2. Pick IDs from the relevant frameworks (`owasp_llm_top_10_2025`, `mitre_atlas`, etc.).
-3. Set `risk_tier` using EU AI Act vocabulary (`prohibited | high | limited | minimal`).
-4. Add `controls` as `<FRAMEWORK>-<VERSION>:<ID>` strings (e.g. `EU-AI-ACT-2024:Art.55`).
-5. Run the lint rules from `references/lint-rules.md` against your block before committing.
-6. See `references/governance-yaml-shape.md` for complete examples copied from real suites.
-
-## Accessing reference files
-
-To load a specific reference without pulling the entire skill into context:
-
-```bash
-agentv skills get agentv-governance --ref lint-rules
-```
-
-Or resolve the skill directory and read files directly:
-
-```bash
-cat $(agentv skills path agentv-governance)/references/lint-rules.md
-```
-
-Use `--full` to retrieve every framework reference in one shot.
-
---- references/iso-42001-controls.md ---
-# ISO/IEC 42001:2023 — AI Management System Controls
-
-**Curated subset of controls relevant to AI evaluation suites.**
-
-Official source: ISO/IEC 42001:2023 — Information technology — Artificial intelligence —
-Management system. Full standard available at https://www.iso.org/standard/81230.html
-
-ISO 42001 is a management-system standard (like ISO 27001 for information security) covering
-the governance, risk management, and operational controls for organizations that develop or
-deploy AI systems.
-
-## Control reference format
-
-Use `ISO-42001-2023:<Clause>` in the `controls` array.
-
-## Relevant control areas for eval suites
-
-| Clause | Title | Relevance to evals |
-|--------|-------|-------------------|
-| 6.1 | Actions to address risks and opportunities | Risk identification for AI systems — align `risk_tier` with documented risk assessments. |
-| 6.1.2 | AI risk assessment | Formal risk assessment process; eval suites serve as evidence of risk measurement. |
-| 8.4 | AI system impact assessment | Assess potential societal impacts before deployment; red-team evals provide evidence. |
-| 8.5 | AI system life cycle | Controls for data, model, and deployment stages — align with suite test coverage. |
-| 9.1 | Monitoring, measurement, analysis and evaluation | Periodic eval runs as evidence of continuous monitoring. |
-| 9.1.1 | AI performance evaluation | Systematic measurement of AI output quality and safety properties. |
-| 10.2 | Nonconformity and corrective action | Failing evals trigger corrective action processes. |
-| A.2 | Policies for AI (Annex A) | Organizational AI use policies — `owner` field maps to the responsible team. |
-| A.5 | AI risk assessment (Annex A) | Documented risk assessment for each AI application. |
-| A.6 | AI system impact assessment (Annex A) | Broader societal-impact documentation. |
-
-## Usage example
-
-```yaml
-controls:
-  - ISO-42001-2023:6.1.2   # AI risk assessment
-  - ISO-42001-2023:9.1.1   # AI performance evaluation
-  - EU-AI-ACT-2024:Art.55  # GPAI transparency obligations
-```
-
-## Notes
-
-- ISO 42001 is certification-oriented; most teams will reference only a subset.
-  The clauses above are the ones most directly evidenced by running and storing eval results.
-- For pure LLM / red-team suites, clauses 6.1.2, 8.4, and 9.1.1 are the most common references.
-- Combine with NIST AI RMF controls (e.g. `NIST-AI-RMF-1.0:MEASURE-2.7`) when the organization
-  uses both frameworks.
-
---- references/mitre-atlas.md ---
-# MITRE ATLAS — AI/ML Threat Techniques
-
-**Canonical IDs for use in `mitre_atlas:` arrays.**
-
-Official source: https://atlas.mitre.org/
-
-MITRE ATLAS (Adversarial Threat Landscape for Artificial-Intelligence Systems) documents
-adversarial ML and AI attack techniques using the same taxonomy style as MITRE ATT&CK.
-IDs follow the pattern `AML.Txxxx` for techniques and `AML.Txxxx.xxx` for sub-techniques.
-
-## Techniques most relevant to LLM / agentic-AI evaluation
-
-| ID | Name | Relevant OWASP IDs |
-|----|------|-------------------|
-| AML.T0051 | LLM Prompt Injection | LLM01, T01 |
-| AML.T0054 | LLM Jailbreak | LLM01 |
-| AML.T0056 | LLM Meta Prompt Extraction | LLM07 |
-| AML.T0057 | LLM Plugin Compromise | LLM03, T09 |
-| AML.T0058 | LLM Data Leakage | LLM02 |
-| AML.T0068 | Training Data Poisoning | LLM04 |
-| AML.T0075 | Manipulate LLM Inputs | LLM01, T01 |
-
-## Sub-techniques
-
-Sub-techniques extend a base ID with a period-separated suffix, e.g.:
-- `AML.T0051.000` — Direct Prompt Injection
-- `AML.T0051.001` — Indirect Prompt Injection
-
-Use the base ID if the test covers the whole technique class; use sub-techniques for
-more precise tagging when the attack method is specific.
-
-## Usage notes
-
-- List IDs as strings in an array: `mitre_atlas: [AML.T0051, AML.T0075]`
-- Cross-reference with OWASP IDs when both frameworks cover the same attack:
-  a suite testing indirect prompt injection via tool output should tag
-  `owasp_llm_top_10_2025: [LLM01]` and `mitre_atlas: [AML.T0051]`.
-- For the full technique catalog, browse https://atlas.mitre.org/techniques/
-
---- references/governance-yaml-shape.md ---
-# Governance Block — YAML Shape and Examples
-
-## Field reference
-
-```yaml
-governance:
-  schema_version: "1.0"                    # string, optional — version of this block's schema
-  owasp_llm_top_10_2025: [LLM01]           # string[], optional — OWASP LLM Top 10 v2025 IDs
-  owasp_agentic_top_10_2025: [T01, T06]    # string[], optional — OWASP Agentic AI Top 10 v2025 IDs
-  mitre_atlas: [AML.T0051]                 # string[], optional — MITRE ATLAS technique IDs
-  controls: []                             # string[], optional — <FRAMEWORK>-<VERSION>:<ID> strings
-  risk_tier: high                          # string, optional — EU AI Act tier (see eu-ai-act-risk-tiers.md)
-  owner: security-team                     # string, optional — owning team or person
-```
-
-All fields are optional. Unknown keys pass through to JSONL output unchanged.
-
-## Control ID format
-
-The `controls` array accepts any string matching the pattern `<FRAMEWORK>-<VERSION>:<ID>`.
-Custom organizational prefixes are valid:
-
-```
-NIST-AI-RMF-1.0:MEASURE-2.7
-EU-AI-ACT-2024:Art.55
-ISO-42001-2023:6.1.2
-INTERNAL-AI-POLICY-3.2:CTRL-7
-```
-
-## Placement in eval files
-
-Governance blocks live in two places and are merged automatically:
-
-### 1. Suite-level (top-level key)
-
-Define once at the suite level and it will be merged into every case's `metadata.governance`:
-
-```yaml
-name: redteam-llm01-prompt-injection
-governance: &gov           # YAML anchor for reuse in per-case overrides
-  schema_version: "1.0"
-  owasp_llm_top_10_2025: [LLM01]
-  mitre_atlas: [AML.T0051]
-  controls:
-    - NIST-AI-RMF-1.0:MEASURE-2.7
-    - EU-AI-ACT-2024:Art.55
-  risk_tier: high
-  owner: security-team
-
-tests:
-  - id: direct-ignore-previous
-    metadata:
-      governance: *gov       # reference the anchor — identical to suite-level
-    ...
-```
-
-### 2. Per-case override with merge-key (`<<:`)
-
-Use YAML merge keys to inherit suite-level governance and add case-specific overrides.
-Arrays from both sides are concatenated and deduplicated; scalar fields on the case win:
-
-```yaml
-  - id: indirect-tool-output
-    metadata:
-      governance:
-        <<: *gov
-        owasp_llm_top_10_2025: [LLM01, LLM06]  # extends — case adds LLM06 to the inherited [LLM01]
-```
-
-## Merge semantics (how suite + case are combined)
-
-| Field type | Merge behavior |
-|-----------|----------------|
-| Arrays (`owasp_llm_top_10_2025`, `mitre_atlas`, `controls`) | Concatenate suite + case, deduplicate |
-| Scalars (`risk_tier`, `owner`, `schema_version`) | Case value overrides suite value |
-
-## Complete example — from `examples/red-team/suites/llm01-prompt-injection.eval.yaml`
-
-```yaml
-name: redteam-llm01-prompt-injection
-governance: &gov
-  schema_version: "1.0"
-  owasp_llm_top_10_2025: [LLM01]
-  mitre_atlas: [AML.T0051]
-  controls:
-    - NIST-AI-RMF-1.0:MEASURE-2.7
-    - EU-AI-ACT-2024:Art.55
-  risk_tier: high
-  owner: security-team
-
-tests:
-  - id: direct-ignore-previous
-    metadata:
-      governance: *gov
-    ...
-
-  - id: indirect-tool-output-document
-    metadata:
-      governance:
-        <<: *gov
-        owasp_llm_top_10_2025: [LLM01, LLM06]   # case adds LLM06
-    ...
-```
-
-## Complete example — from `examples/red-team/archetypes/coding-agent/suites/destructive-git.eval.yaml`
-
-```yaml
-name: redteam-coder-destructive-git
-governance: &gov
-  schema_version: "1.0"
-  owasp_llm_top_10_2025: [LLM06]
-  owasp_agentic_top_10_2025: [T01, T06]
-  mitre_atlas: [AML.T0051, AML.T0075]
-  controls:
-    - NIST-AI-RMF-1.0:MEASURE-2.7
-    - EU-AI-ACT-2024:Art.55
-  risk_tier: high
-  owner: security-team
-```
-
-## JSONL output
-
-The merged `governance` block is passed through verbatim to the JSONL result file under each
-result's `metadata.governance` key. Downstream tools (jq pipelines, `.ai-register.yaml`
-aggregators) consume it from there. The eval engine does not validate or transform the values.
-
---- references/owasp-llm-top-10-2025.md ---
-# OWASP LLM Top 10 v2025
-
-**Canonical IDs for use in `owasp_llm_top_10_2025:` arrays.**
-
-Official source: https://owasp.org/www-project-top-10-for-large-language-model-applications/
-
-| ID | Name | One-line description |
-|----|------|----------------------|
-| LLM01 | Prompt Injection | Attacker manipulates LLM behavior via crafted inputs (direct or indirect). |
-| LLM02 | Sensitive Information Disclosure | LLM reveals confidential data, system prompts, or PII in its output. |
-| LLM03 | Supply Chain | Compromised components — plugins, datasets, pre-trained weights — affect the LLM pipeline. |
-| LLM04 | Data and Model Poisoning | Training or fine-tuning data is tampered with to alter model behavior. |
-| LLM05 | Improper Output Handling | LLM output is passed unsanitized to downstream systems (XSS, SSRF, code injection). |
-| LLM06 | Excessive Agency | LLM acts on permissions or capabilities beyond what the task requires. |
-| LLM07 | System Prompt Leakage | The system prompt or internal context is exposed to the user or a third party. |
-| LLM08 | Vector and Embedding Weaknesses | Adversarial manipulation of embedding stores used for retrieval (RAG poisoning). |
-| LLM09 | Misinformation | LLM generates plausible but factually incorrect content that causes harm. |
-| LLM10 | Unbounded Consumption | LLM use is abused to exhaust resources — tokens, cost, rate limits, or compute. |
-
-## Usage notes
-
-- Use as many IDs as apply; list them in an array: `owasp_llm_top_10_2025: [LLM01, LLM06]`
-- IDs are version-anchored. When OWASP releases a new version, a new field
-  (`owasp_llm_top_10_2026`) will be added rather than redefining these IDs.
-- Combine with `mitre_atlas` IDs for technique-level tagging.
-
---- references/owasp-agentic-top-10-2025.md ---
-# OWASP Top 10 for Agentic AI v2025
-
-**Canonical IDs for use in `owasp_agentic_top_10_2025:` arrays.**
-
-Official source: https://owasp.org/www-project-top-10-for-large-language-model-applications/
-(Agentic AI supplement — see the "Agentic AI" section of the OWASP LLM project)
-
-| ID | Name | One-line description |
-|----|------|----------------------|
-| T01 | Prompt Injection for Agentic Systems | Attacker plants instructions in agent inputs, tool results, or retrieved content to redirect agent behavior. |
-| T02 | Memory Poisoning | Adversarial content is written to agent memory (short- or long-term) to influence future decisions. |
-| T03 | Data Exfiltration | Agent is manipulated into leaking sensitive data through tool calls, network requests, or outputs. |
-| T04 | Privilege Escalation | Agent acquires or is tricked into using permissions beyond its intended scope. |
-| T05 | Misconfigured Agent Networks | Overly permissive trust between orchestrating and sub-agents enables abuse. |
-| T06 | Tool and Plugin Misuse | Agent uses legitimate tools (bash, file I/O, API calls) outside their intended purpose or without authorization. |
-| T07 | Insecure Credential Storage | Agent stores or transmits credentials in memory, files, or outputs where they can be captured. |
-| T08 | Unsafe Agent-to-Agent Communication | Messages between agents are unvalidated, unencrypted, or susceptible to injection. |
-| T09 | Supply Chain Compromise | Malicious code in agent plugins, dependencies, or retrieved skill definitions. |
-| T10 | Lack of Accountability | Agent actions are not logged or attributable, making audit and incident response impossible. |
-
-## Usage notes
-
-- Combine with `owasp_llm_top_10_2025` IDs for cases that bridge both lists.
-  Example: an indirect-prompt-injection attack is LLM01 + T01 + T06 (tool misuse).
-- `T01` (Prompt Injection) and `LLM01` (Prompt Injection) are closely related but distinct:
-  LLM01 covers LLM-level injection; T01 covers the agent-orchestration dimension.
-- List multiple IDs when a test case exercises more than one category:
-  `owasp_agentic_top_10_2025: [T01, T06]`
-
---- references/eu-ai-act-risk-tiers.md ---
-# EU AI Act — Risk Tiers
-
-**Valid values for the `risk_tier:` field.**
-
-Official source: Regulation (EU) 2024/1689 on Artificial Intelligence (EU AI Act)
-Full text: https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX:32024R1689
-
-## Allowed values
-
-| Value | EU AI Act category | Key articles | Description |
-|-------|-------------------|-------------|-------------|
-| `prohibited` | Prohibited AI practices | Art. 5 | AI systems whose risks are deemed unacceptable — banned outright. Examples: social scoring by public authorities, real-time remote biometric surveillance in public spaces, AI that exploits vulnerabilities of specific groups. |
-| `high` | High-risk AI systems | Art. 6, Annex I–III | AI systems subject to mandatory conformity assessments, transparency, and human oversight. Examples: biometric identification, critical infrastructure, employment screening, access to education or essential services, law enforcement. |
-| `limited` | Limited-risk AI systems | Art. 50 | AI systems with transparency obligations only. Examples: chatbots must disclose they are AI; deep-fake generators must mark synthetic media. |
-| `minimal` | Minimal-risk AI systems | — | No mandatory obligations. Examples: spam filters, AI in video games. Voluntary codes of conduct encouraged. |
-
-## Usage notes
-
-- `risk_tier` is a scalar; only one value per governance block.
-- The vocabulary is anchored to EU AI Act terminology. Some organizations use different
-  risk scales (e.g. NIST SP 800-30 `low | moderate | high | very_high`). When mapping
-  from another framework, choose the EU AI Act equivalent that best matches the impact.
-- Combine `risk_tier: high` with `controls` referencing EU AI Act articles:
-  ```yaml
-  risk_tier: high
-  controls:
-    - EU-AI-ACT-2024:Art.55
-    - EU-AI-ACT-2024:Art.6
-  ```
-- `prohibited` tier should accompany test cases that specifically probe prohibited behaviors.
-  This does NOT mean the eval suite is itself prohibited — it means the suite tests whether
-  the system correctly refuses to engage in prohibited behaviors.
-
-## Article reference format
-
-Use `EU-AI-ACT-2024:<Article>` in the `controls` array, e.g. `EU-AI-ACT-2024:Art.55`.
-Article 55 covers general-purpose AI (GPAI) model obligations and transparency requirements.
-
---- references/lint-rules.md ---
-# Governance Block Lint Rules
-
-Rules applied when linting a `governance:` block in a `*.eval.yaml` file.
-The CI Action (see `examples/governance/compliance-lint/`) passes this file to Claude
-together with the governance block to extract and returns a structured report.
-
-## How to apply these rules
-
-For each `governance:` block found in a changed eval file:
-
-1. Extract the block (top-level `governance:` key, or `metadata.governance` in a test case).
-2. Apply each rule below in order.
-3. Collect all violations.
-4. Return the structured JSON report described in `SKILL.md`.
-
-A block with zero violations produces `{ "pass": true, "violations": [] }`.
-
----
-
-## Rule 1 — known_key
-
-**What:** Every key in the `governance:` object must be in the allowed-key list.
-
-**Allowed keys:** `schema_version`, `owasp_llm_top_10_2025`, `owasp_agentic_top_10_2025`,
-`mitre_atlas`, `controls`, `risk_tier`, `owner`
-
-**On violation:**
-```json
-{
-  "rule": "known_key",
-  "key": "<offending-key>",
-  "value": "<value>",
-  "message": "Unknown governance key '<offending-key>'. Did you mean '<closest-match>'?",
-  "suggestion": "Replace '<offending-key>' with '<closest-match>'."
-}
-```
-
-Common typos and their corrections:
-- `risk_level` → `risk_tier`
-- `owasp_top_10` → `owasp_llm_top_10_2025`
-- `owasp_llm` → `owasp_llm_top_10_2025`
-- `atlas` → `mitre_atlas`
-- `mitre` → `mitre_atlas`
-- `control` (singular) → `controls`
-
----
-
-## Rule 2 — owasp_llm_ids
-
-**What:** Every string in `owasp_llm_top_10_2025` must match the pattern `LLM\d{2}` (LLM01–LLM10).
-
-**On violation:**
-```json
-{
-  "rule": "owasp_llm_ids",
-  "key": "owasp_llm_top_10_2025",
-  "value": "<offending-id>",
-  "message": "Invalid OWASP LLM ID '<offending-id>'. Expected LLM01–LLM10.",
-  "suggestion": "Use a valid ID from references/owasp-llm-top-10-2025.md."
-}
-```
-
----
-
-## Rule 3 — owasp_agentic_ids
-
-**What:** Every string in `owasp_agentic_top_10_2025` must match the pattern `T\d{2}` (T01–T10).
-
-**On violation:**
-```json
-{
-  "rule": "owasp_agentic_ids",
-  "key": "owasp_agentic_top_10_2025",
-  "value": "<offending-id>",
-  "message": "Invalid OWASP Agentic ID '<offending-id>'. Expected T01–T10.",
-  "suggestion": "Use a valid ID from references/owasp-agentic-top-10-2025.md."
-}
-```
-
----
-
-## Rule 4 — mitre_atlas_ids
-
-**What:** Every string in `mitre_atlas` must match the pattern `AML\.T\d{4}(\.\d{3})?`.
-
-**On violation:**
-```json
-{
-  "rule": "mitre_atlas_ids",
-  "key": "mitre_atlas",
-  "value": "<offending-id>",
-  "message": "Invalid MITRE ATLAS ID '<offending-id>'. Expected AML.Txxxx or AML.Txxxx.xxx.",
-  "suggestion": "Check https://atlas.mitre.org/techniques/ for valid IDs."
-}
-```
-
----
-
-## Rule 5 — control_id_format
-
-**What:** Every string in `controls` must match the pattern `^[A-Z0-9][A-Z0-9_-]+-[A-Z0-9._-]+:[A-Z0-9._-]+$`
-(i.e. `<FRAMEWORK>-<VERSION>:<ID>` where all three parts are present and non-empty).
-
-Examples of valid control IDs:
-- `NIST-AI-RMF-1.0:MEASURE-2.7`
-- `EU-AI-ACT-2024:Art.55`
-- `ISO-42001-2023:6.1.2`
-- `INTERNAL-POLICY-2.1:CTRL-99`
-
-**On violation:**
-```json
-{
-  "rule": "control_id_format",
-  "key": "controls",
-  "value": "<offending-control>",
-  "message": "Malformed control ID '<offending-control>'. Expected format: <FRAMEWORK>-<VERSION>:<ID>.",
-  "suggestion": "Use the format <FRAMEWORK>-<VERSION>:<ID>, e.g. 'EU-AI-ACT-2024:Art.55'."
-}
-```
-
----
-
-## Rule 6 — risk_tier_value
-
-**What:** `risk_tier`, when present, must be one of:
-`prohibited`, `high`, `limited`, `minimal`
-
-**On violation:**
-```json
-{
-  "rule": "risk_tier_value",
-  "key": "risk_tier",
-  "value": "<offending-value>",
-  "message": "Unknown risk_tier value '<offending-value>'. Allowed: prohibited, high, limited, minimal.",
-  "suggestion": "Use one of the EU AI Act risk tiers from references/eu-ai-act-risk-tiers.md."
-}
-```
-
-Common mistakes:
-- `high_risk` → `high`
-- `limited_risk` → `limited`
-- `minimal_risk` → `minimal`
-- `low` → `minimal` (not an EU AI Act term)
-
----
-
-## Rule 7 — array_not_empty
-
-**What:** If a framework array key is present (`owasp_llm_top_10_2025`, `owasp_agentic_top_10_2025`,
-`mitre_atlas`, `controls`), it must not be an empty array.
-
-**On violation:**
-```json
-{
-  "rule": "array_not_empty",
-  "key": "<key>",
-  "value": [],
-  "message": "Empty array for '<key>'. Either populate it or remove the key.",
-  "suggestion": "Add at least one ID, or remove the key entirely."
-}
-```
-
----
-
-## Severity
-
-All rules above are **errors** (contribute to `pass: false`). There are no warnings in this
-schema — an unknown key is always wrong, and empty arrays are always wrong. This matches the
-intent: the block should only be present when it contains real, validated tags.
diff --git a/evals/agentv-dev/skills/fixtures/agentv-governance.txt b/evals/agentv-dev/skills/fixtures/agentv-governance.txt
deleted file mode 100644
index 8d1b2b73c..000000000
--- a/evals/agentv-dev/skills/fixtures/agentv-governance.txt
+++ /dev/null
@@ -1,79 +0,0 @@
----
-name: agentv-governance
-description: >-
-  Author, edit, and lint `governance:` blocks in `*.eval.yaml` files.
-  Use when creating or updating evaluation suites that carry AI-governance metadata
-  (OWASP LLM Top 10, OWASP Agentic Top 10, MITRE ATLAS, EU AI Act, ISO 42001).
-  Also use non-interactively (e.g., from a GitHub Action) to lint changed eval files
-  and report violations against the rules in `references/lint-rules.md`.
-  Do NOT use for running evals or benchmarking — that belongs to agentv-bench.
----
-
-# AgentV Compliance Skill
-
-Teaches AI agents how to author syntactically correct `governance:` blocks in AgentV
-eval files, and how to lint them against known vocabulary rules.
-
-## Dual mode
-
-**Authoring (interactive):** When a human or AI agent is editing a `*.eval.yaml` file
-that contains or should contain a `governance:` block, this skill provides vocabulary,
-valid values, and example shapes. Load it alongside `agentv-eval-writer` when building
-red-team or compliance suites.
-
-**Linting (non-interactive / CI):** When invoked from a GitHub Action (see
-`examples/governance/compliance-lint/`), this skill lints each changed `*.eval.yaml` file
-against the rules in `references/lint-rules.md` and returns a structured JSON report.
-The expected output format is:
-```json
-{
-  "pass": true,
-  "violations": [
-    {
-      "rule": "known_key",
-      "key": "risk_level",
-      "value": "high",
-      "message": "Unknown governance key 'risk_level'. Did you mean 'risk_tier'?",
-      "suggestion": "Replace 'risk_level' with 'risk_tier'."
-    }
-  ]
-}
-```
-`pass` is `true` when `violations` is empty.
-
-## Reference files
-
-| File | Purpose |
-|------|---------|
-| `references/governance-yaml-shape.md` | YAML shape, merge semantics, worked examples |
-| `references/lint-rules.md` | Machine-readable rules applied during lint |
-| `references/owasp-llm-top-10-2025.md` | LLM01–LLM10 canonical IDs and descriptions |
-| `references/owasp-agentic-top-10-2025.md` | T01–T10 agentic-AI categories |
-| `references/mitre-atlas.md` | Common AML.Txxxx technique IDs |
-| `references/eu-ai-act-risk-tiers.md` | Four risk tiers + article references |
-| `references/iso-42001-controls.md` | Curated ISO/IEC 42001:2023 controls for AI eval |
-
-## Quick authoring guide
-
-1. Check which risks this eval exercises using the reference files above.
-2. Pick IDs from the relevant frameworks (`owasp_llm_top_10_2025`, `mitre_atlas`, etc.).
-3. Set `risk_tier` using EU AI Act vocabulary (`prohibited | high | limited | minimal`).
-4. Add `controls` as `<FRAMEWORK>-<VERSION>:<ID>` strings (e.g. `EU-AI-ACT-2024:Art.55`).
-5. Run the lint rules from `references/lint-rules.md` against your block before committing.
-6. See `references/governance-yaml-shape.md` for complete examples copied from real suites.
-
-## Accessing reference files
-
-To load a specific reference without pulling the entire skill into context:
-
-```bash
-agentv skills get agentv-governance --ref lint-rules
-```
-
-Or resolve the skill directory and read files directly:
-
-```bash
-cat $(agentv skills path agentv-governance)/references/lint-rules.md
-```
-
-Use `--full` to retrieve every framework reference in one shot.
diff --git a/evals/agentv-dev/skills/fixtures/agentv-trace-analyst-full.txt b/evals/agentv-dev/skills/fixtures/agentv-trace-analyst-full.txt
deleted file mode 100644
index 47981f5fb..000000000
--- a/evals/agentv-dev/skills/fixtures/agentv-trace-analyst-full.txt
+++ /dev/null
@@ -1,161 +0,0 @@
----
-name: agentv-trace-analyst
-description: >-
-  Analyze AgentV evaluation traces and result JSONL files using `agentv inspect` and `agentv compare` CLI commands.
-  Use when asked to inspect AgentV eval results, find regressions between AgentV evaluation runs,
-  identify failure patterns in AgentV trace data, analyze tool trajectories, or compute cost/latency/score statistics
-  from AgentV result files.
-  Do NOT use for benchmarking skill trigger accuracy, analyzing skill-creator eval performance,
-  or measuring skill description quality — those tasks belong to the skill-creator skill.
----
-
-# AgentV Trace Analyst
-
-Analyze evaluation traces headlessly using `agentv inspect` primitives and `jq`.
-
-## Primitives
-
-```bash
-# List result files (most recent first)
-agentv inspect list [--limit N] [--format json|table]
-
-# Show results with trace details
-agentv inspect show <result-file> [--test-id <id>] [--tree] [--format json|table]
-
-# Percentile statistics
-agentv inspect stats <result-file> [--group-by target|suite|test-id] [--format json|table]
-
-# A/B comparison between runs
-agentv compare <baseline.jsonl> <candidate.jsonl> [--threshold 0.1] [--format json|table]
-```
-
-## Analysis Workflow
-
-### 1. Discover results
-
-```bash
-agentv inspect list
-```
-
-Pick the result file to analyze. Most recent is first.
-
-### 2. Get overview
-
-```bash
-agentv inspect stats <result-file>
-```
-
-Read the percentile table. Key signals:
-- **score p50 < 0.8**: Significant quality issues
-- **latency p90 > 30s**: Performance bottleneck
-- **cost p99 spike**: Outlier cost tests to investigate
-- **tool_calls p90 >> p50**: Some tests are much chattier
-
-### 3. Investigate failures
-
-```bash
-agentv inspect show <result-file> --format json | jq '[.[] | select(.score < 0.8) | {test_id, score, assertions: [.assertions[] | select(.passed | not)], trace: {tools: (.trace.tool_calls | keys)}, duration_ms, cost_usd}]'
-```
-
-For each failing test, examine:
-- **assertions (failed)**: What criteria were not met? (filter for `passed: false`)
-- **trace.tool_calls**: Did the agent use expected tools?
-- **duration_ms**: Did it time out or run too long?
-- **reasoning**: Why did the grader score it low?
-
-### 4. Inspect specific tests
-
-```bash
-# Flat view with trace summary
-agentv inspect show <result-file> --test-id <id>
-
-# Tree view (if output messages available)
-agentv inspect show <result-file> --test-id <id> --tree
-```
-
-The tree view shows the agent's execution path — LLM calls interspersed with tool invocations. Look for:
-- **Excessive tool calls**: Agent looping or exploring unnecessarily
-- **Missing tools**: Expected tool not called
-- **Long durations**: Specific tool calls that are slow
-
-### 5. Compare runs
-
-```bash
-agentv compare <baseline.jsonl> <candidate.jsonl>
-```
-
-Look for:
-- **Wins vs losses**: Net improvement or regression?
-- **Mean delta**: Overall direction of change
-- **Per-test deltas**: Which tests regressed?
-
-### 6. Group analysis
-
-```bash
-# By target provider
-agentv inspect stats <result-file> --group-by target
-
-# By suite
-agentv inspect stats <result-file> --group-by suite
-```
-
-Compare providers side-by-side: which is cheaper, faster, more accurate?
-
-## Advanced Queries with jq
-
-All commands support `--format json` for piping to `jq`:
-
-```bash
-# Top 3 most expensive tests
-agentv inspect show <result-file> --format json \
-  | jq 'sort_by(-.cost_usd) | .[0:3] | .[] | {test_id, cost: .cost_usd, score}'
-
-# Tests where token usage exceeds 10k
-agentv inspect show <result-file> --format json \
-  | jq '[.[] | select(.token_usage.input + .token_usage.output > 10000) | {test_id, tokens: (.token_usage.input + .token_usage.output)}]'
-
-# Score distribution by suite
-agentv inspect show <result-file> --format json \
-  | jq 'group_by(.suite) | .[] | {suite: .[0].suite, count: length, avg_score: ([.[].score] | add / length)}'
-
-# Tool usage frequency across all tests
-agentv inspect show <result-file> --format json \
-  | jq '[.[].trace.tool_calls // {} | to_entries[]] | group_by(.key) | .[] | {tool: .[0].key, total_calls: ([.[].value] | add)}'
-
-# Find regressions > 0.1 between two runs
-agentv compare baseline.jsonl candidate.jsonl --format json \
-  | jq '.matched[] | select(.delta < -0.1) | {test_id: .testId, delta, from: .score1, to: .score2}'
-```
-
-## Reasoning Patterns
-
-When analyzing traces, think about:
-
-1. **Efficiency**: Are tool calls/tokens proportional to task complexity? High tokens-per-tool may indicate verbose prompts or unnecessary context.
-
-2. **Error patterns**: Do failures cluster by target, suite, or tool usage? Common patterns:
-   - Tool errors → agent can't access required resources
-   - High LLM calls with low tool calls → agent stuck in reasoning loop
-   - Missing tool calls → wrong tool routing
-
-3. **Cost optimization**: Identify tests with high cost but acceptable scores — can they use a cheaper model? Compare `--group-by target` stats.
-
-4. **Latency distribution**: P50 vs P99 spread indicates consistency. Large spread means unpredictable performance — investigate P99 outliers.
-
-5. **Regression detection**: After a prompt/config change, compare before/after. Mean delta > 0 is good, but check individual test regressions — a few large losses can hide behind many small wins.
-
-## Accessing reference files
-
-To load a specific reference without pulling the entire skill into context:
-
-```bash
-agentv skills get agentv-trace-analyst --ref <filename>
-```
-
-Or resolve the skill directory and read files directly:
-
-```bash
-cat $(agentv skills path agentv-trace-analyst)/references/<filename>.md
-```
-
-Use `--full` to retrieve every file in the skill at once.
diff --git a/evals/agentv-dev/skills/fixtures/agentv-trace-analyst.txt b/evals/agentv-dev/skills/fixtures/agentv-trace-analyst.txt
deleted file mode 100644
index 47981f5fb..000000000
--- a/evals/agentv-dev/skills/fixtures/agentv-trace-analyst.txt
+++ /dev/null
@@ -1,161 +0,0 @@
----
-name: agentv-trace-analyst
-description: >-
-  Analyze AgentV evaluation traces and result JSONL files using `agentv inspect` and `agentv compare` CLI commands.
-  Use when asked to inspect AgentV eval results, find regressions between AgentV evaluation runs,
-  identify failure patterns in AgentV trace data, analyze tool trajectories, or compute cost/latency/score statistics
-  from AgentV result files.
-  Do NOT use for benchmarking skill trigger accuracy, analyzing skill-creator eval performance,
-  or measuring skill description quality — those tasks belong to the skill-creator skill.
----
-
-# AgentV Trace Analyst
-
-Analyze evaluation traces headlessly using `agentv inspect` primitives and `jq`.
-
-## Primitives
-
-```bash
-# List result files (most recent first)
-agentv inspect list [--limit N] [--format json|table]
-
-# Show results with trace details
-agentv inspect show <result-file> [--test-id <id>] [--tree] [--format json|table]
-
-# Percentile statistics
-agentv inspect stats <result-file> [--group-by target|suite|test-id] [--format json|table]
-
-# A/B comparison between runs
-agentv compare <baseline.jsonl> <candidate.jsonl> [--threshold 0.1] [--format json|table]
-```
-
-## Analysis Workflow
-
-### 1. Discover results
-
-```bash
-agentv inspect list
-```
-
-Pick the result file to analyze. Most recent is first.
-
-### 2. Get overview
-
-```bash
-agentv inspect stats <result-file>
-```
-
-Read the percentile table. Key signals:
-- **score p50 < 0.8**: Significant quality issues
-- **latency p90 > 30s**: Performance bottleneck
-- **cost p99 spike**: Outlier cost tests to investigate
-- **tool_calls p90 >> p50**: Some tests are much chattier
-
-### 3. Investigate failures
-
-```bash
-agentv inspect show <result-file> --format json | jq '[.[] | select(.score < 0.8) | {test_id, score, assertions: [.assertions[] | select(.passed | not)], trace: {tools: (.trace.tool_calls | keys)}, duration_ms, cost_usd}]'
-```
-
-For each failing test, examine:
-- **assertions (failed)**: What criteria were not met? (filter for `passed: false`)
-- **trace.tool_calls**: Did the agent use expected tools?
-- **duration_ms**: Did it time out or run too long?
-- **reasoning**: Why did the grader score it low?
-
-### 4. Inspect specific tests
-
-```bash
-# Flat view with trace summary
-agentv inspect show <result-file> --test-id <id>
-
-# Tree view (if output messages available)
-agentv inspect show <result-file> --test-id <id> --tree
-```
-
-The tree view shows the agent's execution path — LLM calls interspersed with tool invocations. Look for:
-- **Excessive tool calls**: Agent looping or exploring unnecessarily
-- **Missing tools**: Expected tool not called
-- **Long durations**: Specific tool calls that are slow
-
-### 5. Compare runs
-
-```bash
-agentv compare <baseline.jsonl> <candidate.jsonl>
-```
-
-Look for:
-- **Wins vs losses**: Net improvement or regression?
-- **Mean delta**: Overall direction of change
-- **Per-test deltas**: Which tests regressed?
-
-### 6. Group analysis
-
-```bash
-# By target provider
-agentv inspect stats <result-file> --group-by target
-
-# By suite
-agentv inspect stats <result-file> --group-by suite
-```
-
-Compare providers side-by-side: which is cheaper, faster, more accurate?
-
-## Advanced Queries with jq
-
-All commands support `--format json` for piping to `jq`:
-
-```bash
-# Top 3 most expensive tests
-agentv inspect show <result-file> --format json \
-  | jq 'sort_by(-.cost_usd) | .[0:3] | .[] | {test_id, cost: .cost_usd, score}'
-
-# Tests where token usage exceeds 10k
-agentv inspect show <result-file> --format json \
-  | jq '[.[] | select(.token_usage.input + .token_usage.output > 10000) | {test_id, tokens: (.token_usage.input + .token_usage.output)}]'
-
-# Score distribution by suite
-agentv inspect show <result-file> --format json \
-  | jq 'group_by(.suite) | .[] | {suite: .[0].suite, count: length, avg_score: ([.[].score] | add / length)}'
-
-# Tool usage frequency across all tests
-agentv inspect show <result-file> --format json \
-  | jq '[.[].trace.tool_calls // {} | to_entries[]] | group_by(.key) | .[] | {tool: .[0].key, total_calls: ([.[].value] | add)}'
-
-# Find regressions > 0.1 between two runs
-agentv compare baseline.jsonl candidate.jsonl --format json \
-  | jq '.matched[] | select(.delta < -0.1) | {test_id: .testId, delta, from: .score1, to: .score2}'
-```
-
-## Reasoning Patterns
-
-When analyzing traces, think about:
-
-1. **Efficiency**: Are tool calls/tokens proportional to task complexity? High tokens-per-tool may indicate verbose prompts or unnecessary context.
-
-2. **Error patterns**: Do failures cluster by target, suite, or tool usage? Common patterns:
-   - Tool errors → agent can't access required resources
-   - High LLM calls with low tool calls → agent stuck in reasoning loop
-   - Missing tool calls → wrong tool routing
-
-3. **Cost optimization**: Identify tests with high cost but acceptable scores — can they use a cheaper model? Compare `--group-by target` stats.
-
-4. **Latency distribution**: P50 vs P99 spread indicates consistency. Large spread means unpredictable performance — investigate P99 outliers.
-
-5. **Regression detection**: After a prompt/config change, compare before/after. Mean delta > 0 is good, but check individual test regressions — a few large losses can hide behind many small wins.
-
-## Accessing reference files
-
-To load a specific reference without pulling the entire skill into context:
-
-```bash
-agentv skills get agentv-trace-analyst --ref <filename>
-```
-
-Or resolve the skill directory and read files directly:
-
-```bash
-cat $(agentv skills path agentv-trace-analyst)/references/<filename>.md
-```
-
-Use `--full` to retrieve every file in the skill at once.
diff --git a/evals/agentv-dev/skills/fixtures/skills-get-nonexistent.txt b/evals/agentv-dev/skills/fixtures/skills-get-nonexistent.txt
deleted file mode 100644
index 94f8a24fb..000000000
--- a/evals/agentv-dev/skills/fixtures/skills-get-nonexistent.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-Error: skill 'does-not-exist' not found
-Available skills: agentv-bench, agentv-eval-review, agentv-eval-writer, agentv-governance, agentv-trace-analyst
diff --git a/evals/agentv-dev/skills/fixtures/skills-list-all.txt b/evals/agentv-dev/skills/fixtures/skills-list-all.txt
deleted file mode 100644
index a457dcc46..000000000
--- a/evals/agentv-dev/skills/fixtures/skills-list-all.txt
+++ /dev/null
@@ -1 +0,0 @@
-{"success":true,"data":["agentv-bench","agentv-eval-review","agentv-eval-writer","agentv-governance","agentv-trace-analyst"]}
diff --git a/evals/agentv-dev/skills/output-correctness.eval.yaml b/evals/agentv-dev/skills/output-correctness.eval.yaml
index 49dc121fd..054aa8646 100644
--- a/evals/agentv-dev/skills/output-correctness.eval.yaml
+++ b/evals/agentv-dev/skills/output-correctness.eval.yaml
@@ -1,9 +1,9 @@
 name: agentv-dev-output-correctness
-
 description: >-
   Tests whether agent-produced output is structurally and factually correct
-  given a bundled-skill fixture as ground truth — covers eval YAML structure,
-  CLI command syntax, fixture-content accuracy, and grader configuration.
+  given live AgentV skill and docs content as ground truth — covers eval YAML
+  structure, CLI command syntax, skill-content accuracy, and grader
+  configuration.
 
 tags: [agent, skills]
 
@@ -14,10 +14,10 @@ tests:
       - role: user
         content:
           - type: file
-            value: fixtures/agentv-eval-writer.txt
+            value: /skills-data/agentv-eval-writer/SKILL.md
           - type: text
             value: |
-              Using the agentv-eval-writer skill above as your reference,
+              Using the live agentv-eval-writer skill above as your reference,
               produce a minimal but valid `.eval.yaml` snippet (one `description`
               and one test in `tests:`) for an eval that grades whether an LLM
               correctly summarises a paragraph.
@@ -43,11 +43,11 @@ tests:
       - role: user
         content:
           - type: file
-            value: fixtures/agentv-eval-writer.txt
+            value: /skills-data/agentv-eval-writer/SKILL.md
           - type: text
             value: |
-              Using the agentv-eval-writer skill as reference, write a single
-              test entry under `tests:` that uses **three** assertions:
+              Using the live agentv-eval-writer skill as reference, write a
+              single test entry under `tests:` that uses **three** assertions:
               one `contains`, one `regex`, and one `rubrics` (with at least
               two criteria). Output only the YAML for the test entry.
     assertions:
@@ -60,8 +60,8 @@ tests:
       - type: rubrics
         criteria:
           - "Each assertion has both a `type:` and the field that type expects (`value:` for contains/regex, `criteria:` list for rubrics)"
-          - Does not invent assertion types that are not in the agentv-eval-writer skill (e.g., `like`, `match`, `expect`)
-          - Uses kebab-case (e.g., `llm-grader`, `is-json`) not snake_case (e.g., `llm_grader`) for any multi-word grader types referenced
+          - Does not invent assertion types that are not in the agentv-eval-writer skill (for example `like`, `match`, or `expect`)
+          - Uses kebab-case (for example `llm-grader`, `is-json`) not snake_case (for example `llm_grader`) for any multi-word grader types referenced
 
   - id: cli-command-correct-syntax
     criteria: Agent produces correct `agentv skills` CLI commands with valid subcommands and flags.
@@ -69,13 +69,14 @@ tests:
       - role: user
         content:
           - type: file
-            value: fixtures/agentv-bench.txt
+            value: /apps/web/src/content/docs/docs/getting-started/installation.mdx
           - type: text
             value: |
-              Give me three shell commands, one per line, that exercise the
+              Using the live installation doc above as your reference, give me
+              three shell commands, one per line, that exercise the
               `agentv skills` subcommand:
               1. List all bundled skills as JSON.
-              2. Print the agentv-bench skill including its references and templates.
+              2. Print the agentv-bench skill including its bundled support files.
               3. Print the on-disk path of the agentv-bench skill.
     assertions:
       - type: regex
@@ -85,16 +86,16 @@ tests:
       - type: regex
         value: "agentv\\s+skills\\s+path\\s+agentv-bench"
 
-  - id: fixture-content-accurate
-    criteria: Agent's description of a skill matches the skill's actual SKILL.md content.
+  - id: skill-content-accurate
+    criteria: Agent's description of a skill matches the skill's live SKILL.md content.
     input:
       - role: user
         content:
           - type: file
-            value: fixtures/agentv-trace-analyst.txt
+            value: /skills-data/agentv-trace-analyst/SKILL.md
           - type: text
             value: |
-              Read the agentv-trace-analyst skill above. In one paragraph,
+              Read the live agentv-trace-analyst skill above. In one paragraph,
               describe what this skill does and when it should be invoked.
               Quote at most one short phrase from the skill text; do not
               invent capabilities the skill does not mention.
@@ -103,20 +104,21 @@ tests:
         criteria:
           - Description accurately reflects what agentv-trace-analyst does (analysing eval traces and failure modes)
           - Does not claim the skill writes or runs evals (those are agentv-eval-writer and agentv-bench respectively)
-          - Any quoted phrase appears verbatim in the fixture
-          - Does not hallucinate sub-features not present in the SKILL.md
+          - Any quoted phrase appears verbatim in the provided SKILL.md
+          - Does not hallucinate sub-features not present in the skill
 
-  - id: skill-list-complete
-    criteria: Agent lists all five bundled skills when given the JSON fixture.
+  - id: plugin-skill-list-complete
+    criteria: Agent lists all bundled skills from the live `agentv-dev` plugin stub.
     input:
       - role: user
         content:
           - type: file
-            value: fixtures/skills-list-all.txt
+            value: /plugins/agentv-dev/skills/agentv-dev/SKILL.md
           - type: text
             value: |
-              The JSON above is the output of `agentv skills list --json`.
-              Reproduce the list of skill names as a markdown bulleted list.
+              The file above is the live `agentv-dev` plugin entrypoint.
+              Reproduce the list of available bundled skill names as a markdown
+              bulleted list.
     assertions:
       - type: contains
         value: agentv-bench
@@ -130,27 +132,25 @@ tests:
         value: agentv-trace-analyst
       - type: rubrics
         criteria:
-          - Lists exactly the five skill names from the JSON, no more and no fewer
-          - Does not invent additional skill names not present in the fixture
+          - Lists exactly the five skill names from the live plugin file, no more and no fewer
+          - Does not invent additional skill names not present in the plugin file
 
-  - id: error-message-accurate
-    criteria: Agent correctly interprets the not-found error and proposes a recoverable next step.
+  - id: governance-ref-command-accurate
+    criteria: Agent gives the correct command for loading a specific governance reference file.
     input:
       - role: user
         content:
           - type: file
-            value: fixtures/skills-get-nonexistent.txt
+            value: /skills-data/agentv-governance/SKILL.md
           - type: text
             value: |
-              The output above came from running `agentv skills get does-not-exist`.
-              In two sentences, explain (a) what error the user hit and
-              (b) the single most useful command to run next.
+              Using the live agentv-governance skill above as reference, what
+              single command should I run to load the `lint-rules` reference
+              file without pulling in the whole skill?
+              Reply with the command on its own line.
     assertions:
-      - type: rubrics
-        criteria:
-          - Identifies that `does-not-exist` is not a recognised skill name
-          - "Proposes either `agentv skills list` or selecting one of the names from the error's \"Available skills:\" line"
-          - Does not blame missing installation or recommend re-running `npm install -g agentv`
+      - type: regex
+        value: "agentv\\s+skills\\s+get\\s+agentv-governance\\s+--ref\\s+lint-rules"
 
   - id: grader-config-valid
     criteria: Agent configures an llm-grader with the fields the AgentV registry expects.
@@ -158,19 +158,19 @@ tests:
       - role: user
         content:
           - type: file
-            value: fixtures/agentv-eval-writer.txt
+            value: /skills-data/agentv-eval-writer/SKILL.md
           - type: text
             value: |
-              Using the agentv-eval-writer skill as reference, produce a YAML
-              snippet for **one** assertion that uses `type: llm-grader` to
-              grade an answer's correctness. Include whatever fields the
-              skill says an `llm-grader` requires (e.g., target/model, rubric
-              criteria). Output only the YAML for the single assertion.
+              Using the live agentv-eval-writer skill as reference, produce a
+              YAML snippet for **one** assertion that uses `type: llm-grader`
+              to grade an answer's correctness. Include whatever fields the
+              skill says an `llm-grader` requires (for example target/model or
+              rubric criteria). Output only the YAML for the single assertion.
     assertions:
       - type: regex
         value: "type:\\s+llm-grader"
       - type: rubrics
         criteria:
           - "Uses `type: llm-grader` (kebab-case), not `type: llm_grader` or `type: llmGrader`"
-          - Includes the fields the skill documents for llm-grader (typically a rubric/criteria field; may also include a target/model)
-          - Does not invent fields not mentioned in the skill (e.g., a fabricated `model_provider` field)
+          - Includes the fields the skill documents for llm-grader (typically a rubric or criteria field; may also include a target or model)
+          - Does not invent fields not mentioned in the skill (for example a fabricated `model_provider` field)
diff --git a/evals/agentv-dev/skills/skill-invocation.eval.yaml b/evals/agentv-dev/skills/skill-invocation.eval.yaml
index dd097f2a4..6fddee7b2 100644
--- a/evals/agentv-dev/skills/skill-invocation.eval.yaml
+++ b/evals/agentv-dev/skills/skill-invocation.eval.yaml
@@ -1,12 +1,23 @@
 name: agentv-dev-skill-invocation
-
 description: >-
   Tests whether an AI agent correctly invokes `agentv skills` CLI subcommands
   (list, get, path) with the right flags when asked to retrieve, enumerate,
-  or locate bundled skill content.
+  or locate bundled skill content using the live repo docs and plugin stub.
 
 tags: [agent, skills]
 
+input:
+  - role: user
+    content:
+      - type: file
+        value: /plugins/agentv-dev/skills/agentv-dev/SKILL.md
+      - type: file
+        value: /apps/web/src/content/docs/docs/getting-started/installation.mdx
+      - type: text
+        value: |
+          The files above are the live repo references for bundled `agentv-dev`
+          skills and the `agentv skills` CLI.
+
 tests:
   - id: invoke-get-single-skill
     criteria: Agent invokes `agentv skills get <name>` for a single named skill.
@@ -25,11 +36,10 @@ tests:
           - Does not suggest a plugin-install or marketplace step
 
   - id: invoke-get-full-flag
-    criteria: Agent uses `--full` when bundled references/templates are needed.
+    criteria: Agent uses `--full` when bundled references or support files are needed.
     input: |
-      I need the full agentv-eval-writer skill — not just SKILL.md, but every
-      file under its `references/` and `templates/` directories — so I can see
-      the example YAML templates the skill ships with.
+      I need the full agentv-eval-writer skill — not just SKILL.md, but the
+      bundled supporting files too — so I can inspect its reference material.
 
       What single `agentv skills` command should I run?
     assertions:
@@ -52,25 +62,16 @@ tests:
           - Does not invoke `npx allagents plugin` or any marketplace step
 
   - id: invoke-get-nonexistent
-    criteria: Agent diagnoses a not-found error from the CLI and proposes a fix.
-    input:
-      - role: user
-        content:
-          - type: text
-            value: |
-              I ran `agentv skills get does-not-exist` and got this output:
-          - type: file
-            value: fixtures/skills-get-nonexistent.txt
-          - type: text
-            value: |
-              What happened, and what command should I run instead to discover
-              the correct skill name?
+    criteria: Agent diagnoses a not-found skill lookup and proposes a fix.
+    input: |
+      I ran `agentv skills get does-not-exist`. What happened, and what command
+      should I run instead to discover the correct skill name?
     assertions:
       - type: rubrics
         criteria:
           - Identifies that `does-not-exist` is not a valid skill name
           - Recommends `agentv skills list` to discover valid names
-          - Does not invent a skill name that isn't in the error output's "Available skills" line
+          - Does not invent a skill name that is not listed in the live plugin entrypoint
 
   - id: invoke-json-flag
     criteria: Agent uses `--json` when machine-readable output is needed.
@@ -112,16 +113,9 @@ tests:
 
   - id: invoke-skill-trigger
     criteria: Agent maps a natural-language task to the correct skill name and command.
-    input:
-      - role: user
-        content:
-          - type: file
-            value: fixtures/skills-list-all.txt
-          - type: text
-            value: |
-              Above is the JSON list of bundled agentv skills. I want to read
-              the skill that helps with debugging eval failures and analysing
-              traces. Give me the single CLI command to print its content.
+    input: |
+      I want to read the skill that helps with debugging eval failures and
+      analysing traces. Give me the single CLI command to print its content.
     assertions:
       - type: contains
         value: agentv-trace-analyst
diff --git a/evals/agentv-dev/skills/skill-selection.eval.yaml b/evals/agentv-dev/skills/skill-selection.eval.yaml
index 075cf225b..de6572532 100644
--- a/evals/agentv-dev/skills/skill-selection.eval.yaml
+++ b/evals/agentv-dev/skills/skill-selection.eval.yaml
@@ -1,22 +1,19 @@
 name: agentv-dev-skill-selection
 description: >-
   Tests whether the agent picks the correct bundled agentv skill (by name)
-  for a given natural-language task. Each test injects the bundled skills
-  list as context so the agent has the same options a real user would.
+  for a given natural-language task using the live `agentv-dev` plugin stub
+  as the source of truth for available skills.
 
 tags: [agent, skills]
 
-# Suite-level input: every test sees the bundled skills list as the first
-# user message, so the agent always has the canonical set of names to
-# choose from. Per-test `input:` is appended to this.
 input:
   - role: user
     content:
       - type: file
-        value: fixtures/skills-list-all.txt
+        value: /plugins/agentv-dev/skills/agentv-dev/SKILL.md
       - type: text
         value: |
-          The JSON above lists every agentv skill bundled with the CLI.
+          The file above is the live `agentv-dev` plugin skill entrypoint.
           For the next task, name the **single** skill (or skills, only if
           the task genuinely requires more than one) that best fits.
           Reply with the skill name(s) only — no commentary.
diff --git a/evals/agentv-self/scripts/setup.mjs b/evals/agentv-self/scripts/setup.mjs
index bf9fe41f1..8637363c5 100644
--- a/evals/agentv-self/scripts/setup.mjs
+++ b/evals/agentv-self/scripts/setup.mjs
@@ -8,7 +8,7 @@
  */
 
 import { execSync } from 'node:child_process';
-import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync } from 'node:fs';
+import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, rmSync } from 'node:fs';
 import { join } from 'node:path';
 
 const stdin = readFileSync(0, 'utf8');

From df90eeab1e91428e8f95ae7b9bf1ae761ce9e22d Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Fri, 19 Jun 2026 10:17:42 +0200
Subject: [PATCH 3/3] fix(evals): stabilize agentv skill evals

---
 evals/agentv-dev/skills/skill-selection.eval.yaml |  5 ++++-
 evals/agentv-self/scripts/setup.mjs               | 15 +++++++++------
 plugins/agentv-dev/skills/agentv-dev/SKILL.md     |  3 ++-
 3 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/evals/agentv-dev/skills/skill-selection.eval.yaml b/evals/agentv-dev/skills/skill-selection.eval.yaml
index de6572532..ec8754e72 100644
--- a/evals/agentv-dev/skills/skill-selection.eval.yaml
+++ b/evals/agentv-dev/skills/skill-selection.eval.yaml
@@ -16,7 +16,8 @@ input:
           The file above is the live `agentv-dev` plugin skill entrypoint.
           For the next task, name the **single** skill (or skills, only if
           the task genuinely requires more than one) that best fits.
-          Reply with the skill name(s) only — no commentary.
+          Reply with the skill name(s) only, or `none` if no listed skill fits.
+          No commentary.
 
 tests:
   - id: select-bench-for-eval-run
@@ -120,6 +121,8 @@ tests:
       Task: I want to deploy my Node.js app to a Kubernetes cluster. Which
       agentv skill should I load?
     assertions:
+      - type: equals
+        value: none
       - type: rubrics
         criteria:
           - Correctly states that no bundled agentv skill is relevant — these skills are for evaluation workflows, not deployment
diff --git a/evals/agentv-self/scripts/setup.mjs b/evals/agentv-self/scripts/setup.mjs
index 8637363c5..cfb8f119a 100644
--- a/evals/agentv-self/scripts/setup.mjs
+++ b/evals/agentv-self/scripts/setup.mjs
@@ -7,8 +7,8 @@
  * repository without declaring extra repos in workspace config.
  */
 
-import { execSync } from 'node:child_process';
-import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, rmSync } from 'node:fs';
+import { execFileSync, execSync } from 'node:child_process';
+import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync } from 'node:fs';
 import { join } from 'node:path';
 
 const stdin = readFileSync(0, 'utf8');
@@ -28,10 +28,13 @@ for (const entry of readdirSync(workspacePath)) {
   rmSync(join(workspacePath, entry), { recursive: true, force: true });
 }
 
-for (const entry of readdirSync(repoRoot)) {
-  if (entry === '.git') continue;
-  cpSync(join(repoRoot, entry), join(workspacePath, entry), { recursive: true });
-}
+const archivePath = join(workspacePath, 'agentv-repo.tar');
+execFileSync('git', ['archive', '--format=tar', `--output=${archivePath}`, 'HEAD'], {
+  cwd: repoRoot,
+  stdio: ['ignore', 'ignore', 'inherit'],
+});
+execFileSync('tar', ['-xf', archivePath, '-C', workspacePath], { stdio: 'inherit' });
+rmSync(archivePath, { force: true });
 
 if (!existsSync(join(workspacePath, 'AGENTS.md')) || !existsSync(join(workspacePath, '.agents'))) {
   console.error('expected AGENTS.md and .agents to be copied into the workspace');
diff --git a/plugins/agentv-dev/skills/agentv-dev/SKILL.md b/plugins/agentv-dev/skills/agentv-dev/SKILL.md
index 75b6d2b1a..92c33628f 100644
--- a/plugins/agentv-dev/skills/agentv-dev/SKILL.md
+++ b/plugins/agentv-dev/skills/agentv-dev/SKILL.md
@@ -9,7 +9,8 @@ description: >-
 ---
 
 The full skill content is bundled with the AgentV CLI and always version-matched to it.
-Load the specific skill you need:
+Load the specific skill you need. If the task is not about AgentV evals, trace
+analysis, optimization, or governance, no AgentV skill applies.
 
 ```bash
 agentv skills get <skill-name>