feat(eval): canonicalize repeat policy schema

Author: christso

README.md

@@ -19,7 +19,7 @@ Test AI targets on real repo tasks and measure what actually works.
 - **Workspace / fixtures / graders** are task-owned context: repos, setup scripts, files, fixtures, isolation, deterministic checks, and LLM grading prompts.
 - **Target** is the system under test: an agent, provider, gateway, replay target, CLI wrapper, transcript provider, or future app/service wrapper. Each eval selects one `target`, either by name from `targets.yaml` or with an eval-local target object.
 - **Experiment** is the run/result grouping label being measured over that corpus, such as `backend-with-skills` or `backend-without-skills`.
-- **Run controls** configure repeats, exits, timeouts, budgets, thresholds, and completion hooks with top-level fields such as `runs`, `early_exit`, `timeout_seconds`, `budget_usd`, `threshold`, and `on_run_complete`.
+- **Run controls** configure repeats, timeouts, budgets, thresholds, and completion hooks with fields such as `repeat`, `timeout_seconds`, `budget_usd`, `threshold`, and `on_run_complete`.
 - **Run** is one concrete execution of an experiment against a resolved target that writes portable artifacts for readers such as Dashboard, compare, and trend.
 
 ```mermaid
@@ -60,8 +60,10 @@ agentv init
 description: Code generation quality
 experiment: backend-with-skills
 target: copilot-sdk
-runs: 3
-early_exit: false
+repeat:
+  count: 3
+  strategy: pass_any
+  early_exit: false
 timeout_seconds: 600
 threshold: 0.8
 budget_usd: 5
@@ -91,7 +93,9 @@ target:
   extends: codex-gpt5
   model: gpt-5.1
   reasoning_effort: high
-runs: 2
+repeat:
+  count: 2
+  strategy: pass_any
 timeout_seconds: 900
 threshold: 0.85
 
@@ -186,8 +190,11 @@ export default defineEval({
     extends: 'copilot-sdk',
     model: 'claude-sonnet-4.6',
   },
-  runs: 3,
-  earlyExit: false,
+  repeat: {
+    count: 3,
+    strategy: 'pass_any',
+    earlyExit: false,
+  },
   timeoutSeconds: 600,
   threshold: 0.8,
   budgetUsd: 5,

apps/cli/src/commands/eval/run-eval.ts

@@ -803,17 +803,10 @@ function buildExperimentTrialsConfig(experiment: ExperimentConfig): TrialsConfig
       ...(experiment.repeat.costLimitUsd !== undefined && {
         costLimitUsd: experiment.repeat.costLimitUsd,
       }),
-      ...(experiment.earlyExit !== undefined && { earlyExit: experiment.earlyExit }),
+      ...(experiment.repeat.earlyExit !== undefined && { earlyExit: experiment.repeat.earlyExit }),
     };
   }
-  if (!experiment.runs || experiment.runs <= 1) {
-    return undefined;
-  }
-  return {
-    count: experiment.runs,
-    strategy: 'pass_at_k',
-    ...(experiment.earlyExit !== undefined && { earlyExit: experiment.earlyExit }),
-  };
+  return undefined;
 }
 
 type EffectiveRunPolicy = {

apps/cli/test/commands/eval/artifact-writer.test.ts

@@ -177,7 +177,7 @@ describe('buildGradingArtifact', () => {
         },
       ],
       aggregation: {
-        strategy: 'pass_at_k',
+        strategy: 'pass_any',
         passedAttempts: 1,
         totalAttempts: 2,
       },
@@ -202,7 +202,7 @@ describe('buildGradingArtifact', () => {
       },
     ]);
     expect(grading.aggregation).toEqual({
-      strategy: 'pass_at_k',
+      strategy: 'pass_any',
       passed_attempts: 1,
       total_attempts: 2,
     });

apps/cli/test/eval.integration.test.ts

@@ -611,7 +611,10 @@ describe('agentv eval CLI', () => {
           'timeout_seconds: 12',
           'threshold: 0.8',
           'budget_usd: 3',
-          'runs: 2',
+          'repeat:',
+          '  count: 2',
+          '  strategy: pass_any',
+          '  early_exit: true',
           'tests:',
           '  - include: sample.test.yaml',
           '    type: suite',
@@ -623,6 +626,7 @@ describe('agentv eval CLI', () => {
           '      repeat:',
           '        count: 3',
           '        strategy: pass_all',
+          '        early_exit: true',
           '',
         ].join('\n'),
         'utf8',
@@ -646,6 +650,7 @@ describe('agentv eval CLI', () => {
         trials: {
           count: 3,
           strategy: 'pass_all',
+          earlyExit: true,
         },
       });
 
@@ -655,7 +660,11 @@ describe('agentv eval CLI', () => {
       expect(benchmark.metadata?.experiment).toBe('native-exp');
       expect(benchmark.metadata?.experiment_config).toMatchObject({
         target: 'codex-target',
-        runs: 2,
+        repeat: {
+          count: 2,
+          strategy: 'pass_any',
+          early_exit: true,
+        },
         threshold: 0.8,
         budget_usd: 3,
         timeout_seconds: 12,

apps/web/src/content/docs/docs/evaluation/eval-files.mdx

@@ -5,7 +5,7 @@ sidebar:
   order: 1
 ---
 
-Evaluation files define the test cases, graders, workspace lifecycle, and run controls for an evaluation run. Top-level `experiment` is the run/result grouping label, top-level `target` identifies the system under test, and flat fields such as `runs`, `threshold`, `timeout_seconds`, and `budget_usd` control repeated attempts and gates. Workspace reuse belongs under `workspace.isolation`; Docker/container binding belongs under `workspace.docker`. Install, build, and reset commands belong under `workspace.hooks`; runner-specific setup belongs in the `target` object or `targets.yaml`. AgentV supports two eval data formats: YAML and JSONL.
+Evaluation files define the test cases, graders, workspace lifecycle, and run controls for an evaluation run. Top-level `experiment` is the run/result grouping label, top-level `target` identifies the system under test, and fields such as `repeat`, `threshold`, `timeout_seconds`, and `budget_usd` control repeated attempts and gates. Workspace reuse belongs under `workspace.isolation`; Docker/container binding belongs under `workspace.docker`. Install, build, and reset commands belong under `workspace.hooks`; runner-specific setup belongs in the `target` object or `targets.yaml`. AgentV supports two eval data formats: YAML and JSONL.
 
 YAML is the canonical portable model. TypeScript helpers, generated fixtures, and Python scripts should lower to the same YAML/JSONL shapes rather than inventing a separate eval contract.
 Eval files describe the task, target binding, and run controls. Concurrency is an operator/run setting: pass `--workers` or set `execution.workers` in `agentv.config.*` / `.agentv/config.yaml` instead of authoring `workers` in eval YAML.
@@ -24,7 +24,7 @@ experiment format.
   it with `imports.tests`, `tests: ./cases.yaml`, or string shorthand; parent
   suite context applies because raw cases do not carry their own suite context.
 - A **wrapper eval** is eval YAML that imports one or more suites with
-  `imports.suites` and binds run controls with top-level `target`, `runs`,
+  `imports.suites` and binds run controls with top-level `target`, `repeat`,
   `threshold`, `timeout_seconds`, and `budget_usd`.
   Wrapper evals can live anywhere in the repo. A wrapper that imports suites
   with `imports.suites` must not define parent `workspace`; imported suites own
@@ -64,7 +64,9 @@ A wrapper eval stays ordinary eval YAML while choosing a target and run controls
 # experiments/refunds-codex.eval.yaml
 name: refunds-codex
 target: codex-gpt5
-runs: 2
+repeat:
+  count: 2
+  strategy: pass_any
 
 imports:
   suites:
@@ -115,13 +117,12 @@ tests:
 | `category` | Optional slash-delimited analytics taxonomy path. Overrides the category derived from the eval file path. |
 | `target` | Named system under test from `.agentv/targets.yaml` or `--targets` |
 | `experiment` | Optional run/result grouping label |
-| `runs` | Optional number of attempts per case |
-| `early_exit` | Optional early exit for repeated attempts |
+| `repeat` | Optional repeat policy with `count`, `strategy`, and `early_exit` |
 | `timeout_seconds` | Optional per-case timeout |
 | `budget_usd` | Optional suite budget |
 | `threshold` | Optional suite quality threshold |
 | `workspace` | Suite-level task environment — inline object or string path to an [external workspace file](/docs/guides/workspace-pool/#external-workspace-config). Repo entries declare identity and checkout pins; acquisition is covered in [Workspace Architecture](/docs/guides/workspace-architecture/#repo-provenance-vs-acquisition). |
-| `imports` | Optional import groups. `imports.suites` imports full child eval suites with their task context. `imports.tests` imports raw test rows into this file's context. Import entries may use scoped `run:` overrides for `threshold`, `timeout_seconds`, and `budget_usd`. |
+| `imports` | Optional import groups. `imports.suites` imports full child eval suites with their task context. `imports.tests` imports raw test rows into this file's context. Import entries may use scoped `run:` overrides for `threshold`, `repeat`, `timeout_seconds`, and `budget_usd`. |
 | `tests` | Inline raw tests or a string path to an external raw-case file or directory. Legacy `tests[].include` entries still load with a migration warning; prefer `imports.suites` or `imports.tests`. |
 | `assertions` | Suite-level graders appended to each test unless `execution.skip_defaults: true` is set on the test |
 | `input` | Suite-level input messages prepended to each test's input unless `execution.skip_defaults: true` is set on the test |

apps/web/src/content/docs/docs/evaluation/experiments.mdx

@@ -8,7 +8,7 @@ sidebar:
 AgentV eval files are the runnable authoring artifact. Use top-level
 `description` for display metadata, `experiment` as the run/result grouping
 label, `target` for the system under test, and flat top-level run controls such
-as `runs`, `timeout_seconds`, `budget_usd`, and `threshold`.
+as `repeat`, `timeout_seconds`, `budget_usd`, and `threshold`.
 Concurrency is outside eval YAML. Use `agentv eval --workers N` or project
 config defaults such as `agentv.config.*` / `.agentv/config.yaml`
 `execution.workers` for operator-side parallelism.
@@ -21,7 +21,9 @@ target:
   extends: codex-gpt5
   model: gpt-5.1
   reasoning_effort: high
-runs: 4
+repeat:
+  count: 4
+  strategy: pass_any
 timeout_seconds: 720
 budget_usd: 2.00
 
@@ -151,7 +153,9 @@ test.run > import run > parent top-level run controls
 ```yaml
 target: agent
 threshold: 0.8
-runs: 3
+repeat:
+  count: 3
+  strategy: pass_any
 
 imports:
   suites:
@@ -177,10 +181,10 @@ tests:
       budget_usd: 0.50
 ```
 
-Scoped `run:` supports `threshold`, `timeout_seconds`, and `budget_usd` for
-public eval authoring. Candidate-changing fields stay parent-level. Workspace
-mutation belongs in `workspace.hooks`, and provider-specific setup belongs in
-target configuration.
+Scoped `run:` supports `threshold`, `repeat`, `timeout_seconds`, and
+`budget_usd` for public eval authoring. Candidate-changing fields stay
+parent-level. Workspace mutation belongs in `workspace.hooks`, and
+provider-specific setup belongs in target configuration.
 
 ## Lifecycle Ownership
 
@@ -194,7 +198,7 @@ target-specific runner state.
 | Configure an agent runner or provider variant | `target` object or `targets.yaml` |
 | Choose the target | top-level `target` |
 | Override the target's default model | `target.model` |
-| Configure run count, budget, timeout, threshold | top-level `runs`, `budget_usd`, `timeout_seconds`, `threshold` |
+| Configure repeat policy, budget, timeout, threshold | top-level `repeat`, `budget_usd`, `timeout_seconds`, `threshold` |
 | Bind an existing local workspace directory | `--workspace-path` or `.agentv/config.local.yaml` |
 
 ```yaml
@@ -208,7 +212,9 @@ target:
   hooks:
     before_each:
       command: ["sh", "-c", "cp -R skills \"{{workspace_path}}/.codex/skills\""]
-runs: 3
+repeat:
+  count: 3
+  strategy: pass_any
 ```
 
 Existing local workspace paths are machine-local bindings: pass
@@ -219,16 +225,21 @@ top-level or case-level `workspace`.
 
 ## Repeat Runs
 
-Use top-level `runs` when you want AgentV to try each case more than once:
+Use top-level `repeat` when you want AgentV to try each case more than once:
 
 ```yaml
-runs: 3
+repeat:
+  count: 3
+  strategy: pass_any
+  early_exit: false
 ```
 
-The current repeated-attempt aggregation treats the case as successful when any
-attempt succeeds. AgentV does not expose this as `pass@k`; in ML and
-code-generation evaluation, pass@k is a statistical metric with different
-semantics.
+`repeat.strategy` controls verdict aggregation. `pass_any` treats the case as
+successful when any completed attempt passes; `pass_all` requires every
+completed attempt to pass. `mean` and `confidence_interval` aggregate scores
+where supported today. `repeat.early_exit` is only a scheduling and cost
+optimization: `pass_any` may stop at the first pass, and `pass_all` may stop at
+the first fail. Leave it unset or `false` when you want complete variance data.
 
 ## Result Layout
 

examples/features/README.md

@@ -81,7 +81,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the
 | Example | Description |
 |---------|-------------|
 | [benchmark-tooling](benchmark-tooling/) | N-way benchmarking with `agentv compare` over completed runs |
-| [trials](trials/) | Configure repeated attempts with `runs` |
+| [trials](trials/) | Configure repeated attempts with `repeat` |
 | [trial-output-consistency](trial-output-consistency/) | Measure output consistency across trials using pairwise cosine similarity |
 | [compare](compare/) | Compare a run against a stored baseline |
 

examples/features/trials/README.md

@@ -1,7 +1,7 @@
 # Repeat Runs
 
 This example keeps the runnable contract in one eval file. Top-level `target`
-selects the system under test and top-level `runs` configures repeated attempts.
+selects the system under test and top-level `repeat` configures repeated attempts.
 
 ## Files
 
@@ -13,9 +13,12 @@ selects the system under test and top-level `runs` configures repeated attempts.
 bun agentv eval examples/features/trials/evals/dataset.eval.yaml
 ```
 
-Edit `runs` to change how many attempts AgentV makes for each case:
+Edit `repeat.count` to change how many attempts AgentV makes for each case:
 
 ```yaml
-runs: 2
+repeat:
+  count: 2
+  strategy: pass_any
+  early_exit: false
 budget_usd: 1.00
 ```

examples/features/trials/evals/dataset.eval.baseline.jsonl

@@ -1,2 +1,2 @@
-{"timestamp":"2026-02-20T21:40:25.928Z","test_id":"capital-knowledge","suite":"dataset","score":1,"target":"default","trials":[{"attempt":0,"score":1,"verdict":"pass"}],"aggregation":{"strategy":"pass_at_k","passed_attempts":1,"total_attempts":1},"assertions":[{"text":"Correctly identifies Canberra as the capital of Australia","passed":true,"evidence":"The candidate answer provides the correct and complete information, fully matching the reference answer."}]}
-{"timestamp":"2026-02-20T21:40:26.593Z","test_id":"math-basics","suite":"dataset","score":1,"target":"default","trials":[{"attempt":0,"score":1,"verdict":"pass"}],"aggregation":{"strategy":"pass_at_k","passed_attempts":1,"total_attempts":1},"assertions":[{"text":"Explains step-by-step reasoning","passed":true,"evidence":"The candidate answer breaks down the calculation clearly, explains each step, and arrives at the correct answer, matching the reference reasoning."},{"text":"Splits 15 into 10 and 5 for easier calculation","passed":true},{"text":"Calculates partial products (10\u00d77 and 5\u00d77)","passed":true},{"text":"Arrives at correct final answer (105)","passed":true}]}
+{"timestamp":"2026-02-20T21:40:25.928Z","test_id":"capital-knowledge","suite":"dataset","score":1,"target":"default","trials":[{"attempt":0,"score":1,"verdict":"pass"},{"attempt":1,"score":1,"verdict":"pass"}],"aggregation":{"strategy":"pass_any","passed_attempts":2,"total_attempts":2},"assertions":[{"text":"Correctly identifies Canberra as the capital of Australia","passed":true,"evidence":"The candidate answer provides the correct and complete information, fully matching the reference answer."}]}
+{"timestamp":"2026-02-20T21:40:26.593Z","test_id":"math-basics","suite":"dataset","score":1,"target":"default","trials":[{"attempt":0,"score":1,"verdict":"pass"},{"attempt":1,"score":1,"verdict":"pass"}],"aggregation":{"strategy":"pass_any","passed_attempts":2,"total_attempts":2},"assertions":[{"text":"Explains step-by-step reasoning","passed":true,"evidence":"The candidate answer breaks down the calculation clearly, explains each step, and arrives at the correct answer, matching the reference reasoning."},{"text":"Splits 15 into 10 and 5 for easier calculation","passed":true},{"text":"Calculates partial products (10\u00d77 and 5\u00d77)","passed":true},{"text":"Arrives at correct final answer (105)","passed":true}]}

examples/features/trials/evals/dataset.eval.yaml

@@ -1,11 +1,14 @@
 # AgentV Repeat Runs Example
-# Demonstrates policy runs for handling LLM non-determinism
+# Demonstrates repeat policy for handling LLM non-determinism
 
 name: trials
 description: Repeat runs example with 2 attempts configured inline
 
 target: llm
-runs: 2
+repeat:
+  count: 2
+  strategy: pass_any
+  early_exit: false
 budget_usd: 1.00
 
 tests: