Add verified winners and baseline setup

README.md

@@ -36,7 +36,11 @@ Choosing a coding agent from anecdotes is hard. PatchLeague gives several agents
 - a self-contained offline HTML report;
 - a guarded, preview-first path for applying the winner.
 
-PatchLeague uses your existing CLI subscriptions and authentication. It does not proxy prompts through its own service and does not upload source code.
+PatchLeague uses your existing CLI subscriptions and authentication. PatchLeague
+itself does not proxy prompts through its own service or upload source code.
+However, the agent CLIs you configure may send prompts, repository context, or
+source code to their providers. Review each provider's data-handling policy
+before running it on private or sensitive repositories.
 
 ## Quick start
 
@@ -106,6 +110,7 @@ Run `patchleague init` in a Git repository to create `.patchleague/config.json`:
   "defaultAgents": ["codex", "claude"],
   "maxParallel": 2,
   "timeoutMs": 900000,
+  "setup": [],
   "checks": "auto",
   "agents": {}
 }
@@ -119,6 +124,7 @@ Built-in presets exist for `codex`, `claude`, `gemini`, and `opencode`. Override
   "defaultAgents": ["my-agent"],
   "maxParallel": 1,
   "timeoutMs": 600000,
+  "setup": ["npm ci"],
   "checks": ["npm test"],
   "agents": {
     "my-agent": {
@@ -133,6 +139,11 @@ Built-in presets exist for `codex`, `claude`, `gemini`, and `opencode`. Override
 
 Agent prompts can be passed through stdin or through an argument containing `{prompt}`. Adapter arguments may also use `{worktree}` and `{task_file}`.
 
+Use `setup` for commands that prepare each detached worktree before an agent
+runs, such as `npm ci` or `uv sync --frozen`. Setup commands must be
+non-interactive and should not change tracked files. They can be overridden with
+repeatable `--setup` flags or disabled with `--no-setup`.
+
 ## Verification and scoring
 
 When `--check` is omitted, PatchLeague looks for conventional project commands:
@@ -145,6 +156,11 @@ When `--check` is omitted, PatchLeague looks for conventional project commands:
 | `pyproject.toml` / `pytest.ini` | `python -m pytest` |
 | `Makefile` | `make test` |
 
+Before agents start, PatchLeague runs the configured checks once against the
+base commit in a separate worktree. The saved baseline result distinguishes
+pre-existing failures from failures introduced by a candidate. Setup commands
+run before both the baseline checks and every agent.
+
 The default score is deliberately simple and inspectable:
 
 | Component | Weight |
@@ -154,7 +170,10 @@ The default score is deliberately simple and inspectable:
 | Focused patch size | 15 |
 | Static risk signals | 5 |
 
-No checks means only partial validation credit. An empty patch or failed agent receives a zero score. Scores help sort candidates; they do not replace reviewing the diff.
+No checks means only partial validation credit. An empty patch or failed agent
+receives a zero score. A patch can be selected as the winner only when the agent
+completed successfully and every configured check passed. Scores help sort
+candidates; they do not replace reviewing the diff.
 
 ## Safe patch application
 
@@ -221,7 +240,7 @@ Run artifacts live under `.patchleague/runs/<run-id>/`. Temporary worktrees are
 - Parse provider-native token and cost telemetry.
 - Container and OS-level sandbox backends.
 - GitHub Issue and pull-request URLs as task sources.
-- Baseline checks and flaky-test detection.
+- Flaky-test detection and retry policies.
 - Replayable agent/tool event timelines.
 - CI mode and shareable redacted reports.
 

src/cli.js

@@ -51,6 +51,8 @@ Setup and discovery:
 Competition:
   run <task-file>              Run agents in isolated Git worktrees
     --agents <a,b>             Agents to run (default: config defaultAgents)
+    --setup <command>          Worktree setup command; repeatable
+    --no-setup                 Disable configured setup commands
     --check <command>          Verification command; repeatable
     --no-checks                Disable auto-detected checks
     --timeout <seconds>        Per-agent timeout
@@ -117,6 +119,7 @@ function humanRun(run, reportPath) {
       `Repository: ${run.repository.path}`,
       `Commit:     ${run.repository.commit}`,
       `Agents:     ${run.agents.join(", ")}`,
+      `Setup:      ${run.setup.length ? run.setup.join(" | ") : "(none)"}`,
       `Checks:     ${run.checks.length ? run.checks.join(" | ") : "(none)"}`,
       `Parallel:   ${run.max_parallel}`,
       `Timeout:    ${Math.round(run.timeout_ms / 1000)}s`
@@ -137,9 +140,12 @@ function humanRun(run, reportPath) {
     " #  AGENT           SCORE   CHECKS    LINES  STATUS",
     ...rows,
     "",
+    run.baseline
+      ? `Baseline: ${run.baseline.status}`
+      : "Baseline: skipped",
     run.winner
       ? `Winner: ${run.winner.agent} (${run.winner.score}/100)`
-      : "Winner: none — no valid patch was produced.",
+      : "Winner: none — no patch passed every configured check.",
     reportPath ? `Report: ${reportPath}` : ""
   ]
     .filter((line, index, all) => line !== "" || all[index - 1] !== "")
@@ -261,6 +267,8 @@ async function agentsCommand(args) {
 
 async function runCommand(args, reporter) {
   const agentList = parseCommaList(takeValue(args, "--agents"));
+  const explicitSetup = takeValue(args, "--setup", { multiple: true });
+  const noSetup = takeBoolean(args, "--no-setup");
   const explicitChecks = takeValue(args, "--check", { multiple: true });
   const noChecks = takeBoolean(args, "--no-checks");
   const allowDirty = takeBoolean(args, "--allow-dirty");
@@ -288,6 +296,8 @@ async function runCommand(args, reporter) {
     configPath: loaded.path,
     task,
     agents: agentList,
+    explicitSetup,
+    noSetup,
     explicitChecks,
     noChecks,
     timeoutMs: timeoutSeconds ? timeoutSeconds * 1000 : undefined,

src/config.js

@@ -10,6 +10,7 @@ export const DEFAULT_CONFIG = Object.freeze({
   defaultAgents: ["codex", "claude"],
   maxParallel: 2,
   timeoutMs: 15 * 60 * 1000,
+  setup: [],
   checks: "auto",
   agents: {
     codex: {
@@ -70,6 +71,7 @@ export function sampleConfig() {
     defaultAgents: ["codex", "claude"],
     maxParallel: 2,
     timeoutMs: 900000,
+    setup: [],
     checks: "auto",
     agents: {}
   };
@@ -126,6 +128,17 @@ export function validateConfig(config, source = "configuration") {
       `${source}.timeoutMs must be an integer of at least 1000`
     );
   }
+  if (
+    !Array.isArray(config.setup) ||
+    config.setup.some(
+      (command) => typeof command !== "string" || command.trim().length === 0
+    )
+  ) {
+    throw new PatchLeagueError(
+      "INVALID_CONFIG",
+      `${source}.setup must be an array of non-empty command strings`
+    );
+  }
   return config;
 }
 

src/process.js

@@ -23,12 +23,24 @@ function terminateProcessTree(child) {
       }
     );
     killer.on("error", () => child.kill());
-    return;
+    return undefined;
   }
 
-  child.kill("SIGTERM");
-  const forceTimer = setTimeout(() => child.kill("SIGKILL"), 3000);
+  const signalGroup = (signal) => {
+    try {
+      process.kill(-child.pid, signal);
+    } catch {
+      try {
+        child.kill(signal);
+      } catch {
+        // The process may have exited between the timeout and the signal.
+      }
+    }
+  };
+  signalGroup("SIGTERM");
+  const forceTimer = setTimeout(() => signalGroup("SIGKILL"), 3000);
   forceTimer.unref();
+  return forceTimer;
 }
 
 export function runProcess(
@@ -52,6 +64,7 @@ export function runProcess(
     let truncated = false;
     let timedOut = false;
     let settled = false;
+    let forceTimer;
 
     const needsWindowsCommandShell =
       process.platform === "win32" &&
@@ -64,6 +77,7 @@ export function runProcess(
       cwd,
       env: env ? { ...process.env, ...env } : process.env,
       shell: shell || needsWindowsCommandShell,
+      detached: process.platform !== "win32",
       windowsHide: true,
       stdio: ["pipe", "pipe", "pipe"]
     });
@@ -92,14 +106,15 @@ export function runProcess(
       timeoutMs > 0
         ? setTimeout(() => {
             timedOut = true;
-            terminateProcessTree(child);
+            forceTimer = terminateProcessTree(child);
           }, timeoutMs)
         : undefined;
 
     const finish = (result) => {
       if (settled) return;
       settled = true;
       if (timer) clearTimeout(timer);
+      if (forceTimer) clearTimeout(forceTimer);
       resolve({
         command,
         args,

src/report.js

@@ -37,6 +37,8 @@ export async function generateReport(repoPath, run) {
     task: run.task,
     repository: run.repository,
     winner: run.winner,
+    baseline: run.baseline,
+    setup: run.setup,
     checks: run.checks,
     results
   };
@@ -142,7 +144,7 @@ export function reportHtml(model) {
     <section class="winner" id="winner"></section>
     <section class="grid" id="cards"></section>
     <section class="detail" id="detail"></section>
-    <footer>Generated locally by PatchLeague. No source code or agent logs were uploaded.</footer>
+    <footer>Generated locally by PatchLeague. Configured agent providers may receive repository context.</footer>
   </main>
   <script id="run-data" type="application/json">${safeJson(model)}</script>
   <script>
@@ -153,7 +155,8 @@ export function reportHtml(model) {
     document.getElementById("title").textContent = run.task.title;
     document.getElementById("meta").textContent =
       run.repository.branch + " · " + run.repository.commit.slice(0, 10) + " · " +
-      new Date(run.created_at).toLocaleString();
+      new Date(run.created_at).toLocaleString() +
+      (run.baseline ? " · baseline " + run.baseline.status : "");
     const winner = run.results.find((result) => result.agent === run.winner?.agent);
     document.getElementById("winner").innerHTML = winner
       ? '<div><small>Winning patch</small><strong>' + esc(winner.agent) +