diff --git a/harness2/cli/_final.py b/harness2/cli/_final.py
new file mode 100644
index 00000000..9a0efd93
--- /dev/null
+++ b/harness2/cli/_final.py
@@ -0,0 +1,136 @@
+# Copyright 2026 The Harness2 Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Compose a project harness from a run's successful task selections."""
+
+from __future__ import annotations
+
+import json
+import re
+import shutil
+from datetime import datetime, timezone
+from pathlib import Path
+
+from ..core import library, prompts, recursion, schedule, tree
+from ..core.proposer import Proposer
+from ..settings import ConfigError
+from . import _adopt as A
+from . import _common as C
+
+
+def _context(
+ adapter, root: Path, record: dict, mode: str, dest: Path, base: tree.Tree
+) -> list[str]:
+ surface = adapter.substrate.surface()
+ token = schedule.mode_token(mode)
+ plan = record["schedule"]
+ namespace = C.summary_namespace(adapter, plan["k"], seed=plan.get("seed"))
+ selected = {
+ row["task_id"]
+ for path, row in C.summaries(root)
+ if path.parent.parent.name == namespace
+ and (selection := (row.get("selections") or {}).get(token) or {}).get("ok")
+ and selection.get("chosen_label")
+ }
+ tasks, index = [], []
+ tree.write_tree(surface, base, dest / "harness", stubs=True)
+ lib = C.library_root_for(adapter, plan["k"], seed=plan.get("seed")) / token
+ for domain, task in record["stream"]:
+ if task not in selected:
+ continue
+ _, adopted = A.adopted_tree(adapter, root, task, mode)
+ task_dir = dest / "tasks" / library.slug(task)
+ tree.write_tree(surface, adopted, task_dir / "harness", stubs=False)
+ (task_dir / "task.md").write_text(adapter.task_prompt(task), encoding="utf-8")
+ lesson = (
+ lib / library.domain_dir(domain) / "harnesses" / library.slug(task) / "EXPERIENCE.md"
+ )
+ if lesson.is_file():
+ (task_dir / "EXPERIENCE.md").write_text(lesson.read_text(), encoding="utf-8")
+ index.append(
+ f"- {task} ({domain}): adopted harness "
+ f"{'differs from' if adopted != base else 'same as'} the start; "
+ f"lesson: {'yes' if lesson.is_file() else 'no'}"
+ )
+ tasks.append(task)
+ if not tasks:
+ raise ConfigError(f"no successful selection for any task in {mode}")
+ (dest / "INDEX.md").write_text("\n".join(index) + "\n", encoding="utf-8")
+ return tasks
+
+
+def consolidate(ctx, adapter, root: Path, mode: str) -> dict:
+ mode = C.mode_label(schedule.mode_token(mode))
+ with ctx.progress():
+ base = A.base_tree(adapter)
+ record = C.load_run(root)
+ surface = adapter.substrate.surface()
+ prop = Proposer(adapter.spec.channels["proposer"], audit_dir=root / "consolidated/_audit")
+ cost = 0.0
+ # The context path participates in the proposer's cache key.
+ context = root / "consolidated/_audit" / f"context.{mode}"
+ if context.exists():
+ shutil.rmtree(context)
+ try:
+ tasks = _context(adapter, root, record, mode, context, base)
+ mission = prompts.CONSOLIDATE_MISSION.replace("{n}", str(len(tasks))).replace(
+ "{surface}", prompts._st(surface).surface_md
+ )
+ err = ""
+ for _ in range(2):
+ with ctx.progress():
+ reply = prop.call(
+ mission
+ + (f"\n\n# YOUR PREVIOUS ATTEMPT WAS REJECTED\n\n{err}\n\n" if err else ""),
+ context,
+ f"consolidate.{mode}",
+ harvest="context/harness",
+ )
+ cost += reply.cost_usd
+ if reply.workspace is None:
+ err = "the harness directory did not come back from the session"
+ continue
+ rejected = []
+ new = tree.read_tree(surface, reply.workspace, rejected=rejected)
+ _, err = recursion._accept_tree(surface, adapter, new, base, "consolidate")
+ if rejected:
+ err = "files outside the editable surface: " + ", ".join(sorted(rejected))
+ if not err:
+ break
+ tree.write_tree(surface, new, context / "harness", stubs=True)
+ else:
+ if reply.workspace is not None:
+ shutil.rmtree(reply.workspace)
+ raise ConfigError(err)
+ finally:
+ shutil.rmtree(context, ignore_errors=True)
+ notes = re.search(r"(.*?)", reply.text, re.S)
+ notes = notes.group(1).strip() if notes else reply.text.strip()
+ dest = root / "consolidated" / mode
+ path = tree.write_tree(surface, new, dest / "harness", stubs=False)
+ (dest / "notes.md").write_text(notes + "\n", encoding="utf-8")
+ (dest / "consolidation.json").write_text(
+ json.dumps(
+ {
+ "tasks": tasks,
+ "model": prop.spec.model,
+ "cost_usd": cost,
+ "created": datetime.now(timezone.utc).isoformat(),
+ },
+ indent=2,
+ )
+ + "\n",
+ encoding="utf-8",
+ )
+ return {"path": path, "tasks": tasks, "changes": A.changes(base, new), "notes": notes}
diff --git a/harness2/cli/consolidate.py b/harness2/cli/consolidate.py
deleted file mode 100644
index 47897bab..00000000
--- a/harness2/cli/consolidate.py
+++ /dev/null
@@ -1,178 +0,0 @@
-# Copyright 2026 The Harness2 Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Compose a project harness from a run's successful task selections."""
-
-from __future__ import annotations
-
-import json
-import re
-import tempfile
-from datetime import datetime, timezone
-from pathlib import Path
-
-from ..core import events, library, prompts, recursion, schedule, tree
-from ..core.proposer import Proposer
-from ..settings import ConfigError
-from . import _adopt as A
-from . import _common as C
-from .run import _stopped
-
-
-def add_parser(subparsers):
- p = subparsers.add_parser(
- "consolidate",
- help="combine a run's adopted harnesses into one for the project",
- description="A run adopts one harness per task. Combine them, with the lessons the run "
- "recorded, into one harness for the project's next tasks. apply and diff then use it "
- "by default. Makes one improver session.",
- )
- p.add_argument("name", nargs="?", help="run name (default: the most recent run)")
- p.add_argument(
- "--mode",
- choices=("parallel", "sequential"),
- default="parallel",
- help="take the harnesses adopted in this recursion mode",
- )
- return p
-
-
-def _context(
- adapter, root: Path, record: dict, mode: str, dest: Path, base: tree.Tree
-) -> list[str]:
- surface = adapter.substrate.surface()
- token = schedule.mode_token(mode)
- plan = record["schedule"]
- namespace = C.summary_namespace(adapter, plan["k"], seed=plan.get("seed"))
- selected = {
- row["task_id"]
- for path, row in C.summaries(root)
- if path.parent.parent.name == namespace
- and (selection := (row.get("selections") or {}).get(token) or {}).get("ok")
- and selection.get("chosen_label")
- }
- tasks, index = [], []
- tree.write_tree(surface, base, dest / "harness", stubs=True)
- lib = C.library_root_for(adapter, plan["k"], seed=plan.get("seed")) / token
- for domain, task in record["stream"]:
- if task not in selected:
- continue
- _, adopted = A.adopted_tree(adapter, root, task, mode)
- task_dir = dest / "tasks" / library.slug(task)
- tree.write_tree(surface, adopted, task_dir / "harness", stubs=False)
- (task_dir / "task.md").write_text(adapter.task_prompt(task), encoding="utf-8")
- lesson = (
- lib / library.domain_dir(domain) / "harnesses" / library.slug(task) / "EXPERIENCE.md"
- )
- if lesson.is_file():
- (task_dir / "EXPERIENCE.md").write_text(lesson.read_text(), encoding="utf-8")
- index.append(
- f"- {task} ({domain}): adopted harness "
- f"{'differs from' if adopted != base else 'same as'} the start; "
- f"lesson: {'yes' if lesson.is_file() else 'no'}"
- )
- tasks.append(task)
- if not tasks:
- raise ConfigError(f"no successful selection for any task in {mode}")
- (dest / "INDEX.md").write_text("\n".join(index) + "\n", encoding="utf-8")
- return tasks
-
-
-def run(args, ctx) -> int:
- with ctx.progress():
- adapter, root = A.open_run(ctx, args.name)
- base = A.base_tree(adapter)
- record = C.load_run(root)
- surface = adapter.substrate.surface()
- config = ctx.settings
- events.start(
- root / "events.jsonl",
- budget_usd=config.get("run.budget_usd"),
- max_calls=config.get("run.max_calls"),
- )
- prop = Proposer(adapter.spec.channels["proposer"], audit_dir=root / "consolidated/_audit")
- cost = 0.0
- try:
- with tempfile.TemporaryDirectory(prefix="harness2-consolidate-") as temporary:
- context = Path(temporary)
- tasks = _context(adapter, root, record, args.mode, context, base)
- mission = prompts.CONSOLIDATE_MISSION.replace("{n}", str(len(tasks))).replace(
- "{surface}", prompts._st(surface).surface_md
- )
- err = ""
- for _ in range(2):
- with ctx.progress():
- reply = prop.call(
- mission
- + (f"\n\n# YOUR PREVIOUS ATTEMPT WAS REJECTED\n\n{err}\n\n" if err else ""),
- context,
- f"consolidate.{args.mode}",
- harvest="context/harness",
- )
- cost += reply.cost_usd
- if reply.workspace is None:
- err = "the harness directory did not come back from the session"
- continue
- rejected = []
- new = tree.read_tree(surface, reply.workspace, rejected=rejected)
- _, err = recursion._accept_tree(surface, adapter, new, base, "consolidate")
- if rejected:
- err = "files outside the editable surface: " + ", ".join(sorted(rejected))
- if not err:
- break
- tree.write_tree(surface, new, context / "harness", stubs=True)
- else:
- raise ConfigError(err)
- except events.Stopped as exc:
- reason = str(exc)
- ctx.out(events.totals(), lambda totals: _stopped(root.name, reason, totals))
- return 4
- notes = re.search(r"(.*?)", reply.text, re.S)
- notes = notes.group(1).strip() if notes else reply.text.strip()
- dest = root / "consolidated" / args.mode
- path = tree.write_tree(surface, new, dest / "harness", stubs=False)
- (dest / "notes.md").write_text(notes + "\n", encoding="utf-8")
- (dest / "consolidation.json").write_text(
- json.dumps(
- {
- "tasks": tasks,
- "model": prop.spec.model,
- "cost_usd": cost,
- "created": datetime.now(timezone.utc).isoformat(),
- },
- indent=2,
- )
- + "\n",
- encoding="utf-8",
- )
- changes = A.changes(base, new)
- ctx.out(
- {"path": path, "tasks": tasks, "changes": changes, "notes": notes},
- lambda _: "\n".join(
- [
- f"Consolidated the harnesses adopted for {len(tasks)} task(s) ({args.mode}) into {path}",
- *(
- [f" {kind} {rel}" for rel, kind in changes]
- if changes
- else [
- "The consolidated harness is the same as the starting one: "
- "no task's edits generalized."
- ]
- ),
- f"Next: harness2 eval {root.name} compares it with the starting harness; "
- f"harness2 apply {root.name} writes it into the project.",
- ]
- ),
- )
- return 0
diff --git a/harness2/cli/eval.py b/harness2/cli/eval.py
index 5f438f34..2e44c836 100644
--- a/harness2/cli/eval.py
+++ b/harness2/cli/eval.py
@@ -114,7 +114,7 @@ def run(args, ctx) -> int:
adapter, root = A.open_run(ctx, args.name)
harness = root / "consolidated" / args.mode / "harness"
if not harness.is_dir():
- raise ConfigError("run `harness2 consolidate` first")
+ raise ConfigError("this run has no final harness yet; run `harness2 run` to finish it")
record = C.load_run(root)
allowed = set(adapter.tasks(args.split))
wanted = set(_csv(args.only_tasks))
diff --git a/harness2/cli/main.py b/harness2/cli/main.py
index 091ab2c8..0449ceb8 100644
--- a/harness2/cli/main.py
+++ b/harness2/cli/main.py
@@ -33,7 +33,6 @@
"report",
"runs",
"diff",
- "consolidate",
"eval",
"apply",
"config",
diff --git a/harness2/cli/run.py b/harness2/cli/run.py
index 4d578e72..c95dfe58 100644
--- a/harness2/cli/run.py
+++ b/harness2/cli/run.py
@@ -17,6 +17,7 @@
from __future__ import annotations
import argparse
+import sys
from pathlib import Path
from .. import core, registry
@@ -24,6 +25,7 @@
from ..core.proposer import Proposer
from ..settings import ConfigError
from . import _common as C
+from ._final import consolidate
FLAGS = {
"suite": "suite.name",
@@ -97,7 +99,7 @@ def _estimate(tasks: int, k: int, modes: tuple[str, ...], totals: dict) -> dict:
rollouts = tasks * (k + (ledger.expected_rollouts(k) - k) * len(modes))
sessions = (
tasks * len(modes) * sum(per_mode[key] for key in ("improver", "selector", "reflector"))
- )
+ ) + len(modes)
return {
"per_task_per_mode": per_mode,
"rollouts": rollouts,
@@ -116,12 +118,15 @@ def _stopped(name: str, reason: str, totals: dict) -> str:
"interrupted": "interrupted",
"budget_usd": f"reached --budget-usd {totals['budget_usd']:g}",
"max_calls": f"reached --max-calls {totals['max_calls']}",
+ "usage_limit": "reached the model provider's usage limit",
}.get(reason, reason)
advice = (
"Run the same command to resume."
if reason == "interrupted"
else "Raise the limit and run the same command to resume."
)
+ if reason == "usage_limit":
+ advice = "Run the same command once the limit resets to resume."
return f"Run {name} stopped: {why}{_spend(totals)}.\n{advice}"
@@ -274,13 +279,33 @@ def run(args, ctx) -> int:
else (("failed", "failed tasks") if failed else ("ok", ""))
)
done = sum(int(row.get("done", 0)) for row in result["domains"])
+ final = {}
+ for mode in modes:
+ label = C.mode_label(mode)
+ try:
+ if not any(
+ path.parent.parent.name == audit.name
+ and (selection := (row.get("selections") or {}).get(mode) or {}).get("ok")
+ and selection.get("chosen_label")
+ for path, row in C.summaries(root)
+ ):
+ continue
+ final[label] = consolidate(ctx, adapter, root, label)["path"]
+ except Exception as exc:
+ print(
+ f"Could not build the final harness ({label}): {exc}. "
+ "Run the same command again to retry.",
+ file=sys.stderr,
+ )
ctx.out(
- {"result": result, "totals": events.totals()},
+ {"result": result, "totals": events.totals(), "final": final},
lambda data: (
f"Run {name}: {done} task(s) done, {failed} failed"
+ _spend(data["totals"])
+ (f"\nErrors: {reason}" if errors else "")
- + f"\nNext: harness2 status {name} · harness2 judge {name} · harness2 diff {name}"
+ + "".join(f"\nFinal harness ({mode}): {path}" for mode, path in final.items())
+ + f"\nNext: harness2 diff {name} shows it; harness2 apply {name} writes it into "
+ f"the project; harness2 eval {name} measures it first."
),
)
return 1 if errors else 3 if failed else 0
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 1b34f751..5d0bc03a 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -30,7 +30,7 @@
from unittest.mock import Mock, patch
from harness2 import settings
-from harness2.cli import _adopt, completion, judge, main, run, runs
+from harness2.cli import _adopt, _final, completion, judge, main, run, runs
from harness2.cli import _common as C
from harness2.core import events
from harness2.core.judge import Scored
@@ -81,7 +81,10 @@ def context(self, *args):
return C.Context(parsed, getattr(parsed._module, "FLAGS", {}))
def execute(self, k=1, *extra):
- with patch.object(run, "Proposer", DemoProposer):
+ with (
+ patch.object(run, "Proposer", DemoProposer),
+ patch.object(_final, "Proposer", DemoProposer),
+ ):
return self.invoke("--json", "run", "--k", str(k), *extra)
def test_all_command_arguments(self):
@@ -251,7 +254,7 @@ def test_dry_run_estimate_and_stream_filters(self):
self.assertEqual(code, 0)
data = json.loads(stdout)
self.assertEqual(data["stream"], [["accounting", "ledger-2"]])
- self.assertEqual((data["estimate"]["rollouts"], data["estimate"]["sessions"]), (7, 8))
+ self.assertEqual((data["estimate"]["rollouts"], data["estimate"]["sessions"]), (7, 10))
self.assertEqual(data["estimate"]["per_task_per_mode"]["reflector"], 1)
self.assertEqual(list((self.project / ".harness2/runs").rglob("run.json")), [])
self.assertEqual(list((self.project / ".harness2/runs").rglob("events.jsonl")), [])
@@ -259,7 +262,7 @@ def test_dry_run_estimate_and_stream_filters(self):
self.assertEqual(code, 0)
self.assertIn("would refine 1 task(s)", stdout)
self.assertIn("accounting", stdout)
- self.assertIn("Model calls: 8 agent runs + 8", stdout)
+ self.assertIn("Model calls: 8 agent runs + 9", stdout)
self.assertEqual(
run._estimate(1, 2, ("agg",), {"spent_usd": 4, "calls": 2})["average_cost_per_call"], 2
)
@@ -278,7 +281,7 @@ def test_demo_backed_run_judge_report_diff_for_each_scale(self):
self.assertEqual(code, 0)
data = json.loads(stdout)
self.assertEqual(
- data["totals"]["calls"], 2 * (k + 6 * k + 6 * k + 2 * (1 + int(k > 1)))
+ data["totals"]["calls"], 2 * (k + 6 * k + 6 * k + 2 * (1 + int(k > 1))) + 2
)
name = f"claude-k{k}"
root = self.project / ".harness2/runs" / name
@@ -309,7 +312,7 @@ def test_demo_backed_run_judge_report_diff_for_each_scale(self):
code, stdout, _ = self.invoke("--json", "diff", name)
self.assertEqual(code, 0)
diff = json.loads(stdout)
- self.assertEqual(diff["task"], "ledger-2")
+ self.assertEqual(diff["task"], "consolidated")
self.assertIn("-total = sum", diff["diff"])
self.assertIn("+total = sum", diff["diff"])
code, stdout, _ = self.invoke(
@@ -321,7 +324,10 @@ def test_demo_backed_run_judge_report_diff_for_each_scale(self):
self.assertEqual(self.invoke("--json", "judge", name, "--bases")[0], 0)
code, stdout, _ = self.invoke("--json", "runs", "show", name)
self.assertEqual((code, json.loads(stdout)["done"]), (0, 2))
- with patch.object(run, "Proposer", DemoProposer):
+ with (
+ patch.object(run, "Proposer", DemoProposer),
+ patch.object(_final, "Proposer", DemoProposer),
+ ):
code, stdout, _ = self.invoke(
"--json", "run", "--k", str(k), "--mode", "parallel,sequential"
)
@@ -348,17 +354,23 @@ def test_budget_resume_and_failure_exit_events(self):
for exception, expected, status in (
(KeyboardInterrupt(), 130, "interrupted"),
(events.Stopped("interrupted"), 130, "interrupted"),
+ (events.Stopped("usage_limit"), 4, "stopped"),
(RuntimeError("fixture crash"), 1, "error"),
):
with (
patch.object(run.schedule, "run_sweep", side_effect=exception),
patch.object(run, "Proposer", DemoProposer),
+ patch.object(run, "consolidate") as final,
):
code, out, error = self.invoke("run")
+ final.assert_not_called()
self.assertEqual(code, expected)
self.assertEqual(
json.loads((root / "events.jsonl").read_text().splitlines()[-1])["status"], status
)
+ if isinstance(exception, events.Stopped) and str(exception) == "usage_limit":
+ self.assertIn("reached the model provider's usage limit", out)
+ self.assertIn("Run the same command once the limit resets to resume.", out)
if expected == 130:
self.assertIn("Run the same command to resume", out)
for result, expected in (
@@ -368,6 +380,7 @@ def test_budget_resume_and_failure_exit_events(self):
with (
patch.object(run.schedule, "run_sweep", return_value=result),
patch.object(run, "Proposer", DemoProposer),
+ patch.object(_final, "Proposer", DemoProposer),
):
self.assertEqual(self.invoke("run")[0], expected)
diff --git a/tests/test_consolidate.py b/tests/test_consolidate.py
index 6f1fd024..9897723f 100644
--- a/tests/test_consolidate.py
+++ b/tests/test_consolidate.py
@@ -28,7 +28,7 @@
from types import SimpleNamespace
from unittest.mock import Mock, patch
-from harness2.cli import _adopt, _common, consolidate, eval
+from harness2.cli import _adopt, _common, _final, eval, run
from harness2.cli.main import COMMANDS, build_parser, main
from harness2.core import events
from harness2.settings import ConfigError
@@ -172,29 +172,23 @@ def new_task(self, task, *, split="train", judge=True):
with (directory / "task.toml").open("a") as stream:
stream.write(f'split = "{split}"\n')
- def execute(self, *argv):
- self.invoke("run", "--name", "trial", *argv)
+ def execute(self, *argv, code=0):
+ return self.invoke("run", "--name", "trial", *argv, code=code)
def adapter(self):
args = build_parser().parse_args(["-C", str(self.project), "diff", "trial"])
return _adopt.open_run(_common.Context(args, {}), "trial")[0]
def test_two_tasks_retry_adoption_eval_and_json(self):
- self.execute()
+ text, _ = self.execute()
before = _common.event_totals(self.run_dir)
- fallback = json.loads(self.invoke("--json", "diff", "trial")[0])
- self.assertEqual(fallback["task"], "b-two")
- text, _ = self.invoke("consolidate", "trial")
self.assertEqual(self.calls.read_text(), "2")
path = self.run_dir / "consolidated/parallel/harness"
self.assertEqual((path / "CLAUDE.md").read_text(), "Use shared procedure.\n")
+ self.assertIn(f"Final harness (parallel): {path}", text)
self.assertIn(
- f"Consolidated the harnesses adopted for 2 task(s) (parallel) into {path}", text
- )
- self.assertIn(" M CLAUDE.md", text)
- self.assertIn(
- "Next: harness2 eval trial compares it with the starting harness; "
- "harness2 apply trial writes it into the project.",
+ "Next: harness2 diff trial shows it; harness2 apply trial writes it into the project; "
+ "harness2 eval trial measures it first.",
text,
)
meta = json.loads((path.parent / "consolidation.json").read_text())
@@ -205,12 +199,13 @@ def test_two_tasks_retry_adoption_eval_and_json(self):
(path.parent / "notes.md").read_text(),
"a-one and b-two support the shared procedure.\n",
)
- data = json.loads(self.invoke("--json", "consolidate", "trial")[0])
- self.assertEqual(set(data), {"path", "tasks", "changes", "notes"})
- self.assertEqual(data["path"], str(path))
- self.assertEqual(data["changes"], [["CLAUDE.md", "M"]])
- self.assertEqual(data["tasks"], ["a-one", "b-two"])
- self.assertEqual(data["notes"], "a-one and b-two support the shared procedure.")
+ data = json.loads(self.invoke("--json", "run", "--name", "trial")[0])
+ self.assertEqual(set(data), {"result", "totals", "final"})
+ self.assertEqual(data["final"], {"parallel": str(path)})
+ self.assertEqual(data["totals"]["calls"], before["calls"])
+ self.assertEqual(data["totals"]["spent_usd"], before["spent_usd"])
+ self.assertEqual(self.calls.read_text(), "2")
+ self.assertTrue(list((self.run_dir / "consolidated/_audit").glob("*.response.md")))
consolidated = json.loads(self.invoke("--json", "diff", "trial")[0])
self.assertEqual(consolidated["task"], "consolidated")
self.assertIn("+Use shared procedure.", consolidated["diff"])
@@ -246,7 +241,7 @@ def test_two_tasks_retry_adoption_eval_and_json(self):
self.assertEqual(len(eval_rows), 8)
self.assertEqual({row["cost_usd"] for row in eval_rows}, {0.125})
self.assertAlmostEqual(
- _common.event_totals(self.run_dir)["spent_usd"], before["spent_usd"] + 1.03
+ _common.event_totals(self.run_dir)["spent_usd"], before["spent_usd"] + 1.0
)
self.assertEqual(
{row["run_id"] for row in eval_rows},
@@ -261,7 +256,6 @@ def test_two_tasks_retry_adoption_eval_and_json(self):
def test_eval_filters_skipped_failed_grades_and_limits(self):
self.new_task("c-no-judge", judge=False)
self.execute()
- self.invoke("consolidate", "trial")
text, _ = self.invoke("eval", "trial")
self.assertIn("Skipped 1 task(s) with no check or rubric: c-no-judge", text)
data = json.loads(
@@ -300,35 +294,76 @@ def test_eval_filters_skipped_failed_grades_and_limits(self):
self.assertEqual(data["mean"], {"start": None, "consolidated": None})
self.assertEqual(data["skipped"], ["c-no-judge"])
- def test_modes_unchanged_and_validation_failure_preserves_result(self):
- self.execute("--mode", "parallel,sequential")
- self.invoke("consolidate", "trial", "--mode", "sequential")
+ def test_modes_and_unchanged_harness(self):
+ text, _ = self.execute("--mode", "parallel,sequential")
+ self.assertEqual(self.calls.read_text(), "3")
adapter = self.adapter()
- self.assertEqual(_adopt.harness_for(adapter, self.run_dir, None, "seq")[0], "consolidated")
+ for mode in ("parallel", "sequential"):
+ path = self.run_dir / "consolidated" / mode / "harness"
+ self.assertIn(f"Final harness ({mode}): {path}", text)
+ self.assertEqual(
+ _adopt.harness_for(adapter, self.run_dir, None, mode)[0], "consolidated"
+ )
self.assertEqual(
_adopt.harness_for(adapter, self.run_dir, "a-one", "sequential")[0], "a-one"
)
- self.assertEqual(_adopt.harness_for(adapter, self.run_dir, None, "parallel")[0], "b-two")
- path = self.run_dir / "consolidated/sequential/harness/CLAUDE.md"
+ with patch.dict(os.environ, {"WC_BEHAVIOR": "unchanged"}):
+ self.invoke("run", "--name", "unchanged")
+ self.assertEqual(self.invoke("diff", "unchanged")[0], "\n")
+
+ def test_validation_failure_keeps_exit_code_and_can_retry(self):
with patch.dict(os.environ, {"WC_BEHAVIOR": "invalid"}):
- _, err = self.invoke("consolidate", "trial", "--mode", "sequential", code=1)
+ text, err = self.execute()
+ self.assertIn("Could not build the final harness (parallel): ", err)
self.assertIn("broken.py", err)
+ self.assertTrue(err.rstrip().endswith(". Run the same command again to retry."))
+ self.assertNotIn("Final harness (parallel):", text)
+ self.assertFalse((self.run_dir / "consolidated/parallel/harness").exists())
+ self.assertEqual(self.calls.read_text(), "2")
+ _, err = self.invoke("eval", "trial", code=1)
+ self.assertIn("this run has no final harness yet; run `harness2 run` to finish it", err)
+ with patch.dict(os.environ, {"WC_BEHAVIOR": "valid"}):
+ self.execute()
+ self.assertEqual(self.calls.read_text(), "3")
+ path = self.run_dir / "consolidated/parallel/harness/CLAUDE.md"
self.assertEqual(path.read_text(), "Use shared procedure.\n")
+ for failure, expected in (
+ ("failure", 0),
+ ("failure", 3),
+ ("failure", 1),
+ (events.Stopped("usage_limit"), 0),
+ ):
+ error = failure if isinstance(failure, Exception) else ConfigError(failure)
+ result = {
+ "domains": [
+ {"failed": int(expected == 3), **({"error": "dead"} if expected == 1 else {})}
+ ]
+ }
+ with (
+ patch.object(run.schedule, "run_sweep", return_value=result),
+ patch.object(run, "consolidate", side_effect=error),
+ ):
+ text, err = self.execute(code=expected)
+ self.assertEqual(
+ err,
+ f"Could not build the final harness (parallel): {error}. "
+ "Run the same command again to retry.\n",
+ )
+ self.assertEqual(path.read_text(), "Use shared procedure.\n")
+
+ def test_missing_workspace_failure_keeps_exit_code(self):
with patch.dict(os.environ, {"WC_BEHAVIOR": "missing"}):
- _, err = self.invoke("consolidate", "trial", code=1)
- self.assertIn("the harness directory did not come back", err)
- with patch.dict(os.environ, {"WC_BEHAVIOR": "unchanged"}):
- text, _ = self.invoke("consolidate", "trial")
+ _, err = self.execute()
self.assertIn(
- "The consolidated harness is the same as the starting one: no task's edits generalized.",
- text,
+ "Could not build the final harness (parallel): "
+ "the harness directory did not come back from the session. "
+ "Run the same command again to retry.",
+ err,
)
- self.assertEqual(self.invoke("diff", "trial")[0], "\n")
+ self.assertEqual(self.calls.read_text(), "2")
def test_context_stream_order_optional_lessons_and_errors(self):
self.execute()
- _, err = self.invoke("eval", "trial", code=1)
- self.assertIn("run `harness2 consolidate` first", err)
adapter = self.adapter()
record = _common.load_run(self.run_dir)
plan = record["schedule"]
@@ -347,7 +382,7 @@ def test_context_stream_order_optional_lessons_and_errors(self):
)
row["selections"]["agg"]["chosen_tree_is_base"] = True
task_path.write_text(json.dumps(row))
- tasks = consolidate._context(adapter, self.run_dir, record, "parallel", dest, base)
+ tasks = _final._context(adapter, self.run_dir, record, "parallel", dest, base)
self.assertEqual(tasks, ["b-two", "a-one"])
self.assertEqual(
(dest / "INDEX.md").read_text(),
@@ -360,9 +395,14 @@ def test_context_stream_order_optional_lessons_and_errors(self):
row["selections"]["agg"]["ok"] = False
path.write_text(json.dumps(row))
with self.assertRaisesRegex(ConfigError, "no successful selection"):
- consolidate._context(
- adapter, self.run_dir, record, "parallel", self.root / "empty", base
- )
+ _final._context(adapter, self.run_dir, record, "parallel", self.root / "empty", base)
+ with (
+ patch.object(run.schedule, "run_sweep", return_value={"domains": [{"failed": 2}]}),
+ patch.object(run, "consolidate") as final,
+ ):
+ data = json.loads(self.invoke("--json", "run", "--name", "trial", code=3)[0])
+ final.assert_not_called()
+ self.assertEqual(data["final"], {})
def test_rollout_grades_and_optional_cost(self):
adapter = SimpleNamespace(execute=Mock(return_value=self.root), score=Mock())
@@ -393,15 +433,15 @@ def test_rollout_grades_and_optional_cost(self):
def test_command_order_and_help(self):
i = COMMANDS.index("diff")
- self.assertEqual(COMMANDS[i + 1 : i + 3], ("consolidate", "eval"))
+ self.assertEqual(COMMANDS[i + 1], "eval")
+ self.assertNotIn("consolidate", COMMANDS)
parser = build_parser()
- for name in ("consolidate", "eval"):
- sub = parser._subparsers._group_actions[0].choices[name]
- for action in sub._actions:
- self.assertTrue(action.help)
- if action.dest != "help":
- self.assertEqual(action.help, action.help.lower())
- self.assertFalse(action.help.endswith("."))
+ sub = parser._subparsers._group_actions[0].choices["eval"]
+ for action in sub._actions:
+ self.assertTrue(action.help)
+ if action.dest != "help":
+ self.assertEqual(action.help, action.help.lower())
+ self.assertFalse(action.help.endswith("."))
if __name__ == "__main__":
diff --git a/tests/test_e2e.py b/tests/test_e2e.py
index a89b733e..c41f3c63 100644
--- a/tests/test_e2e.py
+++ b/tests/test_e2e.py
@@ -230,7 +230,7 @@ def invoke(*argv):
Counter(call["role"] for call in all_calls),
{
"executor": 24,
- "improver": 18,
+ "improver": 20,
"selector": 2,
"reflector": 4,
},
@@ -277,8 +277,8 @@ def invoke(*argv):
for row in run_rows
},
{
- "claude-k1": (2, 2, 1.08),
- "k2": (2, 2, 2.16),
+ "claude-k1": (2, 2, 1.09),
+ "k2": (2, 2, 2.17),
},
)
invoke("judge")
@@ -336,7 +336,7 @@ def invoke(*argv):
["git", "worktree", "list", "--porcelain"], cwd=project, text=True
)
self.assertEqual(worktrees.count("worktree "), 1)
- print("E2E: 2 task types; k=1/k=2; 48 calls; 2 graded; adopted; resume=0 calls")
+ print("E2E: 2 task types; k=1/k=2; 50 calls; 2 graded; adopted; resume=0 calls")
if __name__ == "__main__":