Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions harness2/cli/apply.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,11 +91,11 @@ def _apply(args, ctx) -> tuple[int, dict]:
todo = [(rel, kind) for rel, kind in delta if args.force or rel not in moved]

label = (
"consolidated from the run's tasks"
f"Final harness of run {run_dir.name} ({args.mode})"
if args.task is None and task == "consolidated"
else f"adopted for {task}"
else f"Harness adopted for {task} (run {run_dir.name}, {args.mode})"
)
print(f"Harness {label} ({args.mode}, run {run_dir.name}) -> {where}\n")
print(f"{label} -> {where}\n")
for rel, kind in delta:
mark = "!" if rel in moved else kind
print(f" {mark} {rel:<48} {_delta(rel, current, adopted)}")
Expand Down
26 changes: 16 additions & 10 deletions harness2/cli/eval.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,22 +82,28 @@ def _text(data):
def percent(value):
return "-" if value is None else f"{value * 100:g}%"

lines = [
f"Comparing the consolidated harness with the starting one on {len(data['tasks'])} "
f"task(s), {data['repeat']} run(s) each.",
"TASK START CONSOLIDATED",
]
for task, row in data["tasks"].items():
lines.append(
f"{task} {percent(row['start']['mean'])} {percent(row['consolidated']['mean'])}"
)
start, consolidated = data["mean"]["start"], data["mean"]["consolidated"]
delta = (
f"{(consolidated - start) * 100:+g}"
if start is not None and consolidated is not None
else "-"
)
lines.append(f"mean {percent(start)} {percent(consolidated)} ({delta} points)")
rows = [("TASK", "START", "FINAL")]
rows += [
(task, percent(row["start"]["mean"]), percent(row["consolidated"]["mean"]))
for task, row in data["tasks"].items()
]
rows.append(("mean", percent(start), percent(consolidated)))
width = max(len(r[0]) for r in rows)
table = [f" {a:<{width}} {b:>5} {c:>5}" for a, b, c in rows]
table[-1] += f" ({delta} points)"
lines = [
f"Comparing the final harness with the starting one on {len(data['tasks'])} "
f"task(s), {data['repeat']} run(s) each.",
"",
*table,
"",
]
if data["skipped"]:
lines.append(
f"Skipped {len(data['skipped'])} task(s) with no check or rubric: "
Expand Down
3 changes: 1 addition & 2 deletions harness2/cli/init.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ def _run(args) -> tuple[int, dict]:
print(message, file=sys.stderr)
return 1, {"error": message, "exit_code": 1}
ask = _asker(interactive=not args.yes and sys.stdin.isatty())
print(f"Setting up Harness² in {root}\n")
print(f"Setting up Harness² in {root}")

interactive = not args.yes and sys.stdin.isatty()
found = {name: _version(name) for name in ("claude", "codex")}
Expand Down Expand Up @@ -159,7 +159,6 @@ def _run(args) -> tuple[int, dict]:
ignore.write_text("runs/\n.env\n", encoding="utf-8")
(config.parent / "tasks").mkdir(exist_ok=True)

print()
print(_harness_summary(root / harness, executor, editable))
print(_key_summary(root, (model, improver)))
print(f"\nWrote {config.relative_to(root)}")
Expand Down
4 changes: 3 additions & 1 deletion harness2/cli/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,9 @@ def _estimate(tasks: int, k: int, modes: tuple[str, ...], totals: dict) -> dict:


def _spend(totals: dict) -> str:
return f" · ${totals['spent_usd']:.2f} spent · {totals['calls']} model calls"
# Codex and subscriptions report no money; $0.00 would read as "free".
money = f"${totals['spent_usd']:.2f} spent" if totals["spent_usd"] else "spend not reported"
return f" · {money} · {totals['calls']} model calls"


def _stopped(name: str, reason: str, totals: dict) -> str:
Expand Down
3 changes: 2 additions & 1 deletion harness2/cli/status.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,8 @@ def render(snap: dict) -> str:
head,
f"started {_when(snap['created'])}"
+ (f" · last activity {_ago(last.get('t'))}" if last else "")
+ f" · ${snap['spent_usd']:.2f} spent · {snap['calls']} model calls",
+ (f" · ${snap['spent_usd']:.2f} spent" if snap["spent_usd"] else " · spend not reported")
+ f" · {snap['calls']} model calls",
"",
]
if rows:
Expand Down
10 changes: 5 additions & 5 deletions tests/test_consolidate.py
Original file line number Diff line number Diff line change
Expand Up @@ -213,9 +213,9 @@ def test_two_tasks_retry_adoption_eval_and_json(self):
self.assertEqual(task["label"], "a-one")
self.assertIn("+Task note.", task["diffs"]["CLAUDE.md"])
applied, _ = self.invoke("apply", "trial", "--dry-run", "--task", "a-one")
self.assertIn("Harness adopted for a-one (parallel, run trial)", applied)
self.assertIn("Harness adopted for a-one (run trial, parallel)", applied)
applied, _ = self.invoke("apply", "trial", "--yes")
self.assertIn("Harness consolidated from the run's tasks (parallel, run trial)", applied)
self.assertIn("Final harness of run trial (parallel)", applied)
self.assertEqual((self.project / "CLAUDE.md").read_text(), "Use shared procedure.\n")
text, _ = self.invoke("eval", "trial", "--repeat", "2")
self.assertIn(
Expand Down Expand Up @@ -277,8 +277,8 @@ def test_eval_filters_skipped_failed_grades_and_limits(self):
self.assertEqual(json.loads(Path(data["path"]).read_text()), data)
with patch.object(GenericAdapter, "score", return_value=None):
text, _ = self.invoke("eval", "trial", "--only-tasks", "a-one", code=3)
self.assertIn("a-one - -", text)
self.assertIn("mean - - (- points)", text)
self.assertRegex(text, r"a-one +- +-")
self.assertRegex(text, r"mean +- +- \(- points\)")
data = json.loads((self.run_dir / "eval/parallel/results.json").read_text())
self.assertEqual(data["tasks"]["a-one"]["start"], {"runs": [None], "mean": None})
text, _ = self.invoke("-c", "run.max_calls=1", "eval", "trial", code=4)
Expand Down Expand Up @@ -429,7 +429,7 @@ def test_rollout_grades_and_optional_cost(self):
"path": "results.json",
"mean": {"start": 1.0, "consolidated": 0.0},
}
self.assertIn("mean 100% 0% (-100 points)", eval._text(data))
self.assertRegex(eval._text(data), r"mean +100% +0% \(-100 points\)")

def test_command_order_and_help(self):
i = COMMANDS.index("diff")
Expand Down
Loading