diff --git a/harness2/cli/apply.py b/harness2/cli/apply.py index 227938af..659463e4 100644 --- a/harness2/cli/apply.py +++ b/harness2/cli/apply.py @@ -91,11 +91,11 @@ def _apply(args, ctx) -> tuple[int, dict]: todo = [(rel, kind) for rel, kind in delta if args.force or rel not in moved] label = ( - "consolidated from the run's tasks" + f"Final harness of run {run_dir.name} ({args.mode})" if args.task is None and task == "consolidated" - else f"adopted for {task}" + else f"Harness adopted for {task} (run {run_dir.name}, {args.mode})" ) - print(f"Harness {label} ({args.mode}, run {run_dir.name}) -> {where}\n") + print(f"{label} -> {where}\n") for rel, kind in delta: mark = "!" if rel in moved else kind print(f" {mark} {rel:<48} {_delta(rel, current, adopted)}") diff --git a/harness2/cli/eval.py b/harness2/cli/eval.py index 2e44c836..eac094d6 100644 --- a/harness2/cli/eval.py +++ b/harness2/cli/eval.py @@ -82,22 +82,28 @@ def _text(data): def percent(value): return "-" if value is None else f"{value * 100:g}%" - lines = [ - f"Comparing the consolidated harness with the starting one on {len(data['tasks'])} " - f"task(s), {data['repeat']} run(s) each.", - "TASK START CONSOLIDATED", - ] - for task, row in data["tasks"].items(): - lines.append( - f"{task} {percent(row['start']['mean'])} {percent(row['consolidated']['mean'])}" - ) start, consolidated = data["mean"]["start"], data["mean"]["consolidated"] delta = ( f"{(consolidated - start) * 100:+g}" if start is not None and consolidated is not None else "-" ) - lines.append(f"mean {percent(start)} {percent(consolidated)} ({delta} points)") + rows = [("TASK", "START", "FINAL")] + rows += [ + (task, percent(row["start"]["mean"]), percent(row["consolidated"]["mean"])) + for task, row in data["tasks"].items() + ] + rows.append(("mean", percent(start), percent(consolidated))) + width = max(len(r[0]) for r in rows) + table = [f" {a:<{width}} {b:>5} {c:>5}" for a, b, c in rows] + table[-1] += f" ({delta} points)" + lines = [ + f"Comparing the final harness with the starting one on {len(data['tasks'])} " + f"task(s), {data['repeat']} run(s) each.", + "", + *table, + "", + ] if data["skipped"]: lines.append( f"Skipped {len(data['skipped'])} task(s) with no check or rubric: " diff --git a/harness2/cli/init.py b/harness2/cli/init.py index 34f9e4dd..bb8b3f80 100644 --- a/harness2/cli/init.py +++ b/harness2/cli/init.py @@ -107,7 +107,7 @@ def _run(args) -> tuple[int, dict]: print(message, file=sys.stderr) return 1, {"error": message, "exit_code": 1} ask = _asker(interactive=not args.yes and sys.stdin.isatty()) - print(f"Setting up Harness² in {root}\n") + print(f"Setting up Harness² in {root}") interactive = not args.yes and sys.stdin.isatty() found = {name: _version(name) for name in ("claude", "codex")} @@ -159,7 +159,6 @@ def _run(args) -> tuple[int, dict]: ignore.write_text("runs/\n.env\n", encoding="utf-8") (config.parent / "tasks").mkdir(exist_ok=True) - print() print(_harness_summary(root / harness, executor, editable)) print(_key_summary(root, (model, improver))) print(f"\nWrote {config.relative_to(root)}") diff --git a/harness2/cli/run.py b/harness2/cli/run.py index 31680703..fac49701 100644 --- a/harness2/cli/run.py +++ b/harness2/cli/run.py @@ -117,7 +117,9 @@ def _estimate(tasks: int, k: int, modes: tuple[str, ...], totals: dict) -> dict: def _spend(totals: dict) -> str: - return f" · ${totals['spent_usd']:.2f} spent · {totals['calls']} model calls" + # Codex and subscriptions report no money; $0.00 would read as "free". + money = f"${totals['spent_usd']:.2f} spent" if totals["spent_usd"] else "spend not reported" + return f" · {money} · {totals['calls']} model calls" def _stopped(name: str, reason: str, totals: dict) -> str: diff --git a/harness2/cli/status.py b/harness2/cli/status.py index 7a55b273..5ccbbb27 100644 --- a/harness2/cli/status.py +++ b/harness2/cli/status.py @@ -136,7 +136,8 @@ def render(snap: dict) -> str: head, f"started {_when(snap['created'])}" + (f" · last activity {_ago(last.get('t'))}" if last else "") - + f" · ${snap['spent_usd']:.2f} spent · {snap['calls']} model calls", + + (f" · ${snap['spent_usd']:.2f} spent" if snap["spent_usd"] else " · spend not reported") + + f" · {snap['calls']} model calls", "", ] if rows: diff --git a/tests/test_consolidate.py b/tests/test_consolidate.py index fc7589d7..b1ffd7f8 100644 --- a/tests/test_consolidate.py +++ b/tests/test_consolidate.py @@ -213,9 +213,9 @@ def test_two_tasks_retry_adoption_eval_and_json(self): self.assertEqual(task["label"], "a-one") self.assertIn("+Task note.", task["diffs"]["CLAUDE.md"]) applied, _ = self.invoke("apply", "trial", "--dry-run", "--task", "a-one") - self.assertIn("Harness adopted for a-one (parallel, run trial)", applied) + self.assertIn("Harness adopted for a-one (run trial, parallel)", applied) applied, _ = self.invoke("apply", "trial", "--yes") - self.assertIn("Harness consolidated from the run's tasks (parallel, run trial)", applied) + self.assertIn("Final harness of run trial (parallel)", applied) self.assertEqual((self.project / "CLAUDE.md").read_text(), "Use shared procedure.\n") text, _ = self.invoke("eval", "trial", "--repeat", "2") self.assertIn( @@ -277,8 +277,8 @@ def test_eval_filters_skipped_failed_grades_and_limits(self): self.assertEqual(json.loads(Path(data["path"]).read_text()), data) with patch.object(GenericAdapter, "score", return_value=None): text, _ = self.invoke("eval", "trial", "--only-tasks", "a-one", code=3) - self.assertIn("a-one - -", text) - self.assertIn("mean - - (- points)", text) + self.assertRegex(text, r"a-one +- +-") + self.assertRegex(text, r"mean +- +- \(- points\)") data = json.loads((self.run_dir / "eval/parallel/results.json").read_text()) self.assertEqual(data["tasks"]["a-one"]["start"], {"runs": [None], "mean": None}) text, _ = self.invoke("-c", "run.max_calls=1", "eval", "trial", code=4) @@ -429,7 +429,7 @@ def test_rollout_grades_and_optional_cost(self): "path": "results.json", "mean": {"start": 1.0, "consolidated": 0.0}, } - self.assertIn("mean 100% 0% (-100 points)", eval._text(data)) + self.assertRegex(eval._text(data), r"mean +100% +0% \(-100 points\)") def test_command_order_and_help(self): i = COMMANDS.index("diff")