From 721545b5753d5cd5092abdd10b7b1356c413594a Mon Sep 17 00:00:00 2001 From: 3ae3ae Date: Mon, 19 Oct 2026 21:52:27 +0900 Subject: [PATCH] feat: a guide for AI agents, and JSON from init and apply `harness2 guide` prints how an AI agent should drive harness2 on a person's behalf: ask before spending, bound runs with --max-calls, read --json, answer prompts with flags, never edit runs or harness files directly, the init/task/doctor/run --detach/status/stop/diff/eval/apply workflow, where things are stored and what the exit codes mean, followed by the current project's settings, tasks and runs. `guide --install claude` installs the same text as a user-level Claude Code skill, which harness2's own runs never load. `harness2 chat` now builds on it. `init --json` and `apply --json` print a result object on stdout and keep their human-readable text on stderr; `apply --json` refuses to ask for confirmation and needs --yes or --dry-run. Co-Authored-By: Claude Opus 5.5 --- harness2/cli/apply.py | 57 ++++++++++++---- harness2/cli/chat.py | 65 +++---------------- harness2/cli/guide.py | 147 ++++++++++++++++++++++++++++++++++++++++++ harness2/cli/init.py | 33 ++++++++-- harness2/cli/main.py | 1 + tests/test_ui.py | 37 ++++++++++- 6 files changed, 266 insertions(+), 74 deletions(-) create mode 100644 harness2/cli/guide.py diff --git a/harness2/cli/apply.py b/harness2/cli/apply.py index 40125482..2cee874e 100644 --- a/harness2/cli/apply.py +++ b/harness2/cli/apply.py @@ -25,6 +25,8 @@ from __future__ import annotations import argparse +import contextlib +import json import sys from . import _adopt as A @@ -62,6 +64,19 @@ def add_parser(sub) -> argparse.ArgumentParser: def run(args, ctx) -> int: + if not ctx.args.json: + return _apply(args, ctx)[0] + if not (args.yes or args.dry_run): + message = "--json cannot ask for confirmation; add --yes or --dry-run" + print(json.dumps({"error": message, "exit_code": 1})) + return 1 + with contextlib.redirect_stdout(sys.stderr): # the review text; stdout carries the JSON + code, summary = _apply(args, ctx) + print(json.dumps(summary)) + return code + + +def _apply(args, ctx) -> tuple[int, dict]: adapter, run_dir = A.open_run(ctx, args.name) task, adopted = A.harness_for(adapter, run_dir, args.task, args.mode) base = A.base_tree(adapter) @@ -71,7 +86,7 @@ def run(args, ctx) -> int: delta = A.changes(base, adopted) if not delta: ctx.log(f"The harness adopted for {task} is the one the run started from; nothing to do.") - return 0 + return 0, {"run": run_dir.name, "harness": task, "changes": []} moved = {rel for rel, _ in delta if current.get(rel) != base.get(rel)} todo = [(rel, kind) for rel, kind in delta if args.force or rel not in moved] @@ -90,35 +105,55 @@ def run(args, ctx) -> int: + ("will be overwritten (--force)." if args.force else "skipped; --force overwrites.") ) print() + + def summary(applied=(), done=None) -> dict: + return { + "run": run_dir.name, + "harness": task, + "mode": args.mode, + "changes": [ + { + "path": rel, + "kind": kind, + "changed_in_project": rel in moved, + "applied": (rel, kind) in applied, + } + for rel, kind in delta + ], + "written": (done or {}).get("written", []), + "deleted": (done or {}).get("deleted", []), + } + if not todo: ctx.log("Nothing left to apply.") - return 0 + return 0, summary() dirty = A.dirty_paths(ctx.settings, adapter) if dirty and not args.force: - print( + message = ( "These harness files have uncommitted changes:\n " + "\n ".join(dirty) + "\nCommit or stash them first so the result can be reviewed and undone with git, " - "or pass --force.", - file=sys.stderr, + "or pass --force." ) - return 1 + print(message, file=sys.stderr) + return 1, {"error": message, "exit_code": 1} if args.dry_run: - return 0 + return 0, summary() if args.interactive: chosen = _review(todo, current, adopted) elif args.yes: chosen = todo elif not sys.stdin.isatty(): - print("Not a terminal: pass --yes to apply, or --dry-run to only look.", file=sys.stderr) - return 1 + message = "Not a terminal: pass --yes to apply, or --dry-run to only look." + print(message, file=sys.stderr) + return 1, {"error": message, "exit_code": 1} else: chosen = todo if _confirm(f"Apply {len(todo)} change(s)?") else [] if not chosen: ctx.log("Nothing applied.") - return 0 + return 0, summary() final = dict(current) for rel, kind in chosen: @@ -131,7 +166,7 @@ def run(args, ctx) -> int: if args.branch: print(f"On branch {args.branch}.") print("Review with `git diff`; undo with `git restore` on the files listed above.") - return 0 + return 0, summary(chosen, done) def _delta(rel: str, before: dict, after: dict) -> str: diff --git a/harness2/cli/chat.py b/harness2/cli/chat.py index f7e8a23a..9c046b7c 100644 --- a/harness2/cli/chat.py +++ b/harness2/cli/chat.py @@ -27,41 +27,13 @@ import shutil import sys -GUIDE = """\ -You are helping a person use Harness² (the `harness2` command) in the project at {root}. +from . import guide as G -Harness² improves the harness of a coding agent -- the files the agent reads on every run: -CLAUDE.md or AGENTS.md, skills, subagents, helper scripts and hooks. It runs the agent on the -person's own tasks, has an improver propose harness edits from what those runs show, probes -the edits on the same tasks, and composes the best of them. A run never changes the project's -files; `harness2 apply` writes an adopted harness back once the person has reviewed it. +CHAT_INTRO = """\ +You are talking with a person about Harness² (the `harness2` command) in the project at +{root}. Help them write tasks, a check or a rubric, read a run's results, and decide what to +apply. The guide below is how to use harness2. -How you help: -- Writing tasks. A task is a directory .harness2/tasks// holding task.md -- the - instruction the agent receives, written the way the person would really ask -- and an - optional task.toml with: domain, split ("train" or "test"), workspace ("project": a clean - checkout of the repository at `ref`; "files": a scratch directory seeded from files/), - ref, deliverable ("diff", "files" or "message"), check (a shell command that grades the - result), timeout. check/ and rubric.md hold grading material that neither the agent nor - the improver ever sees. Good tasks are real, varied work the person wants done better; a - check command or a rubric.md makes the results measurable. -- Reading results. A run lives in .harness2/runs//: run.json (its settings), - events.jsonl (progress and cost), adapt/// (summary.json, the candidate - harnesses under harnesses/, and the selection/ and reflection/ notes) and library/ (what - carries over between tasks in the same domain). `harness2 status`, `harness2 report` and - `harness2 diff` summarise them. -- Settings. `harness2 config show --origin` prints every setting and where it came from; - `harness2 config set KEY VALUE` changes one. - -Rules: -- `harness2 run`, `harness2 judge` and `harness2 doctor --live` call models and cost money. - Propose the exact command and let the person run or approve it. -- Never edit anything under .harness2/runs/. -- Do not edit the project's harness files to apply a result; `harness2 apply` does that - after review. - -Current state of this project: -{state} """ @@ -82,7 +54,11 @@ def run(args, ctx) -> int: if not binary: print("Neither `claude` nor `codex` is on PATH.", file=sys.stderr) return 1 - guide = GUIDE.format(root=ctx.project, state=_state(ctx)) + guide = ( + CHAT_INTRO.format(root=ctx.project) + + G.GUIDE + + f"\n## This project\n\n{G.state_lines(ctx)}\n" + ) if tool == "claude": argv = [binary, "--append-system-prompt", guide] else: @@ -97,24 +73,3 @@ def _default_tool(settings) -> str | None: if tool in ("claude", "codex") and shutil.which(tool): return tool return "codex" if shutil.which("codex") else None - - -def _state(ctx) -> str: - s = ctx.settings - lines = [ - f"- executor: {s.get('executor.name')} on {s.get('executor.model') or '(model not set)'}", - f"- improver: {s.get('improver.channel')} on {s.get('improver.model') or '(not set)'}", - f"- harness directory: {s.get('paths.harness')}", - ] - tasks_dir = s.path("paths.tasks") - tasks = sorted(p.parent.name for p in tasks_dir.glob("*/task.md")) - shown = ", ".join(tasks[:30]) + (" ..." if len(tasks) > 30 else "") - lines.append(f"- tasks ({len(tasks)}): {shown or 'none yet'}") - runs_dir = s.path("paths.runs") - runs = sorted( - (p for p in runs_dir.glob("*/run.json")), key=lambda p: p.stat().st_mtime, reverse=True - ) - lines.append( - "- runs, newest first: " + (", ".join(p.parent.name for p in runs[:10]) or "none yet") - ) - return "\n".join(lines) diff --git a/harness2/cli/guide.py b/harness2/cli/guide.py new file mode 100644 index 00000000..8dc29532 --- /dev/null +++ b/harness2/cli/guide.py @@ -0,0 +1,147 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The guide for AI agents that drive harness2 on a person's behalf. + +`harness2 guide` prints it with the current project's state; `--install claude` installs it +as a user-level Claude Code skill, which harness2's own runs never load. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +GUIDE = """\ +# harness2: guide for agents + +harness2 improves a coding agent's harness -- the files the agent reads on every run: +CLAUDE.md or AGENTS.md, skills, subagents, helper scripts and hooks. It runs the agent on the +project's own tasks, proposes harness edits from what those runs show, probes them on the +same tasks, and ends with one final harness for the project. A run never changes the +project's files; `harness2 apply` writes the final harness back once the person agrees. + +## Rules + +- Runs spend money or subscription quota. Before `harness2 run` or `harness2 eval`, show the + person the plan from `harness2 run --dry-run` and get their go-ahead. Bound every run with + `--max-calls N` (and `--budget-usd X` when an API key pays for it). +- Read results with `--json`. Errors then arrive on stdout as + {"error": "...", "exit_code": n}. +- Nothing waits for keyboard input when there is no terminal: answer each prompt with its + flag (`--yes`, `--auth`, `--executor`, ...) instead. +- Never edit anything under .harness2/runs/. Never edit the project's harness files to apply + a result: show `harness2 diff`, then run `harness2 apply --yes` once the person agrees. + +## Workflow + +1. `harness2 init --yes [--executor claude|codex] [--auth login|api-key]` creates + .harness2/. `--auth login` uses the agent CLI's own login (a Claude or ChatGPT + subscription). +2. Tasks. `harness2 task new [--files] [--check ""]`, then write the request in + .harness2/tasks//task.md the way the person would really ask for it. Good tasks are + real, varied work the person wants done better. A `check` command (exit status 0 means + pass) makes results measurable; grading files in check/ and rubric.md are never shown to + the agent. More options in task.toml: domain, split ("train"/"test"), workspace + ("project": a clean checkout at `ref`; "files": a scratch directory seeded from files/), + deliverable ("diff", "files", "message"), timeout. List them with + `harness2 task ls --json`. +3. `harness2 doctor --json`: every row must be ok before a run. +4. `harness2 run --dry-run` shows the tasks and the number of model calls. After the + person's go-ahead: `harness2 run --detach --max-calls N`. It returns at once with the run + name; runs take minutes to hours. +5. Follow it with `harness2 status --json`: "active", each task's "state", + "spent_usd", "calls". `harness2 stop ` stops it; running the same `harness2 run` + command again resumes and reuses every finished step. +6. When it has finished, `harness2 diff ` shows the final harness. To measure it + first, `harness2 eval --json` compares it with the starting harness on the tasks + that have a check or rubric. +7. With the person's agreement: `harness2 apply --yes`. + +## Where things are + +- .harness2/config.toml: settings; `harness2 config show --origin --json` resolves them. +- .harness2/runs//: run.json (settings, task list), events.jsonl (every agent run and + model session, with cost), consolidated// (the final harness and the improver's + notes), adapt/ (per-task candidates, selections, lessons), eval/ (eval results). + +## Exit codes + +0 ok, 1 error, 2 bad usage, 3 finished with failed tasks, 4 stopped at a limit (--max-calls, +--budget-usd, or the provider's usage limit), 130 interrupted. After 4 or 130, the same +command resumes. +""" + +SKILL_HEAD = """\ +--- +name: harness2 +description: Use harness2 when the person wants their coding agent's setup (CLAUDE.md, AGENTS.md, skills, subagents, hooks) improved on their own tasks, or asks to set up, run, follow, evaluate or apply a harness2 run. +--- + +""" + + +def add_parser(sub) -> argparse.ArgumentParser: + p = sub.add_parser( + "guide", + help="print the guide for AI agents that use harness2", + description="Print how an AI agent should drive harness2, with this project's current " + "state. Point your agent at it, or install it as a Claude Code skill.", + ) + p.add_argument( + "--install", + choices=("claude",), + help="install the guide as a user-level Claude Code skill (~/.claude/skills/harness2)", + ) + return p + + +def run(args, ctx) -> int: + if args.install: + path = Path.home() / ".claude/skills/harness2/SKILL.md" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(SKILL_HEAD + GUIDE, encoding="utf-8") + ctx.out( + {"installed": str(path)}, + lambda data: ( + f"Installed {data['installed']}. Claude Code loads it in new " + "sessions; harness2's own runs never do." + ), + ) + return 0 + state = state_lines(ctx) if ctx.project_or_none else "" + text = GUIDE + (f"\n## This project\n\n{state}\n" if state else "") + ctx.out({"guide": GUIDE, "project": state or None}, lambda data: text.rstrip()) + return 0 + + +def state_lines(ctx) -> str: + """The project's settings, tasks and runs, in a few lines an agent can act on.""" + s = ctx.settings + lines = [ + f"- root: {ctx.project}", + f"- executor: {s.get('executor.name')} on {s.get('executor.model') or '(model not set)'}", + f"- improver: {s.get('improver.channel')} on {s.get('improver.model') or '(not set)'}", + f"- harness directory: {s.get('paths.harness')}", + ] + tasks = sorted(p.parent.name for p in s.path("paths.tasks").glob("*/task.md")) + shown = ", ".join(tasks[:30]) + (" ..." if len(tasks) > 30 else "") + lines.append(f"- tasks ({len(tasks)}): {shown or 'none yet'}") + runs = sorted( + s.path("paths.runs").glob("*/run.json"), key=lambda p: p.stat().st_mtime, reverse=True + ) + lines.append( + "- runs, newest first: " + (", ".join(p.parent.name for p in runs[:10]) or "none yet") + ) + return "\n".join(lines) diff --git a/harness2/cli/init.py b/harness2/cli/init.py index 81f63cb0..34f9e4dd 100644 --- a/harness2/cli/init.py +++ b/harness2/cli/init.py @@ -17,6 +17,8 @@ from __future__ import annotations import argparse +import contextlib +import json import os import shutil import subprocess @@ -89,11 +91,21 @@ def add_parser(sub) -> argparse.ArgumentParser: def run(args, ctx) -> int: + if not getattr(args, "json", False): + return _run(args)[0] + with contextlib.redirect_stdout(sys.stderr): # the wizard's text; stdout carries the JSON + code, summary = _run(args) + print(json.dumps(summary)) + return code + + +def _run(args) -> tuple[int, dict]: root = Path(args.project or Path.cwd()).resolve() config = S.project_path(root) if config.exists() and not args.force: - print(f"{config} already exists; pass --force to overwrite it.", file=sys.stderr) - return 1 + message = f"{config} already exists; pass --force to overwrite it." + print(message, file=sys.stderr) + return 1, {"error": message, "exit_code": 1} ask = _asker(interactive=not args.yes and sys.stdin.isatty()) print(f"Setting up Harness² in {root}\n") @@ -102,12 +114,12 @@ def run(args, ctx) -> int: default = next((name for name in ("claude", "codex") if found[name]), None) executor = args.executor or (_pick_executor(ask, found, default) if interactive else default) if executor is None: - print( + message = ( "Neither Claude Code (`claude`) nor Codex (`codex`) is on PATH. Install one, or " - "rerun with --executor command to drive your own agent.", - file=sys.stderr, + "rerun with --executor command to drive your own agent." ) - return 1 + print(message, file=sys.stderr) + return 1, {"error": message, "exit_code": 1} command_lines = "" editable: list[str] = [] @@ -159,7 +171,14 @@ def run(args, ctx) -> int: " harness2 doctor check the agent CLIs, keys and tasks\n" " harness2 run --dry-run preview what a run will do and how many calls it makes" ) - return 0 + return 0, { + "config": str(config), + "executor": executor, + "model": model, + "improver": {"channel": channel, "model": improver}, + "auth": auth, + "harness": harness, + } def _asker(*, interactive: bool): diff --git a/harness2/cli/main.py b/harness2/cli/main.py index 0449ceb8..67c82872 100644 --- a/harness2/cli/main.py +++ b/harness2/cli/main.py @@ -37,6 +37,7 @@ "apply", "config", "chat", + "guide", "setup", "demo", "bridge", diff --git a/tests/test_ui.py b/tests/test_ui.py index 80c32028..53a2549e 100644 --- a/tests/test_ui.py +++ b/tests/test_ui.py @@ -1,6 +1,7 @@ """The interactive commands: init, status, apply and the human selector.""" import argparse +import io import json import os import sys @@ -14,7 +15,8 @@ class _Ctx: - def __init__(self, project: Path): + def __init__(self, project: Path, json: bool = False): + self.args = argparse.Namespace(json=json) self.project = project self.settings = type("S", (), {"path": lambda _self, key: project})() self.logged: list[str] = [] @@ -85,6 +87,39 @@ def perform(settings, adapter, tree, *, base, branch): written, {"CLAUDE.md": "base\nmy rule\n", "a.md": "a run\n", "b.md": "b mine\n"} ) + def test_apply_json_reports_the_plan_and_refuses_to_ask(self): + base, adopted = {"CLAUDE.md": "a\n"}, {"CLAUDE.md": "a\nb\n"} + args = argparse.Namespace( + name=None, task=None, mode="parallel", interactive=False, yes=False, + branch=None, dry_run=False, force=False, + ) # fmt: skip + patches = ( + patch.object(apply.A, "open_run", return_value=(None, self.root / "run")), + patch.object(apply.A, "harness_for", return_value=("consolidated", adopted)), + patch.object(apply.A, "base_tree", return_value=base), + patch.object(apply.A, "project_tree", return_value=dict(base)), + patch.object(apply.A, "dirty_paths", return_value=[]), + ) + for item in patches: + item.start() + self.addCleanup(item.stop) + out = io.StringIO() + with patch("sys.stdout", out): + self.assertEqual(apply.run(args, _Ctx(self.root, json=True)), 1) + self.assertIn("--yes or --dry-run", json.loads(out.getvalue())["error"]) + args.dry_run, out = True, io.StringIO() + with patch("sys.stdout", out), patch("sys.stderr"): + self.assertEqual(apply.run(args, _Ctx(self.root, json=True)), 0) + plan = json.loads(out.getvalue()) + self.assertEqual(plan["changes"][0]["path"], "CLAUDE.md") + self.assertFalse(plan["changes"][0]["applied"]) + + def test_guide_mentions_every_agent_contract(self): + from harness2.cli import guide + + for needle in ("--json", "--detach", "--max-calls", "harness2 stop", "apply", "130"): + self.assertIn(needle, guide.GUIDE) + def test_status_reads_tasks_and_spend_from_events(self): run = self.root / "run" run.mkdir()