diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..bd41756b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,40 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + test: + name: Tests (Python ${{ matrix.python }}, ${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + python: ["3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + with: + python-version: ${{ matrix.python }} + activate-environment: true + - name: Install + run: uv pip install -e . -e plugins/harness2-benchmarks + - name: Core tests + run: python -B -m unittest discover -s tests + - name: Plugin tests + run: python -B -m unittest discover -s plugins/harness2-benchmarks/tests + + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + - run: uvx ruff check harness2 tests plugins + - run: uvx ruff format --check harness2 tests plugins diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..fb7bbf5c --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +__pycache__/ +*.py[cod] +*.egg-info/ +.venv/ +build/ +dist/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..ce151ad7 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,56 @@ +# Changelog + +All notable changes to Harness² CLI are documented here. The format follows +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project uses +[Semantic Versioning](https://semver.org/). + +## [Unreleased] + +The first release of Harness² CLI, a general-purpose version of +[google-research/harness2](https://github.com/google-research/harness2) (forked at +`1cfe254`). The method and its prompts are unchanged; everything around them is new. + +### Added + +- One `harness2` command with subcommands: `init`, `doctor`, `task`, `run`, `status`, + `stop`, `diff`, `eval`, `apply`, `judge`, `report`, `runs`, `config`, `chat`, `guide`, `login`, + `setup`, `demo`, `bridge`, `completion`. +- Your own tasks: `.harness2/tasks//task.md` with optional `task.toml`, run in a clean + git worktree of your repository or in a scratch directory, graded optionally by a `check` + command or a `rubric.md`. +- Claude Code and command executors. A project's own harness (CLAUDE.md, `.claude/skills`, + `.claude/agents`, `.claude/scripts`, hooks; or AGENTS.md, `.agents/`, `.codex/hooks.json`) + is imported as the starting harness. +- Subscriptions: `claude-login/` runs on Claude Code's own login and + `chatgpt-login/` on a Codex login kept for harness2 (`harness2 login codex`), with + personal settings, hooks, plugins and instructions kept out of the runs. +- Any model provider: Anthropic, OpenAI, OpenAI- or Anthropic-compatible servers, Vertex AI + and Bedrock, through provider profiles; `harness2 bridge` serves any of them to Codex. +- Layered settings: defaults, `~/.config/harness2/config.toml`, `.harness2/config.toml`, + profiles, `HARNESS2_
__` variables and flags, with `config show --origin`. +- One final harness per run, built from every task's adopted harness and lesson; + `harness2 eval` compares it with the starting harness, and `harness2 apply` writes it into + the project, keeping edits made since the run started. +- Run control: `--dry-run` call estimates, `--budget-usd` and `--max-calls` limits, + clean stops at a provider's usage limit, Ctrl-C and `harness2 stop` with resume, + `--detach` background runs, relocatable run directories, `events.jsonl`. +- Use by AI agents: `--json` for every command including errors, flags for every prompt, + and `harness2 guide` (also installable as a Claude Code skill). +- Interactive use: the `init` wizard, `status --watch`, `apply -i`, `--selector human`, + `harness2 chat`. +- CI on Linux and macOS with Python 3.12 and 3.13. + +### Changed + +- The distribution is `harness2-cli`; the command and the import package stay `harness2`. +- The agent CLIs are the ones installed on the system instead of pinned downloads. +- Codex harness scripts live in `.agents/scripts/`, so they never collide with a project's + `scripts/`. + +### Removed + +- The seven `harness2-*` scripts' old flags (`--bench`, `--substrate`); the script names + remain as aliases of the new subcommands. +- The Vertex-only configuration in `.env` from the core package. JobBench, LAB and + WorkBuddy-Bench moved, with that configuration, to the + [harness2-benchmarks plugin](plugins/harness2-benchmarks/). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b16bd944..a4e86460 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,33 +1,49 @@ -# How to contribute - -We'd love to accept your patches and contributions to this project. - -## Before you begin - -### Sign our Contributor License Agreement - -Contributions to this project must be accompanied by a -[Contributor License Agreement](https://cla.developers.google.com/about) (CLA). -You (or your employer) retain the copyright to your contribution; this simply -gives us permission to use and redistribute your contributions as part of the -project. - -If you or your current employer have already signed the Google CLA (even if it -was for a different project), you probably don't need to do it again. - -Visit to see your current agreements or to -sign a new one. - -### Review our community guidelines - -This project follows -[Google's Open Source Community Guidelines](https://opensource.google/conduct/). - -## Contribution process - -### Code reviews - -All submissions, including submissions by project members, require review. We -use GitHub pull requests for this purpose. Consult -[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more -information on using pull requests. +# Contributing + +Issues and pull requests are welcome. Harness² CLI is an independent fork of +[google-research/harness2](https://github.com/google-research/harness2); changes here are +not sent upstream. Every pull request runs the tests and ruff in CI, and notable changes go +in [CHANGELOG.md](CHANGELOG.md) under "Unreleased". + +## Development setup + +```bash +uv venv --python 3.12 .venv +source .venv/bin/activate +uv pip install -e . -e plugins/harness2-benchmarks +python -B -m unittest discover -s tests +python -B -m unittest discover -s plugins/harness2-benchmarks/tests +ruff check harness2 tests plugins && ruff format --check harness2 tests plugins +``` + +The tests run offline: they use the deterministic demo adapter and small fake `claude` / +`codex` executables, never a real model. `harness2 demo` exercises the whole method locally +in a few seconds. + +## Layout + +| Path | Contents | +|---|---| +| `harness2/core/` | The method: recursion modes, prompts, evidence, selection, library, scheduler, events | +| `harness2/substrate/` | Executors (Claude Code, Codex, OpenCode, command): editable surfaces, native harness import/export, trajectory renderers | +| `harness2/suite.py`, `harness2/workspace.py` | The built-in `tasks` suite: tasks from `.harness2/tasks/`, rollouts in git worktrees or scratch directories | +| `harness2/settings.py`, `harness2/providers.py`, `harness2/registry.py` | Layered settings, provider profiles, suite discovery | +| `harness2/cli/` | The `harness2` command, one module per subcommand | +| `plugins/harness2-benchmarks/` | The paper's benchmarks as a separate distribution | +| `docs/` | User documentation | + +A new suite is a module exposing `make_adapter(ctx)` (returning a +`harness2.core.spec.BenchmarkAdapter`) and, optionally, `check(ctx)`, `setup(argv)` and +`DESCRIPTION`, registered under the `harness2.suites` entry-point group. The benchmarks plugin +is a complete example. + +## Conventions + +- Keep changes small, and reuse what is already there before adding code. Prefer the + standard library; the core package has no runtime dependencies. +- New behaviour comes with an offline test. +- Text that a model reads (prompts in `harness2/core/prompts.py`, suite prompt profiles) is + part of the method: change it deliberately, and say why in the pull request. +- Commit messages follow [Conventional Commits](https://www.conventionalcommits.org/) + (`feat:`, `fix:`, `refactor:`, `docs:`, `test:`). +- Run the tests and `ruff` before opening a pull request, and describe how you tested. diff --git a/MANIFEST.in b/MANIFEST.in index e7cc8b27..6fe169b9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,8 +1,2 @@ -include README.md .env.example -recursive-include docs *.md -recursive-include scripts *.py *.sh -recursive-include harness2/third_party * -include dependencies/README.md dependencies/PINNED.md +include README.md LICENSE global-exclude __pycache__ *.py[cod] *.log - -recursive-include assets *.png diff --git a/README.md b/README.md index a5e91206..633145bd 100644 --- a/README.md +++ b/README.md @@ -1,187 +1,201 @@

Harness²
- Recursive Agent Harnessing for an Open World + Harness² CLI: tune your coding agent's harness on your own tasks

- Paper + CI + Paper Website - Benchmark guides License

-

- - Install, set up, and run JobBench: uv pip install -e '.[render]', harness2-setup jb, then harness2 --bench jb --k 1 --mode parallel. Click for prerequisites and commands. - -

- -A **harness** is the system around a model that enables it to act as an agent: -prompts, tools, memory, skills, orchestration, and more. **Harnessing** is the -process of optimizing that system. - -Harness² is a plug-and-play framework for **recursive agent harnessing** in an -open world, where agents encounter tasks unseen during harness design. It -connects two levels of recursion while keeping model weights fixed: - -* **Task-level recursion** refines the execution harness for the current task - through a contrastive **Propose–Probe–Compose** step: propose edits, probe - their effects on the same task, and compose a refined harness from behavioral - contrasts between executions, without ground-truth feedback. -* **Harnessing-level recursion** updates a persistent *improvement harness* - with adopted harnesses and evidence-supported lessons from each task to guide - future refinement. - -

- - Harness² pipeline. Task-level recursion: an agent executor runs the initial harness, a harness editor proposes invariant and domain edits, candidate probes execute them, and a harness composer combines them into the final harness. Harnessing-level recursion: an experience reflector reflects on the step and updates the improvement harness used for the next task. - -

- -Across three professional benchmarks and four model–harness configurations, a -single Harness² step improves GLM-5 and Gemini 3.5 Flash over their base -harnesses, with gains of up to 11.5 points on WorkBuddy-Bench, 4.5 points on -JobBench, and 11.8 percentage points on LAB's partial pass rate. Increasing the -task-level recursion budget brings further gains. +A coding agent is a model plus a **harness**: the instructions, skills, subagents, helper +scripts and hooks it runs with. `harness2` improves the harness of the agent you already use +(Claude Code's `CLAUDE.md` and `.claude/`, or Codex's `AGENTS.md` and `.agents/`) by running +it on tasks from your own project, studying what happened, and refining the harness from +that evidence. The model stays the same. Your files change only when you apply the result. + +Harness² CLI is a general-purpose version of +[Harness²: Recursive Agent Harnessing for an Open World](https://arxiv.org/abs/2610.14224). +The original release ([google-research/harness2](https://github.com/google-research/harness2)) +reproduces the paper on three fixed benchmarks with Vertex AI. This fork turns the same method +into a command-line tool for any project, any task, Claude Code or Codex (or your own agent), +on your subscription or any model provider. [What changed](#what-changed-from-the-original) +lists the differences; the paper's benchmark setup lives on as a +[plugin](plugins/harness2-benchmarks/). + +- [How it works](#how-it-works) +- [Install](#install) +- [Quick start](#quick-start) +- [Driving it from an agent](#driving-it-from-an-agent) +- [Commands](#commands) +- [Configuration](#configuration) +- [What changed from the original](#what-changed-from-the-original) +- [Documentation](#documentation) -The release supports **parallel and sequential recursion**, **OpenCode and -Codex executors**, and **JobBench, LAB, and WorkBuddy-Bench**. +## How it works -## Video overview +For every task you give it, `harness2`: -https://github.com/user-attachments/assets/76b10548-b14b-4c8f-9aa5-a93662df4d56 +1. runs your agent with its current harness and records what it does; +2. has an improver agent propose harness edits from those trajectories, from two angles: + how the agent works, and what it needs to know about this kind of task; +3. runs the agent with each candidate harness, and composes a refined one from what the + differences between the runs show; +4. keeps the refined harness and a short lesson for the next task in the same area. -## Contents +When all tasks are done, it combines what the tasks taught into **one final harness** for +your project. That is the single result you review with `harness2 diff` and write into the +project with `harness2 apply`. -- [Video overview](#video-overview) -- [Harness components](#harness-components) -- [Install](#install) -- [Run a benchmark](#run-a-benchmark) -- [Experiments](#experiments) -- [Web access](#web-access) -- [Results and resume](#results-and-resume) -- [Citation](#citation) -- [Contributing](#contributing) -- [License](#license) -- [Disclaimer](#disclaimer) - -## Harness components - -Harness² exposes eight editable components around the model: **system prompt, -guardrails, memory, tool guidance, subagents, skills, scripts, and plugins**. -They control context construction, the tool interface, delegation, and behavior -inside the tool loop. +No step needs a grade or a reference answer, so any task you can describe can be used. If +you can also check results, with a test command or a rubric, `harness2 eval` measures the +final harness against the one you started with. [How it works](docs/how-it-works.md) has +the details.

- - Eight harness components grouped by runtime role: system prompt, guardrails, memory, and skills construct context; tool guidance and scripts define the tool interface; subagents orchestrate delegated work; plugins control middleware. + + Harness² pipeline: an agent runs the initial harness, an editor proposes general and domain-specific edits, candidate runs probe them, and a composer combines them into the final harness. A reflector then updates the experience carried to the next task.

## Install -Use **Linux, Python 3.12, and [uv](https://docs.astral.sh/uv/getting-started/installation/)**. -From the repository root: +You need Python 3.12 or 3.13, git, and the agent CLI whose harness you want to tune, +installed, on your `PATH` and logged in: +[Claude Code](https://docs.claude.com/en/docs/claude-code) (`claude`) or +[Codex](https://github.com/openai/codex) (`codex`). ```bash -uv venv --python 3.12 .venv -source .venv/bin/activate -uv pip install -e '.[render]' -harness2-demo --output runs/demo --k 2 +uv tool install git+https://github.com/3ae3ae/harness2-cli +# or: pipx install git+https://github.com/3ae3ae/harness2-cli ``` -The local demo runs both recursion modes with deterministic editors and real -Python executions. It requires no model credentials. +By default runs use your subscription, so a Claude or ChatGPT plan is enough: Claude Code +runs on its own login, and Codex on a login kept for harness2 that you make once with +`harness2 login codex`. You +can use an API key or another provider instead; see [Configuration](#configuration). Runs +make many model calls and use money or subscription quota: the [quick start](#quick-start) +shows how to see the number of calls first and how to cap them. -For model-backed benchmarks, install **Node.js 20+, the Google Cloud CLI**, and -the [benchmark dependencies](docs/SETUP.md#requirements). LAB needs Pandoc and -Poppler; WorkBuddy-Bench needs Docker. Then configure your models: +## Quick start + +Inside the project whose agent setup you want to improve: ```bash -cp .env.example .env -# Edit .env with your Vertex AI project and model IDs. -gcloud auth application-default login +harness2 init # pick the agent, how it logs in, where the harness lives +harness2 task new fix-login-redirect # then write the request in .harness2/tasks/fix-login-redirect/task.md +harness2 doctor # check the CLI, its login and the tasks +harness2 run --dry-run # the tasks and how many model calls a run makes +harness2 run --max-calls 200 # refine the harness on every task, then build the final one ``` -See the [setup guide](docs/SETUP.md) for model settings, authentication -alternatives, and the [Codex executor](docs/SETUP.md#codex-executor). +A run works in temporary git worktrees, so your working tree is never touched. While it +runs, from another terminal: -## Run a benchmark +```bash +harness2 status --watch # progress, spend, the harness picked per task +``` -The following example installs LAB and runs one task, then evaluates and reports -its result. Model-backed runs incur API costs; start with a small task selection. +When it has finished: ```bash -harness2-setup lab -harness2-check --bench lab -harness2 --bench lab --k 1 --mode parallel \ - --only-domains antitrust-competition --max-tasks 1 -harness2-judge --bench lab --k 1 --mode parallel -harness2-report --bench lab --k 1 --mode parallel +harness2 diff # what the final harness changes +harness2 eval # optional: compare it with your current harness on graded tasks +harness2 apply # review the changes and write them into the project ``` -Sequential recursion runs the same commands with `--mode sequential`. `--k` is -the task-level recursion budget; see [Experiments](#experiments). Judging and -reporting must use the same benchmark, executor, subset, `--k`, `--mode`, and -seed as the refinement run. Add `--dry-run` to `harness2` to preview task -selection without model calls. +Running `harness2 run` again resumes an interrupted or limited run and reuses every finished +step. Write a few varied tasks that look like the work you really do: see +[Tasks](docs/tasks.md). -| Benchmark | Tasks | Install | Guide | -|---|---|---|---| -| JobBench | Professional work | `harness2-setup jb` | [JobBench](docs/BENCHMARKS.md#jobbench) | -| LAB | Legal work | `harness2-setup lab` | [LAB](docs/BENCHMARKS.md#lab) | -| WorkBuddy-Bench | Office, web, and code | `harness2-setup wb` | [WorkBuddy](docs/BENCHMARKS.md#workbuddy-bench) | +## Driving it from an agent -Choose one benchmark per command. WorkBuddy also takes `--subset office`, -`--subset web`, or `--subset code` on every command after setup. The -[benchmark guides](docs/BENCHMARKS.md) cover each one's data, domains, and -grading. +`harness2` is built to be driven by an AI agent as well as by hand. Every command has +`--json`, errors included; no command waits for input without a terminal, since every prompt +has a flag; and long runs go to the background with `harness2 run --detach`, are followed with +`harness2 status --json` and are stopped with `harness2 stop`. -## Experiments +`harness2 guide` prints the contract for agents, with your project's current state. Tell your +agent to read it, or install it as a Claude Code skill: -Scale task-level recursion in two ways: **parallel recursion** explores multiple -candidate harnesses and composes refinements from their execution evidence; -**sequential recursion** carries the refined harness and accumulated evidence -through successive Propose–Probe–Compose steps. +```bash +harness2 guide --install claude +``` -

- - Two ways to scale task-level recursion. Parallel: probe k general and k domain-specific candidate harnesses, then produce k compositions from the shared evidence. Sequential: carry the refined harness and accumulated evidence through k Propose–Probe–Compose steps. - -

-

Parallel recursion explores wider; sequential recursion refines deeper.

+## Commands -| Option | Behavior | +| Command | What it does | |---|---| -| `--k 1` | One Propose–Probe–Compose step | -| `--k K --mode parallel` | Probe `K` general and `K` domain-specific candidates, then produce `K` compositions | -| `--k K --mode sequential` | Refine the harness through `K` successive steps | -| `--substrate cx` | Use Codex instead of the default OpenCode executor | +| `harness2 init` | Set up `.harness2/` in a project, with a short wizard (`--yes` for defaults) | +| `harness2 doctor` | Check the agent CLIs, their login or keys, settings and tasks; `--live` also makes one tiny call per model | +| `harness2 task new / ls / import` | Create, list and import tasks | +| `harness2 run` | Refine the harness on the tasks and build the final harness; `--dry-run` previews, `--max-calls` / `--budget-usd` cap spending, `--detach` runs in the background, `--selector human` lets you pick between candidates | +| `harness2 status` | A run's progress; `--watch` keeps it updated | +| `harness2 stop` | Stop a running run; running it again resumes | +| `harness2 diff` | What the final harness changes | +| `harness2 eval` | Compare the final harness with the starting one on graded tasks | +| `harness2 apply` | Write the final harness into the project after showing the changes; `-i` to accept file by file | +| `harness2 judge` / `report` | Grade each task's adopted result, and summarise the grades | +| `harness2 runs ls / show / rm` | List, inspect and delete runs | +| `harness2 config show / get / set` | Read and change settings; `show --origin` says where each value comes from | +| `harness2 login` | Check the subscription logins; `login codex` signs Codex in for harness2 | +| `harness2 chat` | Talk to your agent CLI about tasks, runs and results | +| `harness2 guide` | The guide for AI agents that use harness2 | +| `harness2 demo` | A local run with scripted editors and no model calls | + +Every command takes `--help`. The [command reference](docs/cli.md) lists every flag, the exit +codes and the layout of a run directory. + +## Configuration + +Settings come from `~/.config/harness2/config.toml`, the project's `.harness2/config.toml`, +`HARNESS2_*` environment variables and flags, in that order of precedence. Models are +written `provider/model`: + +```toml +[executor] +name = "claude" +model = "claude-login/claude-sonnet-5-5" # your Claude subscription, through Claude Code + +[improver] +channel = "claude" +model = "claude-login/claude-opus-5-5" +``` -After each task, harnessing-level recursion retains the adopted harness and -evidence-supported lessons to guide future refinement. See the -[experiment guide](docs/EXPERIMENTS.md) for task selection, concurrency, -reproducibility, and baseline comparisons. +`chatgpt-login/` uses your ChatGPT subscription through Codex (after +`harness2 login codex`). API keys and other +endpoints work too: Anthropic, OpenAI, any OpenAI-compatible or Anthropic-compatible +server, Vertex AI and Bedrock. To tune an agent other than Claude Code or Codex, the +`command` executor runs your own command and edits the files you list. +[Configuration](docs/configuration.md) covers every setting. -## Web access +## What changed from the original -Executor network access is unrestricted, and Codex runs with its sandbox -disabled. Public benchmark rubrics may be accessible online. Before evaluation, -configure [network restrictions](docs/EXPERIMENTS.md#web-access) to prevent -access to benchmark answers and grading criteria. +| | google-research/harness2 | Harness² CLI | +|---|---|---| +| Tasks | JobBench, LAB, WorkBuddy-Bench | your own tasks, in your repository or from files; the benchmarks as a plugin | +| Agents | OpenCode, Codex, pinned builds | Claude Code, Codex or any command, as installed on your system | +| Models | Vertex AI | your Claude or ChatGPT subscription, or any API provider | +| Result | one adopted harness per task, kept for scoring | one final harness for the project, which you review and apply | +| Interface | seven scripts, settings in `.env` | one `harness2` command, layered TOML settings, JSON output, background runs, resumable limits | -## Results and resume +The method itself (propose, probe, compose, and the library of lessons) and its prompts are +the paper's. [How it works](docs/how-it-works.md) describes what was added around it. -Results are saved under `runs/`, organized by benchmark and subset where -applicable. They include task summaries, deliverables, adopted harnesses, -reflection notes, and evaluation results. +## Documentation -Repeat the same command to resume. Use a new `HARNESS2_RUNS_DIR` when changing -models or task selection. See [results and resume](docs/EXPERIMENTS.md#results-and-resume) -for details; each command also supports `--help`. +- [How it works](docs/how-it-works.md): the method, the final harness, what a run costs and stores +- [Tasks](docs/tasks.md): writing tasks, workspaces, grading +- [Configuration](docs/configuration.md): settings, subscriptions, models and providers +- [Command reference](docs/cli.md): every command and flag +- [Changelog](CHANGELOG.md) and [contributing](CONTRIBUTING.md) ## Citation +If you use Harness² CLI in research, cite the paper: + ```bibtex @article{xu2026harness2, title = {Harness$^2$: Recursive Agent Harnessing for an Open World}, @@ -192,18 +206,9 @@ for details; each command also supports `--help`. } ``` -## Contributing - -See [CONTRIBUTING.md](CONTRIBUTING.md). - ## License -Harness² is licensed under [Apache 2.0](LICENSE). Third-party code under -[`third_party/`](third_party/README.md), benchmarks, and datasets retain their -upstream licenses; see -[third-party notices](dependencies/README.md). - -## Disclaimer - -This is not an officially supported Google product. This project is not eligible -for the [Google Open Source Software Vulnerability Rewards Program](https://bughunters.google.com/open-source-security). +Apache 2.0; see [LICENSE](LICENSE). The method and the original implementation are by the +paper's authors at Google; Harness² CLI is an independent fork, not a Google product, and is +not maintained by Google. Third-party code in the benchmarks plugin keeps its upstream +licenses. diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 00000000..c5f9319d --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,282 @@ +# Command reference + +``` +harness2 [global options] [options] +``` + +Every command takes `--help`. `python -m harness2` is the same as `harness2`. + +- [Global options](#global-options) +- [Commands](#commands) +- [Exit codes](#exit-codes) +- [A run on disk](#a-run-on-disk) +- [Shell completion](#shell-completion) + +## Global options + +These go before the command: `harness2 -C ../api --json status`. + +| Option | Meaning | +|---|---| +| `-C DIR`, `--project DIR` | Act as if started in `DIR`. The project is the nearest directory upward containing `.harness2/`. | +| `--profile NAME` | Apply the `[profiles.NAME]` settings. | +| `-c KEY=VALUE`, `--config KEY=VALUE` | Override one setting for this command; repeatable. See [Configuration](configuration.md). | +| `-q`, `--quiet` | Print results only, no progress. | +| `-v`, `--verbose` | Print debug details. | +| `--json` | Print results as JSON on stdout; progress still goes to stderr. A failing command prints `{"error": "...", "exit_code": n}` instead of an error line. | +| `--version` | Print the version. | + +## Commands + +### `init` + +Set up harness2 in a project: asks which agent runs your tasks (`claude`, `codex` or a +custom `command`), whether it reaches its models through the CLI's own login (your +subscription) or an API key, which models to use, and where the harness lives; writes +`.harness2/config.toml`, `.harness2/.gitignore` and `.harness2/tasks/`. + +| Option | Meaning | +|---|---| +| `--yes`, `-y` | Take every default without asking. Without a terminal, defaults are taken anyway. | +| `--executor {claude,codex,command}` | Skip the executor question. | +| `--auth {login,api-key}` | Skip the login question. | +| `--force` | Overwrite an existing config file. | + +### `doctor` + +Check that a run has what it needs: the suite and its tasks, each role's model and +provider, the agent CLIs, and provider keys. Makes no model call unless `--live`. + +| Option | Meaning | +|---|---| +| `--live` | Also call each configured model once, with a tiny request. | + +### `task new | ls | import` + +```bash +harness2 task new ID [--files] [--domain D] [--check CMD] +harness2 task ls +harness2 task import FILE [--force] +``` + +`new` creates `.harness2/tasks/ID/` with a `task.md` to fill in and a `task.toml`. `--files` +makes a files task (with a `files/` directory) instead of a project task. `import` reads a +`.jsonl` or `.csv` file; `--force` overwrites existing tasks. See [Tasks](tasks.md). + +### `run` + +Refine the harness on the tasks, then combine what the tasks taught into one final harness +per recursion mode (one more improver session each), stored in the run's `consolidated/` +directory. Running the same command again resumes the run and reuses every finished step; +if building the final harness failed, only that step runs again. + +| Option | Meaning | +|---|---| +| `--name NAME` | Run name. Default: `-k`, plus `-s` with a seed. | +| `--k K` | Recursion budget per task (default 1). | +| `--mode MODES` | `parallel` (default), `sequential`, or `parallel,sequential`. | +| `--executor NAME` | Override `executor.name`. | +| `--suite NAME` | Task suite: `tasks` (default) or one installed by a plugin. | +| `--split {all,train,test}` | Run one split only. | +| `--only-domains D,...` | Run only these domains. | +| `--only-tasks ID,...` | Run only these tasks. | +| `--max-tasks N` | Run at most N tasks per domain. | +| `--seed N` | Shuffle the task order within each domain. | +| `--workers N` | Domains to run at the same time. | +| `--budget-usd USD` | Stop once the run has spent this much. | +| `--max-calls N` | Stop after N model calls. | +| `--selector human` | Pick the final harness yourself when `k` > 1. | +| `--steer TEXT\|@FILE` | Guidance added to every improver session. | +| `--dry-run` | Show the tasks and the number of model calls; run nothing. | +| `--detach` | Start the run in the background and return at once with its name, pid and log file (`run.log` in the run directory). | +| `--split-file FILE` | Task partition file, for benchmark suites. | + +A run keeps the settings it started with. Resuming it with a different executor, model, +improver or task list is refused, listing what changed; use a new `--name` for a new +experiment. The limits apply to the whole run, across resumes. A run that is already going +is not started twice. + +### `stop [NAME]` + +Stop a running run, whether it was started with `--detach` or in another terminal. It +finishes the model calls in flight and exits; running the same `harness2 run` command again +resumes it. + +### `status [NAME]` + +Each task's state, the harness picked for it, rollouts done, and the run's spend. Works +while the run is going, from another terminal. NAME defaults to the most recent run. + +| Option | Meaning | +|---|---| +| `--watch [SECONDS]` | Redraw every SECONDS (default 5) until Ctrl-C. | + +### `judge [NAME]` + +Grade the result each task adopted, with the task's `check` command or `rubric.md`. Tasks +with neither are skipped. Grades are written to the run and reused until the check, the +rubric or the judge model changes. + +| Option | Meaning | +|---|---| +| `--bases` | Grade the runs of the starting harness instead, for comparison. | + +### `report [NAME ...]` + +Average the grades per recursion mode. Several names compare runs. + +### `runs ls | show NAME | rm NAME` + +List runs with their progress and spend, show one run's state and settings, or delete a run +and everything it stored (`--yes` to skip the question). + +### `diff [NAME]` + +Show how the run's final harness differs from the harness the run started from. + +| Option | Meaning | +|---|---| +| `--task ID` | Show the harness adopted for this one task instead of the final harness. | +| `--mode {parallel,sequential}` | The recursion mode to take it from (default parallel). | +| `--stat` | List the changed files only. | + +### `eval [NAME]` + +Run the agent with the starting harness and with the final harness on the run's tasks that +have a check or rubric, grade both, and compare the pass rates. Costs one agent run per +task, harness and repeat. + +| Option | Meaning | +|---|---| +| `--repeat N` | Runs per task and harness (default 1). More runs, steadier numbers. | +| `--split {all,train,test}` | Compare on one split, for example the `test` tasks the run did not use. | +| `--only-tasks ID,...` | Compare on these tasks only. | +| `--mode {parallel,sequential}` | Which mode's final harness to compare (default parallel). | + +### `apply [NAME]` + +Write the run's final harness into the project. Only what the run changed relative to the +harness it started from is applied, so edits you made since are kept; a file changed both +by the run and in the project is skipped unless `--force`. The changes are listed first. +With `--json`, `--yes` or `--dry-run` is required, since nothing is asked. + +| Option | Meaning | +|---|---| +| `--task ID`, `--mode` | As for `diff`. | +| `-i`, `--interactive` | Show each file's diff and accept or skip it. | +| `-y`, `--yes` | Apply without asking. | +| `--branch NAME` | Create and switch to a git branch first. | +| `--dry-run` | Show the changes, write nothing. | +| `--force` | Also overwrite files changed in the project since the run started, and write over uncommitted changes. | + +Only the executor's harness files are written. For Claude Code those are `CLAUDE.md`, +`.claude/skills/`, `.claude/agents/`, `.claude/scripts/` and the `hooks` key of +`.claude/settings.json` (its other keys are kept). + +### `config show | get | set | path` + +```bash +harness2 config show [--origin] # every resolved setting, and where it came from +harness2 config get KEY +harness2 config set KEY VALUE [--global] +harness2 config path +``` + +`set` edits `.harness2/config.toml` (or the global file with `--global`) in place, keeping +comments. The value is read as a TOML literal when it parses: `run.k 2` is a number. + +### `chat` + +Start your own Claude Code or Codex in the project, with a guide to harness2 and the +project's tasks and runs added to its instructions. Ask it to help write tasks, a check or a +rubric, or to explain a run's results. + +| Option | Meaning | +|---|---| +| `--with {claude,codex}` | Which CLI to start. Default: the improver's channel, if installed. | + +### `login [codex]` + +Without an argument, show whether `claude-login` (Claude Code's own login) and +`chatgpt-login` (the Codex login kept for harness2) are ready. `harness2 login codex` signs +Codex in to ChatGPT for harness2, in its own home, apart from your `~/.codex`; `--device` +uses a device code on machines without a browser. + +### `guide` + +Print the guide for AI agents that drive harness2, followed by the current project's +settings, tasks and runs. `--install claude` installs it as a user-level Claude Code skill +(`~/.claude/skills/harness2/`), which harness2's own runs never load. + +### `setup SUITE [ARGS...]` + +Run a plugin suite's installer, for example `harness2 setup lab` for the +[benchmarks plugin](../plugins/harness2-benchmarks/). + +### `demo` + +Run the method on two toy tasks with scripted editors: no model calls, a few seconds. +`--output DIR` (default `runs/demo`), `--k K` (default 2), `--mode {parallel,sequential,both}`. + +### `bridge` + +Serve a configured model as an OpenAI Responses API endpoint through LiteLLM, so the Codex +executor or channel can use providers it cannot reach directly. Needs +`pip install 'harness2-cli[bridge]'`. + +| Option | Meaning | +|---|---| +| `--model PROVIDER/MODEL` | The model to serve (required). | +| `--host HOST` | Interface to listen on (default `127.0.0.1`). | +| `--port PORT` | Port. Default: the one used last time, or a free one. | + +It prints the `[providers.bridge]` table to add to your config and the key to put in +`.harness2/.env`. + +### `completion {bash,zsh}` + +Print a shell completion script; see [below](#shell-completion). + +## Exit codes + +| Code | Meaning | +|---|---| +| 0 | Success. | +| 1 | An error: bad settings, a missing CLI or key, a failed check, a run that could not continue. | +| 2 | Invalid command-line usage. | +| 3 | `run` finished, but some tasks failed. | +| 4 | `run` or `eval` stopped at `--budget-usd`, `--max-calls` or the provider's usage limit. Rerun to resume. | +| 130 | Interrupted with Ctrl-C. Rerun to resume. | + +## A run on disk + +``` +.harness2/runs// + run.json settings, task list and schedule the run started with + run.pid, run.log while it runs: its process, and its output when started with --detach + events.jsonl one line per agent run, model session, selection and task, with costs + consolidated// the final harness, the improver's notes and what it was built from + eval// results of harness2 eval + base_harness/ the project's harness when the run started + rollouts// one agent run: prompt.md, stream.jsonl (trajectory), final.md, + deliverable.patch or output/, meta.json, judge.json + adapt/// + summary.json what happened for the task + harnesses/ every candidate harness + selection/ the final pick and why + reflection/ the lesson recorded for later tasks + adapt/_audit/ every improver session's mission and reply + library/ adopted harnesses and lessons per domain +``` + +Stored paths are relative, so a run directory can be moved or copied and still resumes. + +## Shell completion + +```bash +# zsh +harness2 completion zsh > "${fpath[1]}/_harness2" +# bash +mkdir -p ~/.local/share/bash-completion/completions +harness2 completion bash > ~/.local/share/bash-completion/completions/harness2 +``` diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 00000000..7679b621 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,252 @@ +# Configuration + +Harness² reads its settings from TOML files, environment variables and command-line flags. +Every setting has a default, so a project only needs the few keys that differ. + +- [Where settings come from](#where-settings-come-from) +- [Models and providers](#models-and-providers) +- [Reference](#reference) +- [Profiles](#profiles) +- [Secrets](#secrets) + +## Where settings come from + +Later sources override earlier ones: + +| Source | Location | +|---|---| +| Defaults | built in | +| Global file | `~/.config/harness2/config.toml` (or `$XDG_CONFIG_HOME/harness2/config.toml`) | +| Project file | `/.harness2/config.toml` | +| Profile | a `[profiles.]` table, selected with `--profile ` or `HARNESS2_PROFILE` | +| Environment | `HARNESS2_
__`, for example `HARNESS2_EXECUTOR__MODEL` | +| Command line | `-c key=value` (repeatable) and command flags such as `--k 2` | + +The project is the nearest directory, upward from the current one, that contains +`.harness2/`. `-C ` starts the search somewhere else. `harness2 init` creates +`.harness2/` with a starter `config.toml`. + +To see the resolved settings and where each one comes from: + +```bash +harness2 config show --origin +harness2 config get executor.model +harness2 config set run.workers 2 # edits .harness2/config.toml, keeps comments +harness2 config set --global providers.local.base_url http://localhost:8000/v1 +``` + +Values given in the environment or with `-c` are read as TOML literals, so `-c run.k=2` is +a number and `-c 'executor.editable=["prompts/*.md"]'` is a list. Anything that does not +parse as TOML is taken as a plain string: `-c executor.model=openai/gpt-6.1-sol`. + +## Models and providers + +Every model is written `provider/model`. The provider part names a `[providers.]` +table, which says what kind of endpoint it is and how it is reached. + +### Subscriptions + +Two built-in providers use the login your installed CLI already has, so a subscription is +all you need: + +| Model | Runs on | +|---|---| +| `claude-login/`, e.g. `claude-login/claude-sonnet-5-5` | your Claude subscription, through Claude Code's login (`claude auth login`) | +| `chatgpt-login/`, e.g. `chatgpt-login/gpt-6.1-sol` | your ChatGPT subscription, through a Codex login kept for harness2 (`harness2 login codex`, once) | + +`harness2 init` picks these by default, and `harness2 login` and `harness2 doctor` say +whether they are ready. Runs keep your personal setup out of the harness being measured: + +- **Claude Code** runs on its own login, with `--setting-sources project,local + --strict-mcp-config --no-session-persistence`: your `~/.claude/CLAUDE.md`, skills, plugins, + hooks, MCP servers and settings are not loaded, and no session history is written. +- **Codex** cannot switch off a personal `~/.codex/hooks.json` or `~/.codex/AGENTS.md` with + a flag, so harness2 gives it a home of its own, `~/.local/state/harness2/codex-home`. + `harness2 login codex` signs that home in to ChatGPT once. It is a separate login: your own + `~/.codex` is never read or changed. Runs also pass `--ignore-user-config --ignore-rules`. + +To run Codex on your own home anyway, with whatever hooks and instructions it holds, set +its path; `harness2 doctor` then lists the personal files every run will include: + +```toml +[providers.chatgpt-login] +home = "~/.codex" +``` + +Subscriptions have usage limits. When a run reaches one, it stops cleanly (exit code 4), +keeps everything finished so far, and the same `harness2 run` command resumes once the +limit resets. With a subscription, keep `run.phase_width` and `run.proposer_concurrency` +low, and cap runs with `--max-calls`. + +### API keys and other endpoints + +Two API-key providers are built in: + +```toml +[providers.anthropic] +type = "anthropic" +api_key_env = "ANTHROPIC_API_KEY" + +[providers.openai] +type = "openai" +base_url = "https://api.openai.com/v1" +api_key_env = "OPENAI_API_KEY" +``` + +So `anthropic/claude-sonnet-5-5` and `openai/gpt-6.1-sol` work as soon as the matching key +is set. Add your own provider for anything else: + +```toml +# An OpenAI-compatible server: vLLM, LM Studio, Ollama, a company gateway ... +[providers.local] +type = "openai" +base_url = "http://localhost:8000/v1" +api_key_env = "LOCAL_API_KEY" + +# An Anthropic-compatible gateway +[providers.gateway] +type = "anthropic" +base_url = "https://llm-gateway.example.com" +api_key_env = "GATEWAY_KEY" + +# Google Vertex AI (uses Application Default Credentials) +[providers.vertex] +type = "vertex" +project = "my-gcp-project" +region = "global" + +# Amazon Bedrock (uses the usual AWS credentials) +[providers.bedrock] +type = "bedrock" +region = "us-east-1" +``` + +Each agent CLI speaks to some provider types and not others: + +| Provider type | Claude Code (`claude`) | Codex (`codex`) | OpenCode (`opencode`) | +|---|---|---|---| +| `claude-login` | yes | no | no | +| `chatgpt-login` | no | yes | no | +| `anthropic` | yes | via `harness2 bridge` | yes | +| `openai` | no | yes (Responses API) | yes (Chat Completions) | +| `vertex` | yes | via `harness2 bridge` | yes | +| `bedrock` | yes | via `harness2 bridge` | no | + +`harness2 doctor` reports a model whose provider its CLI cannot use. To run Codex on a model +it cannot reach directly, serve that model as a Responses API endpoint with the bridge +(`pip install 'harness2-cli[bridge]'`), then point an `openai`-type provider at it: + +```bash +harness2 bridge --model anthropic/claude-sonnet-5-5 +``` + +The agent CLIs are the ones installed on your system (`claude`, `codex`, `opencode` on +`PATH`). Set `executor.bin` or `improver.bin` to use another binary or a full command. + +With an API-key provider, every run also gets a fresh, empty configuration directory +(`CLAUDE_CONFIG_DIR` for Claude Code, `CODEX_HOME` for Codex), the strictest isolation. +`harness2 chat` is different from runs: it starts your own CLI exactly as you normally use +it. + +## Reference + +### `[executor]`: the agent that runs the tasks + +| Key | Default | Meaning | +|---|---|---| +| `name` | `"claude"` | `claude`, `codex` or `command`. Plugin suites accept their own executors (the benchmark plugin uses `opencode` and `codex`). | +| `model` | `""` | `provider/model` the executor runs on. Required for `claude` and `codex`. | +| `bin` | `""` | Binary or command to start the CLI. Empty means the executor name on `PATH`. | +| `effort` | `"high"` | Reasoning effort passed to the CLI. | +| `timeout` | `1800` | Seconds one rollout may take. A task's `timeout` overrides it. | +| `command` | `""` | `command` executor only: the shell command that runs one task. `{workspace}`, `{prompt_file}` and `{output}` are replaced with quoted paths. | +| `editable` | `[]` | `command` executor only: glob patterns, relative to `paths.harness`, of the files the improver may edit. | + +### `[improver]`, `[selector]`, `[reflector]`, `[judge]`: the agents that work on the harness + +The improver proposes and composes harness edits. The selector picks the final harness when +several candidates compete (only when `k` > 1). The reflector writes the lesson that carries +over to later tasks. The judge grades tasks that have a `rubric.md`. + +`[improver]` holds the defaults for all four. `[selector]` and `[judge]` override the +improver; `[reflector]` overrides the selector, or the improver when the selector is a +person. + +| Key | Default | Meaning | +|---|---|---| +| `channel` | `"claude"` | Which CLI runs the session: `claude`, `codex` or `opencode`. `[selector]` also accepts `human`. | +| `model` | `""` | `provider/model`. Required for the improver. | +| `bin` | `""` | Binary or command for the channel's CLI. | +| `effort` | `"high"` | Reasoning effort. | +| `timeout` | `3600` | Seconds one session may take. | +| `steps` | `120` | Step limit for OpenCode sessions. | +| `steer` | `""` | Improver only: guidance added to every improver session, as text or `@path/to/file`. Describe what you want from the agent's work, not how it will be measured. | + +### `[paths]` + +| Key | Default | Meaning | +|---|---|---| +| `harness` | `"."` | Where the harness files live, relative to the project root. | +| `tasks` | `".harness2/tasks"` | The task directory. | +| `runs` | `".harness2/runs"` | Where runs are stored. | + +### `[run]` + +| Key | Default | Meaning | +|---|---|---| +| `k` | `1` | Task-level recursion budget. See [how it works](how-it-works.md#scaling-with-k). | +| `modes` | `"parallel"` | `parallel`, `sequential`, or both separated by a comma. | +| `seed` | unset | Shuffles the task order within each domain. Unset keeps sorted order. | +| `workers` | `1` | Domains run at the same time. | +| `phase_width` | `2` | Rollouts and improver sessions run at the same time within one task. | +| `proposer_concurrency` | `2` | Model sessions running at the same time across the whole run. | +| `transport_attempts` | `2` | Attempts per improver session before it counts as failed. | +| `obs_trunc` | `50000` | Characters of each tool output kept when trajectories are shown to the improver. | +| `budget_usd` | `0` | Stop the run once it has spent this many dollars. `0` means no limit. | +| `max_calls` | `0` | Stop the run after this many model calls (rollouts plus sessions). `0` means no limit. | + +Spending is counted from the costs the CLIs report. Claude Code and OpenCode report dollars; +Codex reports tokens only, so `budget_usd` does not see Codex spending, and with a +subscription the reported dollars are an estimate rather than a charge. Use `max_calls` to +bound those runs. A call already in flight when a limit is reached still finishes. + +### `[suite]` + +| Key | Default | Meaning | +|---|---|---| +| `name` | `"tasks"` | Which suite supplies the tasks: `tasks` (your own) or a suite installed by a plugin. | +| `work` | `""` | One sentence describing the work, shown to the improver. For example: "maintain a Django web service and its test suite". | + +## Profiles + +A profile is a named set of overrides you switch on with `--profile`. Put profiles in the +global file to share them between projects: + +```toml +[profiles.cheap.executor] +model = "anthropic/claude-haiku-5-5" + +[profiles.cheap.improver] +model = "anthropic/claude-sonnet-5-5" + +[profiles.cheap.run] +max_calls = 200 +``` + +```bash +harness2 run --profile cheap +``` + +## Secrets + +Keys are never written into config files. A provider names the environment variable that +holds its key (`api_key_env`), and a config file that contains a literal `api_key` is +rejected. Put project-specific keys in `.harness2/.env`, which `harness2 init` adds to +`.harness2/.gitignore`: + +```bash +# .harness2/.env +ANTHROPIC_API_KEY=sk-ant-... +``` + +Variables already set in your shell win over `.harness2/.env`. diff --git a/docs/how-it-works.md b/docs/how-it-works.md new file mode 100644 index 00000000..cc32aee2 --- /dev/null +++ b/docs/how-it-works.md @@ -0,0 +1,133 @@ +# How it works + +Harness² improves an agent without touching the model. It changes the **harness**: the +files the agent reads and runs around every task. For Claude Code those are `CLAUDE.md`, +skills, subagents, helper scripts and hooks; for Codex, `AGENTS.md`, skills, scripts and +hooks. The method is described in the paper +[Harness²: Recursive Agent Harnessing for an Open World](https://arxiv.org/abs/2610.14224). +This page explains what a run does, what it costs, and what you get back, including the +parts Harness² CLI adds to the paper's method: one final harness per run, `eval` and +`apply`. + +- [The harness components](#the-harness-components) +- [One task: propose, probe, compose](#one-task-propose-probe-compose) +- [Scaling with k](#scaling-with-k) +- [Picking each task's harness](#picking-each-tasks-harness) +- [Carrying lessons forward](#carrying-lessons-forward) +- [One harness for the project](#one-harness-for-the-project) +- [What a run gives you](#what-a-run-gives-you) + +## The harness components + +Each component has a different cost and reach, and the improver is told to use the cheapest +one that fixes what it saw: + +| Component | Claude Code | Codex | In context | +|---|---|---|---| +| Always-on instructions | `CLAUDE.md` | `AGENTS.md` | every turn, so every word costs | +| Skills | `.claude/skills//SKILL.md` | `.agents/skills//SKILL.md` | only when invoked | +| Subagents | `.claude/agents/.md` | — | in their own context | +| Helper scripts | `.claude/scripts/*.py` | `.agents/scripts/*.py` | only when run | +| Hooks | `hooks` in `.claude/settings.json` | `.codex/hooks.json` | inside the tool loop, cannot be skipped | + +With the `command` executor the harness is whatever files `executor.editable` matches. + +Harness² edits only these files. Anything else in `.claude/` or in a skill directory, such as +reference documents, is left alone. + +## One task: propose, probe, compose + +For each task, a run: + +1. **Executes** the agent with the current harness, `k` times, to see how it behaves today. +2. **Proposes** edits. Two improvers read those trajectories and results independently. One + works on *how the agent works*: procedure, tool use, checking its own result. The other + works on *what a practitioner knows* about this kind of task. Each writes a candidate + harness. +3. **Probes** every candidate by running the agent with it on the same task. +4. **Composes** a refined harness from the contrast between all those runs: which edits + changed behaviour, and how. The refined harness is run too. + +No step looks at a grade. The improvers see trajectories and results, and decide from the +differences between runs what helped. This is what lets Harness² work on tasks that have no +test at all. + +If the evidence does not support a change, an improver can keep the harness as it is. An +edit that breaks the harness's rules (a skill without a description, a script that is +referenced but missing, malformed hooks) is sent back once with the problems listed, and +otherwise dropped. + +## Scaling with k + +`k` (`--k`, default 1) is how much effort each task gets. There are two ways to spend it: + +- **parallel** (`--mode parallel`, the default): `k` "how it works" and `k` "what it knows" + candidates are proposed from the same evidence, and `k` composed finals are built from all + of them. Wider exploration. +- **sequential** (`--mode sequential`): `k` rounds of propose–probe–compose, each starting + from the previous round's result and seeing all earlier evidence. Deeper refinement. + +Per task and mode, a run makes **4k agent runs** (including the `k` runs of the current +harness, which both modes share) and **3k improver sessions**, plus one reflector session +that writes the task's lesson and, when `k` is more than 1, one selector session that picks +the task's harness. Building the final harness adds one session per run and mode. +`harness2 run --dry-run` prints the exact counts for your tasks before anything is spent, +and `--budget-usd` / `--max-calls` stop a run at a limit. + +## Picking each task's harness + +With `k` = 1 there is one composed harness per task. With more, a **selector** compares the +composed harnesses' results blind: it sees the task and each result under an anonymous id, +never which harness produced it or how that harness was built, and picks the best. That pick +is the task's **adopted** harness. + +You can make the pick yourself with `harness2 run --selector human`: the run stops at each +selection and shows you the anonymous candidates in the terminal. + +## Carrying lessons forward + +After a task adopts a harness, a **reflector** looks at which edits made a difference and +writes a short lesson. The adopted harness and its lesson go into a **library** for the +task's domain. Later tasks in the same domain show the library to their improvers, so useful +components get reused and verified lessons build up. This is the second level of recursion: +the harness that improves harnesses gets better as tasks go by. + +Tasks in a domain run one after another so the library can grow; separate domains run in +parallel (`--workers`). Each run has its own library, so experiments never mix. + +## One harness for the project + +In the paper, every task ends with its own adopted harness: the method adapts the agent to +the task in front of it, and the adopted harnesses are kept for scoring and as examples for +later tasks. A project needs one harness that it can keep. So when all tasks are done, +`harness2 run` adds one step of its own: an improver session reads the starting harness, +every task's request, the harness adopted for it and the lesson recorded, and composes **one +final harness** for the project's next tasks. It keeps what helped on several tasks or what +a lesson supports as general, and drops facts tied to a single task. The result is checked +against the same rules as every other proposal. + +That final harness is what you work with: + +- `harness2 diff` shows how it differs from the harness you started with; +- `harness2 eval` runs the agent with both on the tasks that have a check or rubric and + compares the pass rates, the evidence for whether it actually helps; run it on `test` + tasks the run did not use for the fairest comparison; +- `harness2 apply` writes it into the project, applying only what the run changed, so edits + you made meanwhile are kept, and touching nothing but the harness files. + +## What a run gives you + +Everything is stored under `.harness2/runs//`: + +- `run.json`: the settings and task list the run was started with. +- `events.jsonl`: one line per agent run, session and decision, with costs. +- `adapt/`: per task, every candidate harness, the improvers' notes, the selection and the + lesson. +- `rollouts/`: per agent run, the trajectory, the result and the final message. +- `library/`: the adopted harnesses and lessons per domain. +- `consolidated/`: the final harness and the improver's notes on why each change is there. + +Use `harness2 status` while it runs, `harness2 diff`, `harness2 eval` and `harness2 apply` on +the final harness, and `harness2 judge` / `harness2 report` to grade each task's adopted +result. Running the same `harness2 run` command again resumes where it stopped, reusing every +finished step. diff --git a/docs/tasks.md b/docs/tasks.md new file mode 100644 index 00000000..83904056 --- /dev/null +++ b/docs/tasks.md @@ -0,0 +1,120 @@ +# Tasks + +A task is one piece of work you want your agent to get better at: a bug to fix, a feature to +add, a report to write. Harness² runs the agent on each task, studies how the runs went, and +refines the agent's harness on that evidence. Good tasks are real work, phrased the way you +would actually ask, and varied enough that a better harness has to be better in general. + +- [Layout](#layout) +- [Workspaces](#workspaces) +- [Deliverables](#deliverables) +- [Grading](#grading) +- [Creating and importing tasks](#creating-and-importing-tasks) + +## Layout + +Each task is a directory under `.harness2/tasks/` (the `paths.tasks` setting). Its name is the +task id: lowercase letters, digits, `.`, `_` and `-`. + +``` +.harness2/tasks/ + fix-login-redirect/ + task.md the instruction the agent receives (required) + task.toml options (optional) + check/ files only the grader sees (optional) + rubric.md criteria for an LLM grader (optional) + summarize-incidents/ + task.md + task.toml + files/ input files copied into the workspace (optional) +``` + +`task.md` is given to the agent as-is. Write it as a request, not as a test: the agent, the +improver and the selector all read it, and none of them ever sees `check/` or `rubric.md`. + +`task.toml` keys, all optional: + +| Key | Default | Meaning | +|---|---|---| +| `domain` | `"default"` | Groups related tasks. Lessons and adopted harnesses carry over only between tasks of the same domain, and domains can run in parallel. | +| `split` | none | `"train"` or `"test"`, for `harness2 run --split`. A task without one is only in `all`. | +| `workspace` | `"files"` if `files/` exists, else `"project"` | Where the agent works. See [Workspaces](#workspaces). | +| `ref` | `"HEAD"` | Project workspaces only: the git revision the agent starts from. | +| `deliverable` | `"diff"` for project workspaces, `"files"` otherwise | What counts as the result. See [Deliverables](#deliverables). | +| `check` | none | A shell command that grades a result. See [Grading](#grading). | +| `timeout` | `executor.timeout` | Seconds one run of this task may take. | + +## Workspaces + +Every run gets a fresh workspace in a temporary directory outside your project, so runs +never touch your working tree and never see each other. + +**`project`**: a clean `git worktree` of your repository at `ref`. The project must be a git +repository, and uncommitted changes are not included: commit what the task should start +from. The `.harness2/` directory and the project's own harness files are removed from the +copy, and the harness being tried is installed in their place. + +**`files`**: an empty directory seeded with the contents of the task's `files/`, plus an +empty `output/` directory. The agent is asked to save its deliverables in `output/`. Use +this for work that is not a change to the repository: documents, analyses, data. + +## Deliverables + +| `deliverable` | The result is | Counts as delivered when | +|---|---|---| +| `diff` | the change the agent made to the repository, as a patch | the patch is not empty | +| `files` | the files in `output/` | `output/` holds at least one file | +| `message` | the agent's final message | the message is not empty | + +The agent's final message is always kept alongside the result. Changes to the harness files +themselves are left out of the patch. + +## Grading + +Refining the harness does not need grades: candidates are compared by an independent +selector that reads their results, never a score. Grades are for you, to measure what a run +achieved. `harness2 judge` grades the result each task finally adopted, and `harness2 report` +summarises the grades. + +**`check`**: a shell command run in a fresh workspace that holds the result: for project +tasks, a new worktree at `ref` with the patch applied; for files tasks, a copy of `files/` +with the result in `output/`. The contents of `check/` are copied in first, so hidden tests +can live there. The command passes when it exits with status 0. For partial credit, print a +JSON object as the last line of output: + +```json +{"n_passed": 7, "n_criteria": 9} +``` + +`HARNESS2_TASK_DIR` and `HARNESS2_RUN_DIR` are set for the command. + +```toml +# task.toml +check = "cp check/test_redirect.py tests/ && python -m pytest -q tests/test_redirect.py" +``` + +**`rubric.md`**: criteria an LLM judge applies to the result, one checkable criterion per +line works best. The judge uses the `[judge]` settings. It reports how many criteria the +result meets. + +A task with neither is still refined; it just has no grade. + +## Creating and importing tasks + +```bash +harness2 task new fix-login-redirect # a project task +harness2 task new summarize-incidents --files # a files task, with files/ +harness2 task new add-pagination --domain api --check "python -m pytest -q" +harness2 task ls +``` + +To bring in many tasks at once, import a JSON Lines or CSV file. `id` and `prompt` are +required; the other fields are the `task.toml` keys above. + +```bash +harness2 task import tasks.jsonl +``` + +```json +{"id": "fix-login-redirect", "prompt": "Users who log in from /settings land on / ...", "domain": "auth", "check": "python -m pytest -q tests/test_login.py"} +``` diff --git a/harness2/__main__.py b/harness2/__main__.py new file mode 100644 index 00000000..f900011d --- /dev/null +++ b/harness2/__main__.py @@ -0,0 +1,22 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run the harness2 CLI as a module.""" + +from __future__ import annotations + +from .cli.main import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/harness2/cli/_adopt.py b/harness2/cli/_adopt.py new file mode 100644 index 00000000..b0a8bd0b --- /dev/null +++ b/harness2/cli/_adopt.py @@ -0,0 +1,153 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Open stored runs and export selected harness trees.""" + +from __future__ import annotations + +import difflib +import subprocess +from pathlib import Path + +from .. import registry +from ..core import schedule +from ..core.spec import BenchmarkAdapter +from ..core.tree import Tree, read_tree +from ..settings import ConfigError +from . import _common as C + + +def open_run(ctx, name: str | None) -> tuple[BenchmarkAdapter, Path]: + root = C.resolve_run(ctx, name) + record = C.load_run(root) + config = C.run_settings(ctx, record) + split_file = record["schedule"].get("split_file") + suite_ctx = registry.SuiteContext( + config, ctx.project, root, Path(split_file) if split_file else None + ) + return registry.load(record["suite"]).make_adapter(suite_ctx), root + + +def adopted_tree(adapter, run_dir, task: str | None, mode: str) -> tuple[str, Tree]: + record = C.load_run(run_dir) + mode = schedule.mode_token(mode) + plan = record["schedule"] + ns = schedule.ns( + adapter.spec.name, adapter.substrate.name, mode, plan["k"], seed=plan.get("seed") + ) + summary_ns = C.summary_namespace(adapter, plan["k"], seed=plan.get("seed")) + tasks = [task] if task else [pair[1] for pair in reversed(record["stream"])] + rows = { + row["task_id"]: row + for path, row in C.summaries(run_dir) + if path.parent.parent.name == summary_ns + } + for task_id in tasks: + selection = (rows.get(task_id, {}).get("selections") or {}).get(mode) or {} + if not selection.get("ok") or not selection.get("chosen_label"): + continue + if selection.get("chosen_tree_is_base"): + return task_id, base_tree(adapter) + path = ( + C.runs_root_for(adapter) + / ns + / task_id.replace("/", "__") + / "harnesses" + / selection["chosen_label"] + ) + if not path.is_dir(): + raise ConfigError(f"selected harness is missing: {path}") + return task_id, read_tree(adapter.substrate.surface(), path) + raise ConfigError(f"no successful selection for {task or 'any task'} in {C.mode_label(mode)}") + + +def base_tree(adapter) -> Tree: + return read_tree(adapter.substrate.surface(), adapter.base_harness()) + + +def harness_for(adapter, run_dir, task: str | None, mode: str) -> tuple[str, Tree]: + if task is None: + path = run_dir / "consolidated" / C.mode_label(schedule.mode_token(mode)) / "harness" + if path.is_dir(): + return "consolidated", read_tree(adapter.substrate.surface(), path) + return adopted_tree(adapter, run_dir, task, mode) + + +def project_tree(adapter, settings) -> Tree: + return adapter.substrate.import_tree(settings.path("paths.harness")) + + +def changes(before: Tree, after: Tree) -> list[tuple[str, str]]: + return [ + (rel, "A" if rel not in before else "D" if rel not in after else "M") + for rel in sorted(before.keys() | after.keys()) + if before.get(rel) != after.get(rel) + ] + + +def unified(rel: str, before: Tree, after: Tree) -> str: + lines = [] + for line in difflib.unified_diff( + before.get(rel, "").splitlines(keepends=True), + after.get(rel, "").splitlines(keepends=True), + fromfile=f"a/{rel}" if rel in before else "/dev/null", + tofile=f"b/{rel}" if rel in after else "/dev/null", + ): + lines.append(line if line.endswith("\n") else line + "\n\\ No newline at end of file\n") + return "".join(lines) + + +def dirty_paths(settings, adapter) -> list[str]: + root = settings.path("paths.harness") + probe = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--show-toplevel"], capture_output=True, text=True + ) + if probe.returncode: + return [] + result = subprocess.run( + [ + "git", + "-C", + str(root), + "status", + "--porcelain", + "-z", + "--untracked-files=all", + "--", + *adapter.substrate.NATIVE_PATHS, + ], + capture_output=True, + check=True, + ) + fields = result.stdout.decode("utf-8", errors="replace").split("\0") + paths, index = set(), 0 + while index < len(fields) and fields[index]: + entry = fields[index] + paths.add(entry[3:]) + index += 1 + if "R" in entry[:2] or "C" in entry[:2]: + paths.add(fields[index]) + index += 1 + return sorted(paths) + + +def perform(settings, adapter, tree: Tree, *, base: Tree, branch: str | None) -> dict: + root = settings.path("paths.harness") + if branch: + result = subprocess.run( + ["git", "-C", str(root), "switch", "-c", branch], capture_output=True, text=True + ) + if result.returncode: + raise ConfigError(result.stderr.strip() or f"cannot create branch {branch!r}") + return adapter.substrate.export_tree(tree, root, base=base) diff --git a/harness2/cli/_common.py b/harness2/cli/_common.py index 06d41307..c948c96b 100644 --- a/harness2/cli/_common.py +++ b/harness2/cli/_common.py @@ -12,75 +12,242 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Shared plumbing for the CLIs: adapter construction and the on-disk conventions. - -The conventions are defined here once, because run_sweep writes them and -judge/report read them back. -""" +"""CLI settings, output, and persisted run conventions.""" from __future__ import annotations -import argparse +import json +import os +import sys +from contextlib import contextmanager, redirect_stdout +from dataclasses import dataclass +from datetime import datetime, timezone +from functools import cached_property +from importlib.metadata import version from pathlib import Path +from .. import settings from ..core import library as LIB -from ..core import schedule as SCHED -from ..core.spec import BenchmarkAdapter +from ..core import schedule +from ..settings import ConfigError -BENCHES = ("jb", "wb", "lab") -WB_SUBSETS = ("office", "web", "code") -SUBSTRATES = ("oc", "cx") -MODES_DEFAULT = "parallel" -MODES_HELP = ( - "parallel | sequential (agg / seq accepted: the on-disk tokens); " - "comma list, e.g. 'parallel' or 'parallel,sequential'" -) +@dataclass +class Context: + args: object + flags: dict[str, str] + @cached_property + def project_or_none(self) -> Path | None: + return settings.find_project(Path(self.args.project or Path.cwd())) -class HelpFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter): - """Defaults in the help text, module docstring kept verbatim as the description.""" + @property + def project(self) -> Path: + if self.project_or_none is None: + raise ConfigError( + "not inside a harness2 project (no .harness2/ found); run `harness2 init`" + ) + return self.project_or_none + @cached_property + def overrides(self) -> dict: + values = {} + for raw in self.args.config: + key, sep, value = raw.partition("=") + if not sep or not key.strip(): + raise ConfigError("--config requires KEY=VALUE") + values[key.strip()] = (settings.parse_value(value), "-c") + for name, key in self.flags.items(): + value = getattr(self.args, name, None) + if value is not None: + values[key] = (value, f"flag --{name.replace('_', '-')}") + return values -def make_adapter( - bench: str, subset: str | None, substrate: str, split_file: Path | None = None -) -> BenchmarkAdapter: - """The one place a CLI turns (--bench, --subset, --substrate) into an adapter.""" - if bench not in BENCHES: - raise ValueError(f"unknown bench {bench!r}; expected one of {BENCHES}") - if substrate not in SUBSTRATES: - raise ValueError(f"unknown substrate {substrate!r}; expected one of {SUBSTRATES}") - if bench == "wb": - if subset not in WB_SUBSETS: - raise ValueError(f"--bench wb needs --subset from {WB_SUBSETS}, got {subset!r} ") - from ..benchmarks.workbuddy import WbAdapter + @cached_property + def settings(self) -> settings.Settings: + return settings.load( + self.project_or_none, profile=self.args.profile, overrides=self.overrides + ) - return WbAdapter(subset, substrate=substrate, split_file=split_file) - if subset: - raise ValueError(f"--subset applies to --bench wb only, got {subset!r} for {bench}") - if bench == "jb": - from ..benchmarks.jobbench import JobBenchAdapter + def out(self, data, text_fn) -> None: + if self.args.json: + print(json.dumps(data, ensure_ascii=False, default=str)) + else: + text = text_fn(data) + if text is not None: + print(text) - return JobBenchAdapter(substrate=substrate, split_file=split_file) - from ..benchmarks.lab import LabAdapter + def log(self, message: str) -> None: + if not self.args.quiet: + print(message, file=sys.stderr) - return LabAdapter(substrate=substrate, split_file=split_file) + def debug(self, message: str) -> None: + if self.args.verbose: + self.log(message) + @contextmanager + def progress(self): + if self.args.quiet: + with open(os.devnull, "w") as stream, redirect_stdout(stream): + yield + else: + with redirect_stdout(sys.stderr): + yield -def runs_root_for(adapter: BenchmarkAdapter) -> Path: - """Scheduler state: summaries, harnesses and selections, not the rollout store.""" - return adapter.spec.runs_root / "adapt" + +def run_path(ctx: Context, name: str) -> Path: + if not name or name in (".", "..") or Path(name).name != name: + raise ConfigError("run name must be a single directory name") + return ctx.settings.path("paths.runs") / name + + +def resolve_run(ctx: Context, name: str | None) -> Path: + ctx.project + if name is not None: + path = run_path(ctx, name) + if not (path / "run.json").is_file(): + raise ConfigError(f"run {name!r} has no run.json") + return path + root = ctx.settings.path("paths.runs") + runs = ( + [p for p in root.iterdir() if p.is_dir() and (p / "run.json").is_file()] + if root.is_dir() + else [] + ) + if not runs: + raise ConfigError(f"no runs under {root}") + return max(runs, key=lambda p: p.stat().st_mtime_ns) + + +def run_process(run_dir: Path) -> dict | None: + path = run_dir / "run.pid" + try: + record = json.loads(path.read_text(encoding="utf-8")) + if ( + not isinstance(record, dict) + or type(record.get("pid")) is not int + or record["pid"] <= 0 + or not isinstance(record.get("started"), str) + or not isinstance(record.get("argv"), list) + or any(not isinstance(arg, str) for arg in record["argv"]) + ): + raise ValueError("invalid run process record") + except FileNotFoundError: + return None + except ValueError: + path.unlink(missing_ok=True) + return None + try: + os.kill(record["pid"], 0) + except ProcessLookupError: + path.unlink(missing_ok=True) + return None + except PermissionError: + pass + return record + + +def load_run(run_dir: Path) -> dict: + try: + record = json.loads((run_dir / "run.json").read_text(encoding="utf-8")) + if not isinstance(record, dict) or not all( + key in record for key in ("name", "settings", "suite", "executor", "stream", "schedule") + ): + raise ValueError("missing run metadata") + if not isinstance(record["settings"], dict) or not isinstance(record["schedule"], dict): + raise ValueError("settings and schedule must be objects") + if not isinstance(record["stream"], list) or any( + not isinstance(pair, list) + or len(pair) != 2 + or not all(isinstance(item, str) for item in pair) + for pair in record["stream"] + ): + raise ValueError("stream must contain [domain, task] pairs") + return record + except (OSError, ValueError) as exc: + raise ConfigError(f"cannot read {run_dir / 'run.json'}: {exc}") from exc + + +def identity(config: settings.Settings) -> dict: + return { + key: value + for key, value in config.flat().items() + if key.startswith(("suite.", "executor.", "improver.", "selector.", "reflector.")) + or key in ("paths.harness", "paths.tasks", "run.seed", "run.obs_trunc") + } + + +def default_name(config: settings.Settings) -> str: + name = f"{config.get('executor.name')}-k{config.get('run.k')}" + seed = config.get("run.seed") + return name + (f"-s{seed}" if seed is not None else "") + + +def save_run(run_dir: Path, config: settings.Settings, stream: list, plan: dict) -> dict: + subset = identity(config) + path = run_dir / "run.json" + old = load_run(run_dir) if path.is_file() else None + if old: + differences = [ + f"{key}: {old['settings'].get(key)!r} -> {subset.get(key)!r}" + for key in sorted(old["settings"].keys() | subset.keys()) + if old["settings"].get(key) != subset.get(key) + or (key in old["settings"]) != (key in subset) + ] + if old["stream"] != [list(pair) for pair in stream]: + differences.append(f"stream: {old['stream']!r} -> {stream!r}") + if differences: + raise ConfigError("run identity differs:\n" + "\n".join(differences)) + record = { + "name": run_dir.name, + "created": old["created"] if old else datetime.now(timezone.utc).isoformat(), + "harness2_version": version("harness2-cli"), + "suite": config.get("suite.name"), + "executor": config.get("executor.name"), + "settings": subset, + "stream": [list(pair) for pair in stream], + "schedule": plan, + } + run_dir.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(".json.tmp") + temporary.write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8") + temporary.replace(path) + return record -def library_root_for(adapter: BenchmarkAdapter, k: int, *, seed: int | None, tag: str = "") -> Path: - """Per-(benchmark, substrate, k, seed, tag) experience-library root (the improver's R). +def run_settings(ctx: Context, record: dict) -> settings.Settings: + overrides = {key: (value, "run.json") for key, value in record["settings"].items()} + for key in ("k", "modes", "seed"): + if key in record["schedule"]: + overrides[f"run.{key}"] = (record["schedule"][key], "run.json") + overrides.update( + {key: value for key, value in ctx.overrides.items() if key.startswith("judge.")} + ) + current = settings.load(ctx.project, profile=ctx.args.profile, overrides=overrides) + data = {name: current.section(name) for name in settings.DEFAULTS} + origins = {key: current.origin(key) for key in current.flat()} + # Empty snapshot sections must not inherit fields added to the project since the run. + for name in ("suite", "executor", "improver", "selector", "reflector"): + data[name] = {} + for key, value in record["settings"].items(): + parts = key.split(".") + table = data + for part in parts[:-1]: + table = table.setdefault(part, {}) + table[parts[-1]] = value + origins[key] = "run.json" + return settings.Settings(ctx.project, data, origins) + + +def runs_root_for(adapter) -> Path: + return adapter.spec.runs_root / "adapt" + - Separate seeds and tags keep independent task streams from sharing entries. - """ - SCHED.validate_tag(tag) +def library_root_for(adapter, k: int, *, seed: int | None, tag: str = "") -> Path: + schedule.validate_tag(tag) variant = adapter.substrate.name if seed is not None: - variant += f"-s{int(seed)}" + variant += f"-s{seed}" if tag: variant += f"-{tag}" return LIB.root_for( @@ -88,57 +255,46 @@ def library_root_for(adapter: BenchmarkAdapter, k: int, *, seed: int | None, tag ) -def summary_namespace(adapter: BenchmarkAdapter, k: int, *, seed: int | None, tag: str = "") -> str: - """The namespace whose task dirs hold summary.json for BOTH modes: run_task keeps the - summary in the parallel mode's namespace (on-disk token "agg") whichever modes ran.""" - return SCHED.ns(adapter.spec.name, adapter.substrate.name, "agg", k, seed=seed, tag=tag) - - -def add_cell_args(p: argparse._ActionsContainer) -> None: - """The (benchmark, substrate) cell selectors; `p` may be a parser or an argument group.""" - p.add_argument("--bench", required=True, choices=BENCHES, help="jb | wb | lab") - p.add_argument( - "--subset", default=None, choices=WB_SUBSETS, help="WB only: office | web | code" - ) - p.add_argument( - "--substrate", - default="oc", - choices=SUBSTRATES, - help="task-executor substrate: oc (opencode) | cx (codex)", - ) +def summary_namespace(adapter, k: int, *, seed: int | None, tag: str = "") -> str: + return schedule.ns(adapter.spec.name, adapter.substrate.name, "agg", k, seed=seed, tag=tag) -def add_modes_arg(p: argparse._ActionsContainer) -> None: - p.add_argument("--mode", "--modes", dest="modes", default=MODES_DEFAULT, help=MODES_HELP) +def parse_modes(raw: str) -> tuple[str, ...]: + tokens = {schedule.mode_token(s.strip()) for s in raw.split(",") if s.strip()} + if not tokens: + raise ConfigError("--mode requires parallel or sequential") + return tuple(t for t in schedule.MODES if t in tokens) -def add_run_identity_args(p: argparse._ActionsContainer) -> None: - """The run-identity knobs that enter the namespace.""" - p.add_argument( - "--k", type=int, default=1, help="scale: R=4k rollouts, P=3k improver sessions per task" - ) - p.add_argument( - "--seed", type=int, default=None, help="stream-order seed; omit for sorted task order" - ) - add_modes_arg(p) +def mode_label(token: str) -> str: + return schedule.MODE_NAMES.get(token, token) -def parse_modes(raw: str) -> tuple[str, ...]: - """'parallel,sequential' (or the on-disk tokens 'agg,seq') -> tokens in MODES order.""" - names = [s.strip() for s in raw.split(",") if s.strip()] - if not names: - raise ValueError( - f"--modes takes a comma list of parallel | sequential (agg | seq), got {raw!r}" - ) - tokens = set() - for name in names: +def summaries(run_dir: Path) -> list[tuple[Path, dict]]: + rows = [] + for path in sorted((run_dir / "adapt").glob("*/*/summary.json")): try: - tokens.add(SCHED.mode_token(name)) - except ValueError as exc: - raise ValueError(f"--modes: {exc} (got {raw!r})") from None - return tuple(t for t in SCHED.MODES if t in tokens) + rows.append((path, json.loads(path.read_text(encoding="utf-8")))) + except (OSError, ValueError) as exc: + raise ConfigError(f"cannot read {path}: {exc}") from exc + return rows -def mode_label(token: str) -> str: - """Human name for an on-disk token ('agg' -> 'parallel'); unknown tokens pass through.""" - return SCHED.MODE_NAMES.get(token, token) +def event_totals(run_dir: Path) -> dict: + records = [] + path = run_dir / "events.jsonl" + if path.is_file(): + try: + records = [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + except (OSError, ValueError) as exc: + raise ConfigError(f"cannot read {path}: {exc}") from exc + calls = [ + r + for r in records + if r["kind"] == "rollout" or (r["kind"] == "session" and not r.get("cached")) + ] + return { + "spent_usd": round(sum(r.get("cost_usd", 0) or 0 for r in records), 6), + "calls": len(calls), + "last_activity": records[-1].get("t") if records else None, + } diff --git a/harness2/cli/_final.py b/harness2/cli/_final.py new file mode 100644 index 00000000..9a0efd93 --- /dev/null +++ b/harness2/cli/_final.py @@ -0,0 +1,136 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compose a project harness from a run's successful task selections.""" + +from __future__ import annotations + +import json +import re +import shutil +from datetime import datetime, timezone +from pathlib import Path + +from ..core import library, prompts, recursion, schedule, tree +from ..core.proposer import Proposer +from ..settings import ConfigError +from . import _adopt as A +from . import _common as C + + +def _context( + adapter, root: Path, record: dict, mode: str, dest: Path, base: tree.Tree +) -> list[str]: + surface = adapter.substrate.surface() + token = schedule.mode_token(mode) + plan = record["schedule"] + namespace = C.summary_namespace(adapter, plan["k"], seed=plan.get("seed")) + selected = { + row["task_id"] + for path, row in C.summaries(root) + if path.parent.parent.name == namespace + and (selection := (row.get("selections") or {}).get(token) or {}).get("ok") + and selection.get("chosen_label") + } + tasks, index = [], [] + tree.write_tree(surface, base, dest / "harness", stubs=True) + lib = C.library_root_for(adapter, plan["k"], seed=plan.get("seed")) / token + for domain, task in record["stream"]: + if task not in selected: + continue + _, adopted = A.adopted_tree(adapter, root, task, mode) + task_dir = dest / "tasks" / library.slug(task) + tree.write_tree(surface, adopted, task_dir / "harness", stubs=False) + (task_dir / "task.md").write_text(adapter.task_prompt(task), encoding="utf-8") + lesson = ( + lib / library.domain_dir(domain) / "harnesses" / library.slug(task) / "EXPERIENCE.md" + ) + if lesson.is_file(): + (task_dir / "EXPERIENCE.md").write_text(lesson.read_text(), encoding="utf-8") + index.append( + f"- {task} ({domain}): adopted harness " + f"{'differs from' if adopted != base else 'same as'} the start; " + f"lesson: {'yes' if lesson.is_file() else 'no'}" + ) + tasks.append(task) + if not tasks: + raise ConfigError(f"no successful selection for any task in {mode}") + (dest / "INDEX.md").write_text("\n".join(index) + "\n", encoding="utf-8") + return tasks + + +def consolidate(ctx, adapter, root: Path, mode: str) -> dict: + mode = C.mode_label(schedule.mode_token(mode)) + with ctx.progress(): + base = A.base_tree(adapter) + record = C.load_run(root) + surface = adapter.substrate.surface() + prop = Proposer(adapter.spec.channels["proposer"], audit_dir=root / "consolidated/_audit") + cost = 0.0 + # The context path participates in the proposer's cache key. + context = root / "consolidated/_audit" / f"context.{mode}" + if context.exists(): + shutil.rmtree(context) + try: + tasks = _context(adapter, root, record, mode, context, base) + mission = prompts.CONSOLIDATE_MISSION.replace("{n}", str(len(tasks))).replace( + "{surface}", prompts._st(surface).surface_md + ) + err = "" + for _ in range(2): + with ctx.progress(): + reply = prop.call( + mission + + (f"\n\n# YOUR PREVIOUS ATTEMPT WAS REJECTED\n\n{err}\n\n" if err else ""), + context, + f"consolidate.{mode}", + harvest="context/harness", + ) + cost += reply.cost_usd + if reply.workspace is None: + err = "the harness directory did not come back from the session" + continue + rejected = [] + new = tree.read_tree(surface, reply.workspace, rejected=rejected) + _, err = recursion._accept_tree(surface, adapter, new, base, "consolidate") + if rejected: + err = "files outside the editable surface: " + ", ".join(sorted(rejected)) + if not err: + break + tree.write_tree(surface, new, context / "harness", stubs=True) + else: + if reply.workspace is not None: + shutil.rmtree(reply.workspace) + raise ConfigError(err) + finally: + shutil.rmtree(context, ignore_errors=True) + notes = re.search(r"(.*?)", reply.text, re.S) + notes = notes.group(1).strip() if notes else reply.text.strip() + dest = root / "consolidated" / mode + path = tree.write_tree(surface, new, dest / "harness", stubs=False) + (dest / "notes.md").write_text(notes + "\n", encoding="utf-8") + (dest / "consolidation.json").write_text( + json.dumps( + { + "tasks": tasks, + "model": prop.spec.model, + "cost_usd": cost, + "created": datetime.now(timezone.utc).isoformat(), + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + return {"path": path, "tasks": tasks, "changes": A.changes(base, new), "notes": notes} diff --git a/harness2/cli/apply.py b/harness2/cli/apply.py new file mode 100644 index 00000000..659463e4 --- /dev/null +++ b/harness2/cli/apply.py @@ -0,0 +1,208 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Write a harness a run adopted back into the project, after showing what will change. + +Only what the run changed relative to the harness it started from is applied, so edits made +to the project since then are kept. A file the run changed that has also changed in the +project is skipped unless --force. Only the executor's harness files are touched (for +Claude Code: CLAUDE.md, .claude/skills, .claude/agents, .claude/scripts and the hooks in +.claude/settings.json), and they must have no uncommitted changes, so git can show and undo +the result. +""" + +from __future__ import annotations + +import argparse +import contextlib +import json +import sys + +from . import _adopt as A + + +def add_parser(sub) -> argparse.ArgumentParser: + p = sub.add_parser( + "apply", + help="write an adopted harness into the project", + description="Show what the harness a run adopted changes, then write those changes " + "into the project. NAME defaults to the most recent run; the task defaults to the " + "last one the run finished.", + ) + p.add_argument("name", nargs="?", help="run name (default: the most recent run)") + p.add_argument("--task", help="take the harness adopted for this task") + p.add_argument( + "--mode", + choices=("parallel", "sequential"), + default="parallel", + help="take the harness adopted in this recursion mode", + ) + p.add_argument( + "-i", "--interactive", action="store_true", help="review and accept each file separately" + ) + p.add_argument("-y", "--yes", action="store_true", help="apply without asking") + p.add_argument("--branch", help="create and switch to this git branch first") + p.add_argument("--dry-run", action="store_true", help="show the changes, write nothing") + p.add_argument( + "--force", + action="store_true", + help="also overwrite files changed in the project since the run started, and apply " + "over uncommitted changes", + ) + return p + + +def run(args, ctx) -> int: + if not ctx.args.json: + return _apply(args, ctx)[0] + if not (args.yes or args.dry_run): + message = "--json cannot ask for confirmation; add --yes or --dry-run" + print(json.dumps({"error": message, "exit_code": 1})) + return 1 + with contextlib.redirect_stdout(sys.stderr): # the review text; stdout carries the JSON + code, summary = _apply(args, ctx) + print(json.dumps(summary)) + return code + + +def _apply(args, ctx) -> tuple[int, dict]: + adapter, run_dir = A.open_run(ctx, args.name) + task, adopted = A.harness_for(adapter, run_dir, args.task, args.mode) + base = A.base_tree(adapter) + current = A.project_tree(adapter, ctx.settings) + where = ctx.settings.path("paths.harness") + + delta = A.changes(base, adopted) + if not delta: + ctx.log(f"The harness adopted for {task} is the one the run started from; nothing to do.") + return 0, {"run": run_dir.name, "harness": task, "changes": []} + moved = {rel for rel, _ in delta if current.get(rel) != base.get(rel)} + todo = [(rel, kind) for rel, kind in delta if args.force or rel not in moved] + + label = ( + f"Final harness of run {run_dir.name} ({args.mode})" + if args.task is None and task == "consolidated" + else f"Harness adopted for {task} (run {run_dir.name}, {args.mode})" + ) + print(f"{label} -> {where}\n") + for rel, kind in delta: + mark = "!" if rel in moved else kind + print(f" {mark} {rel:<48} {_delta(rel, current, adopted)}") + if moved: + print( + "\n ! also changed in the project since the run started: " + + ("will be overwritten (--force)." if args.force else "skipped; --force overwrites.") + ) + print() + + def summary(applied=(), done=None) -> dict: + return { + "run": run_dir.name, + "harness": task, + "mode": args.mode, + "changes": [ + { + "path": rel, + "kind": kind, + "changed_in_project": rel in moved, + "applied": (rel, kind) in applied, + } + for rel, kind in delta + ], + "written": (done or {}).get("written", []), + "deleted": (done or {}).get("deleted", []), + } + + if not todo: + ctx.log("Nothing left to apply.") + return 0, summary() + + dirty = A.dirty_paths(ctx.settings, adapter) + if dirty and not args.force: + message = ( + "These harness files have uncommitted changes:\n " + + "\n ".join(dirty) + + "\nCommit or stash them first so the result can be reviewed and undone with git, " + "or pass --force." + ) + print(message, file=sys.stderr) + return 1, {"error": message, "exit_code": 1} + if args.dry_run: + return 0, summary() + + if args.interactive and not sys.stdin.isatty(): + message = "--interactive needs a terminal; use --yes or --dry-run instead." + print(message, file=sys.stderr) + return 1, {"error": message, "exit_code": 1} + if args.interactive: + chosen = _review(todo, current, adopted) + elif args.yes: + chosen = todo + elif not sys.stdin.isatty(): + message = "Not a terminal: pass --yes to apply, or --dry-run to only look." + print(message, file=sys.stderr) + return 1, {"error": message, "exit_code": 1} + else: + chosen = todo if _confirm(f"Apply {len(todo)} change(s)?") else [] + if not chosen: + ctx.log("Nothing applied.") + return 0, summary() + + final = dict(current) + for rel, kind in chosen: + if kind == "D": + final.pop(rel, None) + else: + final[rel] = adopted[rel] + done = A.perform(ctx.settings, adapter, final, base=current, branch=args.branch) + print(f"Wrote {len(done.get('written', []))} file(s), deleted {len(done.get('deleted', []))}.") + if args.branch: + print(f"On branch {args.branch}.") + print("Review with `git diff`; undo with `git restore` on the files listed above.") + return 0, summary(chosen, done) + + +def _delta(rel: str, before: dict, after: dict) -> str: + body = A.unified(rel, before, after).splitlines()[2:] # skip the ---/+++ header + added = sum(line.startswith("+") for line in body) + removed = sum(line.startswith("-") for line in body) + return " ".join( + s for s in (f"+{added}" if added else "", f"-{removed}" if removed else "") if s + ) + + +def _review(changes, current, adopted): + chosen = [] + for i, (rel, kind) in enumerate(changes, 1): + print(A.unified(rel, current, adopted) or f"(no difference in {rel})") + answer = _ask(f"[{i}/{len(changes)}] Apply {rel}? [y,n,q]", ["y", "n", "q"]) + if answer == "q": + break + if answer == "y": + chosen.append((rel, kind)) + return chosen + + +def _confirm(question: str) -> bool: + return _ask(f"{question} [y/N]", ["y", "n", ""]) == "y" + + +def _ask(prompt: str, allowed: list[str]) -> str: + while True: + try: + answer = input(prompt + " ").strip().lower() + except EOFError: + return "q" if "q" in allowed else "n" + if answer in allowed: + return answer diff --git a/harness2/cli/bridge.py b/harness2/cli/bridge.py index 3e340bf8..f82222e4 100644 --- a/harness2/cli/bridge.py +++ b/harness2/cli/bridge.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Serve a Responses-API endpoint on Vertex AI for the Codex executor.""" +"""Serve a Responses-API endpoint for a configured provider/model.""" from __future__ import annotations @@ -24,19 +24,8 @@ import sys from pathlib import Path -from harness2 import config - -LITELLM_CONFIG = """model_list: - - model_name: {model} - litellm_params: - model: vertex_ai/{model} - vertex_project: os.environ/GOOGLE_CLOUD_PROJECT - vertex_location: os.environ/GOOGLE_CLOUD_LOCATION -litellm_settings: - drop_params: true -general_settings: - master_key: os.environ/HARNESS2_BRIDGE_KEY -""" +from harness2 import providers, settings +from harness2.settings import ConfigError def _free_port(host: str) -> int: @@ -46,7 +35,7 @@ def _free_port(host: str) -> int: def _endpoint(path: Path, host: str, port: int) -> dict: - """Reuse the saved port and key, so the Codex config written by setup stays valid.""" + """Reuse the saved port and key across bridge invocations.""" try: saved = json.loads(path.read_text(encoding="utf-8")) except (OSError, ValueError): @@ -61,47 +50,92 @@ def _endpoint(path: Path, host: str, port: int) -> dict: return endpoint -def main(argv=None): - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--model", default=config.SOLVER_MODEL_CX, help="Vertex model ID to serve") +def _arguments(p): + p.add_argument("--model", required=True, help="provider/model to serve") p.add_argument( "--host", default="127.0.0.1", - help="interface to bind; use 0.0.0.0 so WorkBuddy containers can reach it", + help="interface to listen on; use 0.0.0.0 when containers must reach it", ) - p.add_argument("--port", type=int, default=0, help="default: the saved port, else a free one") - args = p.parse_args(argv) + p.add_argument("--port", type=int, default=0, help="port (default: the saved one, or free)") + return p + + +def add_parser(subparsers): + return _arguments( + subparsers.add_parser( + "bridge", + help="serve a model as a Responses API endpoint for Codex", + description="Serve any configured provider/model as an OpenAI Responses API " + "endpoint through LiteLLM, so Codex can run on it. Needs the bridge extra: " + "pip install 'harness2-cli[bridge]'.", + ) + ) + +def run(args, ctx): litellm = Path(sys.executable).with_name("litellm") if not litellm.is_file(): - p.exit(1, "LiteLLM is not installed; run: uv pip install -e '.[render,bridge]'\n") - if not config.VERTEX_PROJECT: - p.exit(1, "Set GOOGLE_CLOUD_PROJECT in .env\n") + raise ConfigError("LiteLLM is not installed; run: uv pip install -e '.[bridge]'") + config = ctx.settings + provider_name, model = providers.split_model(args.model) + provider = config.provider(provider_name) + params = providers.litellm_params(provider, model) - root = config.DEPENDENCIES / "codex" + root = settings.cache_dir() / "bridge" endpoint = _endpoint(root / "bridge.json", args.host, args.port) port = endpoint["port"] litellm_config = root / "bridge.yaml" - litellm_config.write_text(LITELLM_CONFIG.format(model=args.model), encoding="utf-8") + # JSON is valid YAML and safely preserves arbitrary model and endpoint strings. + litellm_config.write_text( + json.dumps( + { + "model_list": [{"model_name": model, "litellm_params": params}], + "litellm_settings": {"drop_params": True}, + "general_settings": {"master_key": "os.environ/HARNESS2_BRIDGE_KEY"}, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) - settings = { - "HARNESS2_CODEX_BASE_URL": f"http://127.0.0.1:{port}/v1", - "HARNESS2_CODEX_API_KEY": endpoint["key"], - "HARNESS2_CODEX_MODEL": args.model, - } - if args.host != "127.0.0.1": - settings["HARNESS2_CODEX_CONTAINER_URL"] = f"http://host.docker.internal:{port}/v1" - print("Add these to .env, then run harness2-setup --substrate cx:") - print("\n".join(f"{name}={value}" for name, value in settings.items()), flush=True) - - env = { - **os.environ, - "HARNESS2_BRIDGE_KEY": endpoint["key"], - "GOOGLE_CLOUD_PROJECT": config.VERTEX_PROJECT, - "GOOGLE_CLOUD_LOCATION": config.VERTEX_REGION, - } + snippet = ( + "[providers.bridge]\n" + 'type = "openai"\n' + f'base_url = "http://127.0.0.1:{port}/v1"\n' + 'api_key_env = "HARNESS2_BRIDGE_KEY"' + ) + ctx.out( + {"provider": snippet, "environment": {"HARNESS2_BRIDGE_KEY": endpoint["key"]}}, + lambda data: ( + f"Serving {args.model} on port {port}. Add this provider to .harness2/config.toml " + f"and use the model as bridge/{model}:\n\n" + + data["provider"] + + "\n\nand put its key in .harness2/.env:\n\nHARNESS2_BRIDGE_KEY=" + + data["environment"]["HARNESS2_BRIDGE_KEY"] + + "\n" + ), + ) + sys.stdout.flush() + + env = {**os.environ, "HARNESS2_BRIDGE_KEY": endpoint["key"]} command = [str(litellm), "--config", str(litellm_config), "--host", args.host] os.execve(str(litellm), [*command, "--port", str(port)], env) + return 0 + + +def main(argv=None): + from ._common import Context + + p = _arguments(argparse.ArgumentParser()) + args = p.parse_args(argv) + args.project, args.profile, args.config = None, None, [] + args.json, args.quiet, args.verbose = False, False, False + try: + return run(args, Context(args, {})) + except ConfigError as exc: + p.exit(1, f"{exc}\n") if __name__ == "__main__": diff --git a/harness2/cli/chat.py b/harness2/cli/chat.py new file mode 100644 index 00000000..6d5f838a --- /dev/null +++ b/harness2/cli/chat.py @@ -0,0 +1,78 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Talk to your agent CLI about this project's harness2 setup: tasks, runs and results. + +Starts Claude Code or Codex interactively in the project with a guide to harness2 and the +project's current state added to its instructions. It is your own session, with your own +CLI login and settings; harness2 only adds the guide. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import sys + +from . import guide as G + +CHAT_INTRO = """\ +You are talking with a person about Harness² (the `harness2` command) in the project at +{root}. Help them write tasks, a check or a rubric, read a run's results, and decide what to +apply. The guide below is how to use harness2. + +""" + + +def add_parser(sub) -> argparse.ArgumentParser: + p = sub.add_parser( + "chat", + help="talk to your agent CLI about tasks, runs and results", + description="Start Claude Code or Codex in this project with a guide to harness2 " + "added to its instructions.", + ) + p.add_argument("--with", dest="tool", choices=("claude", "codex"), help="which CLI to start") + return p + + +def run(args, ctx) -> int: + if not sys.stdin.isatty(): + print("harness2 chat is a conversation and needs a terminal.", file=sys.stderr) + return 1 + tool = args.tool or _default_tool(ctx.settings) + binary = shutil.which(tool) if tool else None + if not binary: + print("Neither `claude` nor `codex` is on PATH.", file=sys.stderr) + return 1 + guide = ( + CHAT_INTRO.format(root=ctx.project) + + G.GUIDE + + f"\n## This project\n\n{G.state_lines(ctx)}\n" + ) + if tool == "claude": + argv = [binary, "--append-system-prompt", guide] + else: + argv = [binary, "-c", f"developer_instructions={json.dumps(guide)}"] + os.chdir(ctx.project) + os.execv(binary, argv) + return 1 # unreachable + + +def _default_tool(settings) -> str | None: + for tool in (settings.get("improver.channel"), settings.get("executor.name"), "claude"): + if tool in ("claude", "codex") and shutil.which(tool): + return tool + return "codex" if shutil.which("codex") else None diff --git a/harness2/cli/check.py b/harness2/cli/check.py deleted file mode 100644 index 638eaab5..00000000 --- a/harness2/cli/check.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Check one installed benchmark before running adaptation and judging.""" - -from __future__ import annotations - -import argparse - -from harness2.cli._common import make_adapter -from harness2.preflight import check - - -def main(argv=None): - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--bench", choices=("jb", "lab", "wb"), required=True) - p.add_argument("--subset", choices=("office", "web", "code")) - p.add_argument("--substrate", choices=("oc", "cx"), default="oc") - args = p.parse_args(argv) - try: - adapter = make_adapter(args.bench, args.subset, args.substrate) - errors = check(adapter, judge=True) - except (OSError, RuntimeError, ValueError) as exc: - errors = [str(exc)] - for error in errors: - print(f"FAIL: {error}") - if errors: - print(f"Setup incomplete: {len(errors)} issue(s). See docs/BENCHMARKS.md.") - return 1 - print( - f"Ready: {adapter.spec.name}/{args.substrate}; {len(adapter.tasks())} tasks. " - "Model access is not exercised; the first real run is the first model call." - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/harness2/cli/completion.py b/harness2/cli/completion.py new file mode 100644 index 00000000..a2a3c023 --- /dev/null +++ b/harness2/cli/completion.py @@ -0,0 +1,98 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generate shell completion from the argparse command tree.""" + +from __future__ import annotations + +import argparse +import shlex + + +def add_parser(subparsers): + p = subparsers.add_parser( + "completion", + help="print a shell completion script", + description="Print a completion script. zsh: harness2 completion zsh > " + '"${fpath[1]}/_harness2" bash: harness2 completion bash > ' + "~/.local/share/bash-completion/completions/harness2", + ) + p.add_argument("shell", choices=("bash", "zsh"), help="your shell") + return p + + +def generate(parser, shell: str) -> str: + choices, transitions = {}, [] + + def visit(parser, path=""): + words = { + option + for action in parser._actions + for option in action.option_strings + if option.startswith("--") and action.help != argparse.SUPPRESS + } + for action in parser._actions: + if isinstance(action, argparse._SubParsersAction): + words.update(action.choices) + for name, child in action.choices.items(): + next_path = f"{path} {name}".strip() + transitions.append( + f" {shlex.quote(path + ':' + name)}) command_path={shlex.quote(next_path)} ;;" + ) + visit(child, next_path) + choices[path] = " ".join(sorted(words)) + + visit(parser) + start = ["#compdef harness2"] if shell == "zsh" else [] + start += [ + "_harness2_complete() {", + " local command_path='' word candidates", + " local -a previous", + ] + if shell == "zsh": + start += [' previous=("${words[@]:1:$((CURRENT-2))}")'] + else: + start += [' previous=("${COMP_WORDS[@]:1:$((COMP_CWORD-1))}")'] + start += [ + ' for word in "${previous[@]}"; do', + ' case "$command_path:$word" in', + *transitions, + " esac", + " done", + ' case "$command_path" in', + ] + start += [ + f" {shlex.quote(path)}) candidates={shlex.quote(words)} ;;" + for path, words in choices.items() + ] + start += [" esac"] + if shell == "zsh": + start += [" compadd -- ${=candidates}", "}", "compdef _harness2_complete harness2"] + else: + start += [ + ' COMPREPLY=( $(compgen -W "$candidates" -- "${COMP_WORDS[COMP_CWORD]}") )', + "}", + "complete -F _harness2_complete harness2", + ] + return "\n".join(start) + + +def run(args, ctx) -> int: + from .main import build_parser + + ctx.out( + {"shell": args.shell, "script": generate(build_parser(), args.shell)}, + lambda data: data["script"], + ) + return 0 diff --git a/harness2/cli/config.py b/harness2/cli/config.py new file mode 100644 index 00000000..a2b164ab --- /dev/null +++ b/harness2/cli/config.py @@ -0,0 +1,91 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Inspect and edit layered settings.""" + +from __future__ import annotations + +from .. import settings + + +def add_parser(subparsers): + p = subparsers.add_parser("config", help="read and change settings") + commands = p.add_subparsers(dest="action", required=True) + show = commands.add_parser("show", help="print the resolved settings") + show.add_argument( + "--origin", action="store_true", help="also print where each value comes from" + ) + get = commands.add_parser("get", help="print one setting") + get.add_argument("key", help="dotted key, e.g. executor.model") + set_parser = commands.add_parser("set", help="change a setting in the project config file") + set_parser.add_argument("key", help="dotted key, e.g. run.workers") + set_parser.add_argument("value", help="new value, read as a TOML literal when it parses") + set_parser.add_argument( + "--global", + dest="global_", + action="store_true", + help="change ~/.config/harness2/config.toml instead", + ) + commands.add_parser("path", help="print where the config files are") + return p + + +def run(args, ctx) -> int: + config = ctx.settings + if args.action == "show": + values = config.flat() + if args.origin: + values = { + key: {"value": value, "origin": config.origin(key)} for key, value in values.items() + } + ctx.out(values, _show) + elif args.action == "get": + value = config.get(args.key) + ctx.out(value, str) + elif args.action == "set": + path = settings.global_path() if args.global_ else settings.project_path(ctx.project) + value = settings.parse_value(args.value) + settings.load( + ctx.project_or_none, + profile=ctx.args.profile, + overrides={**ctx.overrides, args.key: (value, "config set")}, + ) + settings.set_value(path, args.key, value) + ctx.out( + {"path": path, "key": args.key, "value": value}, + lambda data: f"Set {data['key']} = {settings._literal(data['value'])} in {path}", + ) + else: + ctx.out( + { + "global": settings.global_path(), + "project": settings.project_path(ctx.project_or_none) + if ctx.project_or_none + else None, + }, + lambda data: "\n".join(f"{key}: {value}" for key, value in data.items()), + ) + return 0 + + +def _show(values: dict) -> str: + """`key = value` lines in TOML syntax; with --origin, where each value comes from.""" + lines = [] + for key, entry in sorted(values.items()): + value, origin = ( + (entry["value"], entry["origin"]) if isinstance(entry, dict) else (entry, "") + ) + literal = "(unset)" if value is None else settings._literal(value) + lines.append(f"{key} = {literal}" + (f" # {origin}" if origin else "")) + return "\n".join(lines) diff --git a/harness2/cli/demo.py b/harness2/cli/demo.py index 2d957dba..d2b0b6ef 100644 --- a/harness2/cli/demo.py +++ b/harness2/cli/demo.py @@ -16,7 +16,6 @@ from __future__ import annotations -import argparse import json from pathlib import Path @@ -24,35 +23,64 @@ from ..demo import DemoAdapter, DemoProposer -def main(argv=None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--output", type=Path, default=Path("runs/demo")) - parser.add_argument("--k", type=int, default=2) - parser.add_argument("--mode", choices=("parallel", "sequential", "both"), default="both") - args = parser.parse_args(argv) +def add_parser(subparsers): + parser = subparsers.add_parser( + "demo", + help="run the method locally, with scripted editors and no model calls", + description="Run the real recursion and library workflow on two toy tasks, with " + "deterministic editors instead of models. Takes a few seconds and costs nothing.", + ) + parser.add_argument( + "--output", type=Path, default=Path("runs/demo"), help="where to write the demo run" + ) + parser.add_argument("--k", type=int, default=2, help="recursion budget per task") + parser.add_argument( + "--mode", + choices=("parallel", "sequential", "both"), + default="both", + help="recursion mode", + ) + return parser + + +def run(args, ctx) -> int: if args.k < 1: - parser.error("--k must be positive") + raise ValueError("--k must be positive") root = args.output.expanduser().resolve() adapter = DemoAdapter(root) proposer = DemoProposer(adapter.spec.channels["proposer"]) modes = ("agg", "seq") if args.mode == "both" else (mode_token(args.mode),) - result = run_domain( - "accounting", - adapter.tasks(), - args.k, - adapter=adapter, - prop=proposer, - runs_root=adapter.spec.runs_root, - library_root=root / "library" / f"k{args.k}", - modes=modes, - ) + with ctx.progress(): + result = run_domain( + "accounting", + adapter.tasks(), + args.k, + adapter=adapter, + prop=proposer, + runs_root=adapter.spec.runs_root, + library_root=root / "library" / f"k{args.k}", + modes=modes, + ) result["execution_count"] = adapter.execution_count result["editor_calls"] = len(proposer._calls) report = root / "demo.json" report.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") - print(f"Local demo: {result['done']} tasks completed; {result['failed']} failed. {report}") + ctx.out( + result, + lambda data: ( + f"Local demo: {data['done']} tasks completed; {data['failed']} failed. {report}" + ), + ) return int(result["failed"] > 0) +def main(argv=None) -> int: + import sys + + from .main import main as cli_main + + return cli_main(["demo", *(sys.argv[1:] if argv is None else argv)]) + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/harness2/cli/diff.py b/harness2/cli/diff.py new file mode 100644 index 00000000..4a2e8fcc --- /dev/null +++ b/harness2/cli/diff.py @@ -0,0 +1,62 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compare the stored base harness with a selection.""" + +from __future__ import annotations + +from ..core.tree import diff_summary +from . import _adopt + + +def add_parser(subparsers): + p = subparsers.add_parser( + "diff", + help="show what an adopted harness changes", + description="Compare the harness a run adopted for a task with the harness the run " + "started from. NAME defaults to the most recent run, the task to the last one the run " + "finished.", + ) + p.add_argument("name", nargs="?", help="run name (default: the most recent run)") + p.add_argument("--task", help="show the harness adopted for this task") + p.add_argument( + "--mode", + choices=("parallel", "sequential"), + default="parallel", + help="take the harness adopted in this recursion mode", + ) + p.add_argument("--stat", action="store_true", help="list the changed files only") + return p + + +def run(args, ctx) -> int: + with ctx.progress(): + adapter, root = _adopt.open_run(ctx, args.name) + task, after = _adopt.harness_for(adapter, root, args.task, args.mode) + before = _adopt.base_tree(adapter) + changes = _adopt.changes(before, after) + data = { + "label": task, + "changes": changes, + "diffs": {rel: _adopt.unified(rel, before, after) for rel, _ in changes}, + } + ctx.out( + data, + lambda row: ( + str(diff_summary(adapter.substrate.surface(), before, after)) + if args.stat + else "".join(row["diffs"].values()).rstrip("\n") + ), + ) + return 0 diff --git a/harness2/cli/doctor.py b/harness2/cli/doctor.py new file mode 100644 index 00000000..cae3c435 --- /dev/null +++ b/harness2/cli/doctor.py @@ -0,0 +1,170 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Check suite configuration and model clients.""" + +from __future__ import annotations + +import subprocess + +from .. import providers, registry +from ..core.proposer import Proposer +from ..core.spec import ChannelSpec +from ..substrate._native import cli_argv +from . import _common as C + + +def add_parser(subparsers): + p = subparsers.add_parser( + "doctor", + help="check that a run has what it needs", + description="Check the agent CLIs, provider keys, settings and tasks without calling a " + "model. --live also sends one tiny request per configured model.", + ) + p.add_argument("--live", action="store_true", help="also call each model once (costs a little)") + return p + + +def run(args, ctx) -> int: + config = ctx.settings + rows = [] + + def check(name, operation): + try: + detail = operation() + rows.append({"check": name, "ok": True, "detail": detail or "ok"}) + except (OSError, RuntimeError, ValueError, subprocess.SubprocessError) as exc: + rows.append({"check": name, "ok": False, "detail": str(exc)}) + + def suite_check(): + suite = registry.load(config.get("suite.name")) + suite_ctx = registry.SuiteContext( + config, ctx.project, C.run_path(ctx, C.default_name(config)) + ) + with ctx.progress(): + problems = suite.check(suite_ctx) if hasattr(suite, "check") else [] + if problems: + raise RuntimeError("; ".join(problems)) + + check(f"suite {config.get('suite.name')}", suite_check) + names = ("executor", "improver", "selector", "reflector", "judge") + roles = [config.role(name) for name in names[1:]] + roles.insert(0, {**config.section("executor"), "channel": config.get("executor.name")}) + binaries, models, referenced = set(), {}, set() + for role_name, role in zip(names, roles): + channel = role["channel"] + if channel == "human": + continue + if channel != "command": + binaries.add((role["bin"], channel)) + model = role["model"] + if model: + + def resolve(model=model, channel=channel, role=role): + provider_name, bare = providers.split_model(model) + provider = config.provider(provider_name) + referenced.add(provider_name) + # each raises ConfigError when this CLI cannot use this kind of provider + if channel == "claude": + providers.claude_env(provider) + elif channel == "codex": + providers.codex_args(provider, bare, role["effort"]) + elif channel == "opencode": + providers.opencode_config(provider, bare) + if channel != "command": + models.setdefault( + (channel, model), + ChannelSpec( + channel, + model, + role["effort"], + role.get("steps", 120), + role["timeout"], + role["bin"], + provider, + ), + ) + + return f"{channel} · {model}" + + check(role_name, resolve) + elif channel != "command": + rows.append( + { + "check": role_name, + "ok": False, + "detail": f"set {role_name}.model (provider/model)", + } + ) + for binary, channel in sorted(binaries): + + def binary_check(binary=binary, channel=channel): + argv = cli_argv(binary, channel) + result = subprocess.run( + [*argv, "--version"], check=True, capture_output=True, text=True, timeout=10 + ) + return (result.stdout or result.stderr).strip() + + check(f"{binary or channel} CLI", binary_check) + for name in sorted(referenced): + provider = config.provider(name) + problems = providers.check(provider) + detail = f"key in {provider.api_key_env}" if provider.api_key_env else "ok" + rows.append( + { + "check": f"provider {name}", + "ok": not problems, + "detail": "; ".join(problems) or detail, + } + ) + if provider.type == "chatgpt-login": + home = providers.codex_login_home(provider) + files = [ + str(path) + for name in ("AGENTS.md", "hooks.json") + if (path := home / name).is_file() and path.stat().st_size + ] + if files: + rows.append( + { + "check": "Codex home", + "ok": True, + "detail": " and ".join(files) + " are part of every Codex run", + } + ) + if args.live: + for (channel, model), spec in sorted(models.items()): + + def live_check(spec=spec): + with ctx.progress(): + reply = Proposer(spec).call('Reply "ok".', None, "doctor.live") + if not reply.ok or reply.text.strip().lower() != "ok": + raise RuntimeError(reply.error or "expected reply ok") + return reply.text + + check(f"live {channel} · {model}", live_check) + ctx.out(rows, _render) + return int(any(not row["ok"] for row in rows)) + + +def _render(rows: list[dict]) -> str: + width = max(len(row["check"]) for row in rows) + lines = [ + f"{'ok' if row['ok'] else 'FAIL':<4} {row['check']:<{width}} {row['detail']}" + for row in rows + ] + failed = sum(not row["ok"] for row in rows) + lines.append("") + lines.append("All checks passed." if not failed else f"{failed} problem(s) to fix.") + return "\n".join(lines) diff --git a/harness2/cli/eval.py b/harness2/cli/eval.py new file mode 100644 index 00000000..eac094d6 --- /dev/null +++ b/harness2/cli/eval.py @@ -0,0 +1,194 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compare consolidated and starting harnesses on a run's tasks.""" + +from __future__ import annotations + +import json +from concurrent.futures import ThreadPoolExecutor + +from ..core import events +from ..settings import ConfigError +from . import _adopt as A +from . import _common as C +from .run import _csv, _stopped + + +def add_parser(subparsers): + p = subparsers.add_parser( + "eval", + help="compare the consolidated harness with the starting one", + description="Run the agent with the starting harness and with the consolidated one on " + "the run's tasks, grade both with each task's check or rubric, and compare. Costs one " + "agent run per task, harness and repeat.", + ) + p.add_argument("name", nargs="?", help="run name (default: the most recent run)") + p.add_argument( + "--mode", + choices=("parallel", "sequential"), + default="parallel", + help="compare the harness consolidated in this recursion mode", + ) + p.add_argument( + "--split", choices=("all", "train", "test"), default="all", help="compare one split only" + ) + p.add_argument("--only-tasks", default="", metavar="ID,...", help="compare only these tasks") + p.add_argument("--repeat", type=int, default=1, metavar="N", help="runs per task and harness") + return p + + +def _rollout(adapter, task, harness, run_id): + events.check() + run_dir = adapter.execute(task, harness, run_id) + cost = getattr(adapter, "rollout_cost", None) + events.emit( + "rollout", + task=task, + run_id=run_id, + cost_usd=float(cost(run_dir)) if callable(cost) else 0.0, + ) + try: + grade = adapter.score(run_dir, task) + except events.Stopped: + raise + except Exception: + return None + if not grade: + return None + passed, total = grade.get("n_passed"), grade.get("n_criteria") + if type(passed) is not int or type(total) is not int or total <= 0 or not 0 <= passed <= total: + return None + return {"n_passed": passed, "n_criteria": total} + + +def _mean(values): + graded = [value for value in values if value is not None] + return sum(graded) / len(graded) if graded else None + + +def _text(data): + def percent(value): + return "-" if value is None else f"{value * 100:g}%" + + start, consolidated = data["mean"]["start"], data["mean"]["consolidated"] + delta = ( + f"{(consolidated - start) * 100:+g}" + if start is not None and consolidated is not None + else "-" + ) + rows = [("TASK", "START", "FINAL")] + rows += [ + (task, percent(row["start"]["mean"]), percent(row["consolidated"]["mean"])) + for task, row in data["tasks"].items() + ] + rows.append(("mean", percent(start), percent(consolidated))) + width = max(len(r[0]) for r in rows) + table = [f" {a:<{width}} {b:>5} {c:>5}" for a, b, c in rows] + table[-1] += f" ({delta} points)" + lines = [ + f"Comparing the final harness with the starting one on {len(data['tasks'])} " + f"task(s), {data['repeat']} run(s) each.", + "", + *table, + "", + ] + if data["skipped"]: + lines.append( + f"Skipped {len(data['skipped'])} task(s) with no check or rubric: " + + ", ".join(data["skipped"]) + ) + lines.append(f"Saved to {data['path']}") + return "\n".join(lines) + + +def run(args, ctx) -> int: + if args.repeat < 1: + raise ConfigError("--repeat must be a positive integer") + with ctx.progress(): + adapter, root = A.open_run(ctx, args.name) + harness = root / "consolidated" / args.mode / "harness" + if not harness.is_dir(): + raise ConfigError("this run has no final harness yet; run `harness2 run` to finish it") + record = C.load_run(root) + allowed = set(adapter.tasks(args.split)) + wanted = set(_csv(args.only_tasks)) + tasks = [ + task for _, task in record["stream"] if task in allowed and (not wanted or task in wanted) + ] + missing = wanted - set(tasks) + if missing: + raise ConfigError( + f"--only-tasks names unknown task id(s) {sorted(missing)} for split {args.split!r}" + ) + has_judge = getattr(adapter, "has_judge", lambda task: True) + skipped = [task for task in tasks if not has_judge(task)] + tasks = [task for task in tasks if task not in skipped] + config = ctx.settings + events.start( + root / "events.jsonl", + budget_usd=config.get("run.budget_usd"), + max_calls=config.get("run.max_calls"), + ) + rows = {task: {name: {"runs": []} for name in ("start", "consolidated")} for task in tasks} + jobs = [ + (task, name, directory, f"eval/{name}/{task}/{i}") + for task in tasks + for i in range(args.repeat) + for name, directory in (("start", None), ("consolidated", harness)) + ] + try: + with ctx.progress(), ThreadPoolExecutor(max_workers=config.get("run.phase_width")) as pool: + futures = [ + pool.submit(_rollout, adapter, task, directory, run_id) + for task, _, directory, run_id in jobs + ] + for (task, name, _, _), future in zip(jobs, futures): + rows[task][name]["runs"].append(future.result()) + except events.Stopped as exc: + reason = str(exc) + ctx.out(events.totals(), lambda totals: _stopped(root.name, reason, totals)) + return 4 + for row in rows.values(): + for result in row.values(): + result["mean"] = _mean( + [ + grade["n_passed"] / grade["n_criteria"] if grade else None + for grade in result["runs"] + ] + ) + path = root / "eval" / args.mode / "results.json" + data = { + "tasks": rows, + "mean": { + name: _mean([row[name]["mean"] for row in rows.values()]) + for name in ("start", "consolidated") + }, + "repeat": args.repeat, + "skipped": skipped, + "path": str(path), + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") + ctx.out(data, _text) + return ( + 3 + if any( + grade is None + for row in rows.values() + for result in row.values() + for grade in result["runs"] + ) + else 0 + ) diff --git a/harness2/cli/guide.py b/harness2/cli/guide.py new file mode 100644 index 00000000..24b09160 --- /dev/null +++ b/harness2/cli/guide.py @@ -0,0 +1,149 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The guide for AI agents that drive harness2 on a person's behalf. + +`harness2 guide` prints it with the current project's state; `--install claude` installs it +as a user-level Claude Code skill, which harness2's own runs never load. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +GUIDE = """\ +# harness2: guide for agents + +harness2 improves a coding agent's harness -- the files the agent reads on every run: +CLAUDE.md or AGENTS.md, skills, subagents, helper scripts and hooks. It runs the agent on the +project's own tasks, proposes harness edits from what those runs show, probes them on the +same tasks, and ends with one final harness for the project. A run never changes the +project's files; `harness2 apply` writes the final harness back once the person agrees. + +## Rules + +- Runs spend money or subscription quota. Before `harness2 run` or `harness2 eval`, show the + person the plan from `harness2 run --dry-run` and get their go-ahead. Bound every run with + `--max-calls N` (and `--budget-usd X` when an API key pays for it). +- Read results with `--json`. Errors then arrive on stdout as + {"error": "...", "exit_code": n}. +- Nothing waits for keyboard input when there is no terminal: answer each prompt with its + flag (`--yes`, `--auth`, `--executor`, ...) instead. +- Never edit anything under .harness2/runs/. Never edit the project's harness files to apply + a result: show `harness2 diff`, then run `harness2 apply --yes` once the person agrees. + +## Workflow + +1. `harness2 init --yes [--executor claude|codex] [--auth login|api-key]` creates + .harness2/. `--auth login` uses a Claude or ChatGPT subscription: Claude Code's own login + as it is, or for Codex a separate login the person makes once with `harness2 login codex` + (it opens a browser; ask them to run it). `harness2 login --json` shows which are ready. Change any choice afterwards with `harness2 config set KEY VALUE`, for + example `executor.model`. +2. Tasks. `harness2 task new [--files] [--check ""]`, then write the request in + .harness2/tasks//task.md the way the person would really ask for it. Good tasks are + real, varied work the person wants done better. A `check` command (exit status 0 means + pass) makes results measurable; grading files in check/ and rubric.md are never shown to + the agent. More options in task.toml: domain, split ("train"/"test"), workspace + ("project": a clean checkout at `ref`; "files": a scratch directory seeded from files/), + deliverable ("diff", "files", "message"), timeout. List them with + `harness2 task ls --json`. +3. `harness2 doctor --json`: every row must be ok before a run. +4. `harness2 run --dry-run` shows the tasks and the number of model calls. After the + person's go-ahead: `harness2 run --detach --max-calls N`. It returns at once with the run + name; runs take minutes to hours. +5. Follow it with `harness2 status --json`: "active", each task's "state", + "spent_usd", "calls". `harness2 stop ` stops it; running the same `harness2 run` + command again resumes and reuses every finished step. +6. When it has finished, `harness2 diff ` shows the final harness. To measure it + first, `harness2 eval --json` compares it with the starting harness on the tasks + that have a check or rubric. +7. With the person's agreement: `harness2 apply --yes`. + +## Where things are + +- .harness2/config.toml: settings; `harness2 config show --origin --json` resolves them. +- .harness2/runs//: run.json (settings, task list), events.jsonl (every agent run and + model session, with cost), consolidated// (the final harness and the improver's + notes), adapt/ (per-task candidates, selections, lessons), eval/ (eval results). + +## Exit codes + +0 ok, 1 error, 2 bad usage, 3 finished with failed tasks, 4 stopped at a limit (--max-calls, +--budget-usd, or the provider's usage limit), 130 interrupted. After 4 or 130, the same +command resumes. +""" + +SKILL_HEAD = """\ +--- +name: harness2 +description: Use harness2 when the person wants their coding agent's setup (CLAUDE.md, AGENTS.md, skills, subagents, hooks) improved on their own tasks, or asks to set up, run, follow, evaluate or apply a harness2 run. +--- + +""" + + +def add_parser(sub) -> argparse.ArgumentParser: + p = sub.add_parser( + "guide", + help="print the guide for AI agents that use harness2", + description="Print how an AI agent should drive harness2, with this project's current " + "state. Point your agent at it, or install it as a Claude Code skill.", + ) + p.add_argument( + "--install", + choices=("claude",), + help="install the guide as a user-level Claude Code skill (~/.claude/skills/harness2)", + ) + return p + + +def run(args, ctx) -> int: + if args.install: + path = Path.home() / ".claude/skills/harness2/SKILL.md" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(SKILL_HEAD + GUIDE, encoding="utf-8") + ctx.out( + {"installed": str(path)}, + lambda data: ( + f"Installed {data['installed']}. Claude Code loads it in new " + "sessions; harness2's own runs never do." + ), + ) + return 0 + state = state_lines(ctx) if ctx.project_or_none else "" + text = GUIDE + (f"\n## This project\n\n{state}\n" if state else "") + ctx.out({"guide": GUIDE, "project": state or None}, lambda data: text.rstrip()) + return 0 + + +def state_lines(ctx) -> str: + """The project's settings, tasks and runs, in a few lines an agent can act on.""" + s = ctx.settings + lines = [ + f"- root: {ctx.project}", + f"- executor: {s.get('executor.name')} on {s.get('executor.model') or '(model not set)'}", + f"- improver: {s.get('improver.channel')} on {s.get('improver.model') or '(not set)'}", + f"- harness directory: {s.get('paths.harness')}", + ] + tasks = sorted(p.parent.name for p in s.path("paths.tasks").glob("*/task.md")) + shown = ", ".join(tasks[:30]) + (" ..." if len(tasks) > 30 else "") + lines.append(f"- tasks ({len(tasks)}): {shown or 'none yet'}") + runs = sorted( + s.path("paths.runs").glob("*/run.json"), key=lambda p: p.stat().st_mtime, reverse=True + ) + lines.append( + "- runs, newest first: " + (", ".join(p.parent.name for p in runs[:10]) or "none yet") + ) + return "\n".join(lines) diff --git a/harness2/cli/human.py b/harness2/cli/human.py new file mode 100644 index 00000000..87fce0b9 --- /dev/null +++ b/harness2/cli/human.py @@ -0,0 +1,87 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""`--selector human`: a person picks the final output in the terminal. + +Only the blind final selection is handed to a person. The candidates are the same anonymised +files an LLM selector would read; the reply uses the selector's format so the method records +the choice exactly as it records a model's. +""" + +from __future__ import annotations + +import pydoc +import sys +import threading +from pathlib import Path + +from ..core.proposer import Proposer, ProposerError, Reply + +_TERMINAL = threading.Lock() + + +class HumanSelector(Proposer): + """A Proposer whose select sessions are answered by the person at the terminal. + + Domains run in parallel threads; the lock keeps two selections from sharing the prompt. + """ + + def call(self, mission, context_dir, label, **kwargs) -> Reply: + if not label.startswith("select."): + raise ProposerError(f"[{label}] the human selector only answers selections") + if not sys.stdin.isatty(): + raise ProposerError(f"[{label}] --selector human needs an interactive terminal") + root = Path(context_dir) + files = sorted((root / "candidates").glob("*.md")) + ids = [p.stem for p in files] + with _TERMINAL: + print(_header(root, label, len(ids)), file=sys.stderr) + for p in files: + print(f" {p.stem} {_gist(p)}", file=sys.stderr) + print(f"\nRead them in full: {root / 'candidates'}", file=sys.stderr) + choice = _ask(ids, files) + text = f"Chosen by a person.\n```choice\n{choice}\n```" + reply = Reply(label=label, text=text, ok=True, attempts=1, channel="human") + self._calls.append(reply) + return reply + + +def _header(root: Path, label: str, n: int) -> str: + task = (root / "task.md").read_text(encoding="utf-8", errors="replace").strip() + first = task.splitlines()[0][:72] if task else "(no task text)" + mode = label.removeprefix("select.") + return f"\nPick the final output — {mode}, {n} candidates, shown anonymously.\nTask: {first}\n" + + +def _gist(path: Path) -> str: + """One line per candidate: its size and, for a patch, how many files it touches.""" + body = path.read_text(encoding="utf-8", errors="replace") + files = body.count("\ndiff --git ") + body.startswith("diff --git ") + size = f"{body.count(chr(10)) + 1:,} lines" + return f"{files} file(s) changed · {size}" if files else size + + +def _ask(ids: list[str], files: list[Path]) -> str: + choices = "/".join(ids) + while True: + try: + raw = input(f"Choose {choices}, or v to read one here: ").strip() + except EOFError: + raise ProposerError("selection cancelled") from None + if raw in ids: + return raw + if raw[:1] == "v" and raw[1:] in ids: + pydoc.pager(files[ids.index(raw[1:])].read_text(encoding="utf-8", errors="replace")) + continue + print(f"Type one of {choices}, or v followed by an id.", file=sys.stderr) diff --git a/harness2/cli/init.py b/harness2/cli/init.py new file mode 100644 index 00000000..b39aeebc --- /dev/null +++ b/harness2/cli/init.py @@ -0,0 +1,309 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Set up harness2 in a project: pick the agent, its models and where the harness lives.""" + +from __future__ import annotations + +import argparse +import contextlib +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +from .. import settings as S +from ..core import tree as HT + +# executor/channel -> (model that runs tasks, model that edits the harness) +MODELS = { + "claude": ("claude-sonnet-5-5", "claude-opus-5-5"), + "codex": ("gpt-6.1-sol", "gpt-6.1-sol"), +} +# executor/channel -> (subscription provider, API-key provider) +PROVIDERS = {"claude": ("claude-login", "anthropic"), "codex": ("chatgpt-login", "openai")} +AUTH = { + "claude": ("your Claude subscription, through Claude Code's own login", "ANTHROPIC_API_KEY"), + "codex": ("your ChatGPT subscription, through a Codex login for harness2", "OPENAI_API_KEY"), +} + +CONFIG = """\ +# Harness² project settings. See docs/configuration.md for every key; +# `harness2 config show --origin` prints the resolved values and where each comes from. + +[executor] +# The agent CLI that runs your tasks, and the model it runs on (provider/model). +name = {name} +model = {model} +{command_lines} +[improver] +# The agent that studies the runs and edits the harness. +channel = {channel} +model = {improver} + +# The selector picks the final harness when several candidates compete, and the judge +# grades tasks that have a rubric.md. Both use the improver's settings unless set here. +# [selector] +# model = {selector_example} + +[paths] +# Where the harness lives, relative to the project root. +harness = {harness} +""" + +COMMAND_LINES = """\ +# The command that runs one task. {{workspace}}, {{prompt_file}} and {{output}} are filled in. +command = {command} +# The files the improver may edit, as glob patterns relative to paths.harness. +editable = {editable} +""" + + +def add_parser(sub) -> argparse.ArgumentParser: + p = sub.add_parser( + "init", + help="set up harness2 in this project", + description="Create .harness2/ with a config file, a tasks directory and a .gitignore. " + "Asks a few questions; --yes takes the defaults.", + ) + p.add_argument("--yes", "-y", action="store_true", help="take every default, ask nothing") + p.add_argument("--force", action="store_true", help="overwrite an existing config file") + p.add_argument("--executor", choices=("claude", "codex", "command"), help="skip that question") + p.add_argument( + "--auth", + choices=("login", "api-key"), + help="use the agent CLI's own login (subscription) or a provider API key", + ) + return p + + +def run(args, ctx) -> int: + if not getattr(args, "json", False): + return _run(args)[0] + with contextlib.redirect_stdout(sys.stderr): # the wizard's text; stdout carries the JSON + code, summary = _run(args) + print(json.dumps(summary)) + return code + + +def _run(args) -> tuple[int, dict]: + root = Path(args.project or Path.cwd()).resolve() + config = S.project_path(root) + if config.exists() and not args.force: + message = f"{config} already exists; pass --force to overwrite it." + print(message, file=sys.stderr) + return 1, {"error": message, "exit_code": 1} + ask = _asker(interactive=not args.yes and sys.stdin.isatty()) + print(f"Setting up Harness² in {root}") + + interactive = not args.yes and sys.stdin.isatty() + found = {name: _version(name) for name in ("claude", "codex")} + default = next((name for name in ("claude", "codex") if found[name]), None) + executor = args.executor or (_pick_executor(ask, found, default) if interactive else default) + if executor is None: + message = ( + "Neither Claude Code (`claude`) nor Codex (`codex`) is on PATH. Install one, or " + "rerun with --executor command to drive your own agent." + ) + print(message, file=sys.stderr) + return 1, {"error": message, "exit_code": 1} + + command_lines = "" + editable: list[str] = [] + if executor == "command": + command = ask("Command that runs one task", "") + editable = [g.strip() for g in ask("Editable files, comma-separated globs", "").split(",")] + editable = [g for g in editable if g] + command_lines = COMMAND_LINES.format(command=_q(command), editable=_q(editable)) + channel = "claude" if found["claude"] else "codex" + else: + channel = executor + auth = args.auth or (_pick_auth(ask, channel) if interactive else "login") + provider = PROVIDERS[channel][auth != "login"] + model, improver = (f"{provider}/{m}" for m in MODELS[channel]) + if executor == "command": + model = "" + if executor != "command": + model = ask("Model that runs the tasks (provider/model)", model) + improver = ask("Model that edits the harness (provider/model)", improver) + harness = ask("Where the harness lives, relative to the project root", ".") + + config.parent.mkdir(parents=True, exist_ok=True) + config.write_text( + CONFIG.format( + name=_q(executor), + model=_q(model), + command_lines=command_lines, + channel=_q(channel), + improver=_q(improver), + harness=_q(harness), + selector_example=_q(improver), + ), + encoding="utf-8", + ) + ignore = config.parent / ".gitignore" + if not ignore.exists(): + ignore.write_text("runs/\n.env\n", encoding="utf-8") + (config.parent / "tasks").mkdir(exist_ok=True) + + print(_harness_summary(root / harness, executor, editable)) + print(_key_summary(root, (model, improver))) + print(f"\nWrote {config.relative_to(root)}") + if executor == "command" and not (command_lines and editable): + print("Fill in executor.command and executor.editable before the first run.") + print( + "\nNext:\n" + " harness2 task new describe a task the agent should get better at\n" + " harness2 doctor check the agent CLIs, keys and tasks\n" + " harness2 run --dry-run preview what a run will do and how many calls it makes" + ) + return 0, { + "config": str(config), + "executor": executor, + "model": model, + "improver": {"channel": channel, "model": improver}, + "auth": auth, + "harness": harness, + } + + +def _asker(*, interactive: bool): + def ask(question: str, default: str) -> str: + if not interactive: + return default + shown = f" [{default}]" if default else "" + try: + answer = input(f"{question}{shown}: ").strip() + except EOFError: + answer = "" + return answer or default + + return ask + + +def _version(binary: str) -> str: + path = shutil.which(binary) + if not path: + return "" + try: + out = subprocess.run([path, "--version"], capture_output=True, text=True, timeout=15) + except (OSError, subprocess.TimeoutExpired): + return "installed" + return (out.stdout or out.stderr).strip().splitlines()[0] if out.returncode == 0 else "" + + +def _pick_executor(ask, found: dict[str, str], default: str | None) -> str | None: + options = ["claude", "codex", "command"] + if default is None: + return None + print("Which agent should run your tasks?") + for i, name in enumerate(options, 1): + detail = found.get(name) or ("not found" if name != "command" else "your own command") + print(f" {i}) {name:<8} {detail}") + while True: + answer = ask("Choose", str(options.index(default) + 1)) + if answer in options: + return answer + if answer.isdigit() and 1 <= int(answer) <= len(options): + return options[int(answer) - 1] + print(f"Type a number from 1 to {len(options)}.") + + +def _harness_summary(where: Path, executor: str, editable: list[str]) -> str: + """One line on what the harness holds today, so a wrong path shows up immediately.""" + if executor == "command": + found = sorted({p for g in editable for p in where.glob(g) if p.is_file()}) + return f"Harness: {len(found)} editable file(s) under {where}." + from ..substrate.claude import ClaudeSubstrate + from ..substrate.codex import CodexSubstrate + + sub = ClaudeSubstrate() if executor == "claude" else CodexSubstrate() + tree = sub.import_tree(where) + surface = sub.surface() + always = surface.always_on[0] + parts = [always if tree.get(always, "").strip() else f"no {always} yet"] + if skills := HT.skill_names(surface, tree): + parts.append(f"{len(skills)} skill(s)") + if agents := HT.subagent_names(surface, tree): + parts.append(f"{len(agents)} subagent(s)") + if surface.hooks_file in tree: + parts.append("hooks") + return f"Harness found in {where}: {', '.join(parts)}." + + +def _pick_auth(ask, channel: str) -> str: + login, key = AUTH[channel] + print(f"\nHow should {channel} reach its models?") + print(f" 1) {login}") + print(f" 2) an API key in {key}") + while True: + answer = ask("Choose", "1") + if answer in ("1", "2"): + return "login" if answer == "1" else "api-key" + print("Type 1 or 2.") + + +def _key_summary(root: Path, models: tuple[str, ...]) -> str: + """Whether each chosen model can be reached: the CLI login, or the API key.""" + from .. import providers + + S.load_dotenv(root / ".harness2/.env") + loaded = S.load(None) + lines = [] + for name in dict.fromkeys(providers.split_model(m)[0] for m in models if "/" in m): + if name == "chatgpt-login": + home = providers.codex_login_home(loaded.provider(name)) + ok = providers.codex_logged_in(home) + lines.append( + "Codex is signed in for harness2." + if ok + else "Codex is not signed in for harness2 yet: run `harness2 login codex` once " + "(a separate login, so your own ~/.codex settings and hooks stay out of runs)." + ) + continue + if name in LOGIN_CHECKS: + argv, cli, fix = LOGIN_CHECKS[name] + ok = shutil.which(argv[0]) and _quiet(argv) + lines.append(f"{cli} is logged in." if ok else f"{cli} is not logged in: run `{fix}`.") + continue + try: + env = loaded.provider(name).api_key_env + except S.ConfigError: + lines.append(f"Provider {name}: not configured; add [providers.{name}] to the config.") + continue + if env and not os.environ.get(env): + lines.append( + f"{env} is not set: export it, or put {env}=... in .harness2/.env (ignored by git)." + ) + elif env: + lines.append(f"{env} is set.") + return "\n".join(lines) + + +def _q(value) -> str: + return S._literal(value) + + +LOGIN_CHECKS = { + "claude-login": (["claude", "auth", "status"], "Claude Code", "claude auth login"), +} + + +def _quiet(argv: list[str]) -> bool: + try: + return subprocess.run(argv, capture_output=True, timeout=20).returncode == 0 + except (OSError, subprocess.TimeoutExpired): + return False diff --git a/harness2/cli/judge.py b/harness2/cli/judge.py index 6e8be6f0..f45a5666 100644 --- a/harness2/cli/judge.py +++ b/harness2/cli/judge.py @@ -12,32 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Judge the SELECTED run of each finished task in one namespace, and cache the rows. - -Only the run the blind terminal selector chose is scored. A judge failure is recorded as -ungraded (score=None), never as a zero. Re-running re-judges every selection, but -adapters serve cached judge details and a failed re-judge never replaces an existing -graded row. - - python3 -m harness2.cli.judge --bench jb --substrate oc --k 2 --modes parallel - -With --bases it scores this namespace's reference executions instead, so the initial -harness can be reported beside the adapted one. - -Output: //judged.json (judge.write format), consumed by cli.report; -or base_judged.json under --bases, one row per (task, base). -""" +"""Judge finished selections using the saved run settings.""" from __future__ import annotations -import argparse import json -import sys from pathlib import Path -from .. import config from ..core import judge as J +from ..core.runmark import resolve_path +from ..settings import ConfigError from . import _common as C +from ._adopt import open_run _DERIVED = ("frac", "graded") @@ -54,8 +40,8 @@ def _load_rows(path: Path) -> list[J.Scored]: except (OSError, json.JSONDecodeError) as exc: raise ValueError(f"unreadable judged cache {path}: {exc}; move it aside to rebuild") rows = [] - for row in raw: - row = {k: v for k, v in row.items() if k not in _DERIVED} + for row in raw.get("rows", []) if isinstance(raw, dict) else raw: + row = {k: v for k, v in row.items() if k not in (*_DERIVED, "judge_model")} if "mode" not in row and "arm" in row: row["mode"] = row.pop("arm") rows.append(J.Scored(**row)) @@ -96,8 +82,7 @@ def collect_selections(ns_dir: Path, modes: tuple[str, ...]) -> list[dict]: try: summary = json.loads(summary_path.read_text()) except (OSError, json.JSONDecodeError) as exc: - print(f" ! skipping unreadable {summary_path}: {exc}", flush=True) - continue + raise ConfigError(f"cannot read {summary_path}: {exc}") from exc for mode in modes: sel = (summary.get("selections") or {}).get(mode) if sel: @@ -105,100 +90,107 @@ def collect_selections(ns_dir: Path, modes: tuple[str, ...]) -> list[dict]: return out -def judge_bases(adapter, ns_dir: Path) -> int: - """Score every reference execution recorded in this namespace's task summaries.""" - rows, legacy = [], 0 +def add_parser(subparsers): + p = subparsers.add_parser( + "judge", + help="grade the result each task adopted", + description="Grade each task's adopted result with the task's check command or " + "rubric.md; tasks with neither are skipped. NAME defaults to the most recent run.", + ) + p.add_argument("name", nargs="?", help="run name (default: the most recent run)") + p.add_argument( + "--bases", + action="store_true", + help="grade the runs of the starting harness instead, for comparison", + ) + return p + + +def _judge_model(path, model): + if not path.is_file(): + return + try: + data = json.loads(path.read_text()) + except (OSError, ValueError) as exc: + raise ConfigError(f"cannot read {path}: {exc}") from exc + old = data.get("judge_model") if isinstance(data, dict) else None + if old is not None and old != model: + raise ConfigError(f"judge model differs: {old!r} -> {model!r}; delete {path} to re-judge") + + +def judge_bases(adapter, ns_dir: Path, *, log, allowed) -> list[dict]: + rows = [] for summary_path in sorted(ns_dir.glob("*/summary.json")): - try: - summary = json.loads(summary_path.read_text()) - except (OSError, json.JSONDecodeError) as exc: - print(f" ! skipping unreadable {summary_path}: {exc}", flush=True) + summary = json.loads(summary_path.read_text()) + task = summary["task_id"] + if not allowed(task): continue - task_id = summary.get("task_id", "") for entry in summary.get("base") or []: if not isinstance(entry, dict) or not entry.get("run_dir"): - legacy += 1 + log(f"{task}: reference execution has no run directory") continue - label, run_dir = entry.get("label", ""), Path(entry["run_dir"]) - try: - got = adapter.score(run_dir, task_id) if run_dir.is_dir() else None - except Exception as exc: - got = None - print(f" {task_id[:52]:<52} {label} ERROR {type(exc).__name__}: {exc}", flush=True) - else: - shown = "-" if got is None else f"{got.get('n_passed')}/{got.get('n_criteria')}" - print(f" {task_id[:52]:<52} {label} {shown}", flush=True) - rows.append( - {"task": task_id, "label": label, "run_id": entry.get("run_id", ""), "score": got} + run_dir = resolve_path(entry["run_dir"], adapter.spec.runs_root) + row = J.judge_one( + adapter, + task, + "base", + summary["k"], + entry.get("label", ""), + entry.get("run_id", ""), + run_dir, + chosen_is_base=True, ) - if legacy: - print( - f" {legacy} reference execution(s) predate run-directory recording and cannot be " - f"scored; re-run those tasks to include them.", - flush=True, - ) - if not rows: - print(f"no reference executions recorded under {ns_dir}; wrote nothing", flush=True) - return 2 - out = ns_dir / "base_judged.json" - out.write_text(json.dumps(rows, indent=1, default=str), encoding="utf-8") - graded = sum(1 for r in rows if r["score"] is not None) - print(f"wrote {out}: {len(rows)} row(s), {graded} graded") - return 0 if graded == len(rows) else 2 - - -def main(argv: list[str] | None = None) -> int: - p = argparse.ArgumentParser( - prog="harness2.cli.judge", description=__doc__, formatter_class=C.HelpFormatter - ) - C.add_cell_args(p.add_argument_group("cell")) - C.add_run_identity_args(p.add_argument_group("run identity (names the sweep)")) - p.add_argument( - "--bases", - action="store_true", - help="score this namespace's reference executions into base_judged.json " - "instead of the selected runs", + rows.append(row.__dict__ | {"graded": row.graded, "frac": row.frac}) + return rows + + +def run(args, ctx) -> int: + with ctx.progress(): + adapter, root = open_run(ctx, args.name) + record = C.load_run(root) + config = C.run_settings(ctx, record) + model = config.role("judge")["model"] + plan = record["schedule"] + ns_dir = C.runs_root_for(adapter) / C.summary_namespace( + adapter, plan["k"], seed=plan.get("seed") ) - args = p.parse_args(argv) - modes = C.parse_modes(args.modes) - - config.validate_models(judge=True) - adapter = C.make_adapter(args.bench, args.subset, args.substrate) - ns = C.summary_namespace(adapter, args.k, seed=args.seed) - ns_dir = C.runs_root_for(adapter) / ns - identity = ns_dir / "config.json" - if identity.is_file() and json.loads(identity.read_text()).get("judge") != config.JUDGE_MODEL: - raise SystemExit("Judge model changed for this experiment; choose a new HARNESS2_RUNS_DIR") - if not ns_dir.is_dir(): - raise SystemExit( - f"nothing to judge: {ns_dir} does not exist — has the sweep `{ns}` run on this machine?" - ) + path = ns_dir / ("base_judged.json" if args.bases else "judged.json") + _judge_model(path, model) + skipped = set() + + def allowed(task): + if not hasattr(adapter, "has_judge") or adapter.has_judge(task): + return True + if task not in skipped: + ctx.log(f"{task}: no judge; skipped") + skipped.add(task) + return False + if args.bases: - return judge_bases(adapter, ns_dir) - - sels = collect_selections(ns_dir, modes) - if not sels: - raise SystemExit( - f"nothing to judge: no selections for modes " - f"{[C.mode_label(m) for m in modes]} under {ns_dir} (tasks may still " - f"be running, or the wrong --k/--seed was given)" - ) - - print(f"judging {len(sels)} selection(s) under {ns} ...", flush=True) - rows = J.judge_selections(adapter, sels, parallel=_PARALLEL) - - out_path = ns_dir / "judged.json" - merged = _merge(_load_rows(out_path), rows) - J.write(merged, out_path) - - graded = sum(1 for r in merged if r.graded) - print( - f"judged.json: {len(merged)} row(s), {graded} graded, " - f"{len(merged) - graded} ungraded (judge failure or no pick — NOT zeros)" + with ctx.progress(): + rows = judge_bases(adapter, ns_dir, log=ctx.log, allowed=allowed) + else: + selections = [ + s + for s in collect_selections(ns_dir, C.parse_modes(plan["modes"])) + if allowed(s["task_id"]) + ] + with ctx.progress(): + new = J.judge_selections(adapter, selections, parallel=_PARALLEL) + merged = _merge([row for row in _load_rows(path) if allowed(row.task_id)], new) + rows = [row.__dict__ | {"graded": row.graded, "frac": row.frac} for row in merged] + if not rows: + ctx.out([], lambda _: "no tasks to judge") + return 0 + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps({"judge_model": model, "rows": rows}, indent=2) + "\n", encoding="utf-8" ) - print(f"wrote {out_path}") - return 0 if rows and all(r.graded for r in rows) else 2 - - -if __name__ == "__main__": - sys.exit(main()) + ctx.out( + rows, + lambda data: ( + f"Graded {sum(row['graded'] for row in data)} of {len(data)} result(s); " + f"saved to {path}. Summarise with `harness2 report`." + ), + ) + return 0 if all(row["graded"] for row in rows) else 1 diff --git a/harness2/cli/login.py b/harness2/cli/login.py new file mode 100644 index 00000000..3cc39362 --- /dev/null +++ b/harness2/cli/login.py @@ -0,0 +1,102 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Show whether the subscription logins harness2 uses are ready, and set up Codex's. + +claude-login uses Claude Code's own login as it is: runs switch off your personal settings, +hooks and plugins with flags. Codex has no such switch for its hooks.json and AGENTS.md, so +chatgpt-login runs use a Codex home of their own, signed in once with `harness2 login codex`. +""" + +from __future__ import annotations + +import argparse +import os +import shutil +import subprocess +import sys + +from .. import providers + + +def add_parser(sub) -> argparse.ArgumentParser: + p = sub.add_parser( + "login", + help="check the subscription logins, or sign Codex in for harness2", + description="Without an argument, show whether claude-login and chatgpt-login are " + "ready. `harness2 login codex` signs in to ChatGPT in a Codex home kept for harness2, " + "apart from your own ~/.codex.", + ) + p.add_argument("service", nargs="?", choices=("codex",), help="sign this CLI in for harness2") + p.add_argument( + "--device", + action="store_true", + help="sign in with a device code, for machines without a browser", + ) + return p + + +def run(args, ctx) -> int: + home = providers.codex_login_home(ctx.settings.provider("chatgpt-login")) + if args.service is None: + rows = _status(home) + ctx.out(rows, lambda data: "\n".join(f"{r['provider']:<14} {r['detail']}" for r in data)) + return 0 + if shutil.which("codex") is None: + print("`codex` is not on PATH; install Codex first.", file=sys.stderr) + return 1 + if not sys.stdin.isatty(): + print("Signing in needs a terminal and a browser (or --device).", file=sys.stderr) + return 1 + home.mkdir(parents=True, exist_ok=True) + print( + f"Signing Codex in to ChatGPT for harness2. This login lives in {home}, apart from " + "your own ~/.codex, so your personal hooks and AGENTS.md stay out of harness2's runs.\n" + ) + env = {k: v for k, v in os.environ.items() if k not in ("OPENAI_API_KEY", "CODEX_API_KEY")} + subprocess.run( + ["codex", "login", *(["--device-auth"] if args.device else [])], + env={**env, "CODEX_HOME": str(home)}, + ) + if not providers.codex_logged_in(home): + print("Codex did not finish signing in.", file=sys.stderr) + return 1 + print("Signed in. Models written chatgpt-login/ now run on this login.") + return 0 + + +def _status(home) -> list[dict]: + claude = shutil.which("claude") and _ok(["claude", "auth", "status"]) + codex = providers.codex_logged_in(home) + return [ + { + "provider": "claude-login", + "ready": bool(claude), + "detail": "ready (Claude Code's own login)" + if claude + else "Claude Code is not logged in; run `claude auth login`", + }, + { + "provider": "chatgpt-login", + "ready": codex, + "detail": f"ready ({home})" if codex else "not signed in; run `harness2 login codex`", + }, + ] + + +def _ok(argv: list[str]) -> bool: + try: + return subprocess.run(argv, capture_output=True, timeout=20).returncode == 0 + except (OSError, subprocess.TimeoutExpired): + return False diff --git a/harness2/cli/main.py b/harness2/cli/main.py new file mode 100644 index 00000000..400c0237 --- /dev/null +++ b/harness2/cli/main.py @@ -0,0 +1,117 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The harness2 command dispatcher.""" + +from __future__ import annotations + +import argparse +import importlib +import sys +from importlib.metadata import version + +from ._common import Context + +COMMANDS = ( + "init", + "doctor", + "task", + "run", + "status", + "stop", + "judge", + "report", + "runs", + "diff", + "eval", + "apply", + "config", + "chat", + "guide", + "login", + "setup", + "demo", + "bridge", + "completion", +) +ALIASES = {"harness2-" + name: name for name in ("judge", "report", "setup", "demo", "bridge")} +ALIASES["harness2-check"] = "doctor" + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="harness2", + description="Tune your coding agent's harness on your own tasks.", + epilog="Start with `harness2 init` in your project. " + "Documentation: https://github.com/3ae3ae/harness2-cli", + ) + parser.add_argument("-C", "--project", metavar="DIR", help="run as if started in DIR") + parser.add_argument("--profile", metavar="NAME", help="apply the settings profile NAME") + parser.add_argument( + "-c", + "--config", + action="append", + default=[], + metavar="KEY=VALUE", + help="override a setting, e.g. -c run.k=2 (repeatable)", + ) + parser.add_argument("-q", "--quiet", action="store_true", help="print results only") + parser.add_argument("-v", "--verbose", action="store_true", help="print debug details") + parser.add_argument("--json", action="store_true", help="print results as JSON") + parser.add_argument("--version", action="version", version=version("harness2-cli")) + subparsers = parser.add_subparsers(dest="command", required=True) + for name in COMMANDS: + module = importlib.import_module(f"harness2.cli.{name}") + module.add_parser(subparsers) + subparsers.choices[name].set_defaults(_module=module) + return parser + + +def main(argv: list[str] | None = None) -> int: + argv = list(sys.argv[1:] if argv is None else argv) + alias = ALIASES.get(sys.argv[0].rsplit("/", 1)[-1]) + if alias: + # Aliases still accept global options before their subcommand's arguments. + index = 0 + while index < len(argv): + flag = argv[index].split("=", 1)[0] + if flag in ("-C", "--project", "--profile", "-c", "--config"): + index += 1 if "=" in argv[index] else 2 + elif argv[index].startswith(("-C", "-c")): + index += 1 + elif flag in ("-q", "--quiet", "-v", "--verbose", "--json", "--version"): + index += 1 + else: + break + argv.insert(index, alias) + args = build_parser().parse_args(argv) + args.argv = argv + ctx = Context(args, getattr(args._module, "FLAGS", {})) + try: + ctx.debug(f"command: {args.command}") + return args._module.run(args, ctx) + except KeyboardInterrupt: + if args.json: + ctx.out({"error": "interrupted", "exit_code": 130}, str) + return 130 + except (OSError, RuntimeError, ValueError, KeyError) as exc: + if args.json: + ctx.out({"error": str(exc), "exit_code": 1}, str) + else: + ctx.log(f"error: {exc}") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/harness2/cli/report.py b/harness2/cli/report.py index 8f80fc56..cbc7e01e 100644 --- a/harness2/cli/report.py +++ b/harness2/cli/report.py @@ -12,24 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Render the score report for one cell from cached judged.json rows — read-only. - -The figure each benchmark reports per (mode, k) — partial credit and all-criteria pass on LAB, -one score elsewhere — with pick provenance, the k=1 noise-floor note and the sanity warnings; -fell-back counts and improver spend come from the task summaries. - - python3 -m harness2.cli.report --bench jb --substrate oc --k 1,2 -""" +"""Render cached scores for one or more runs.""" from __future__ import annotations -import argparse import json -import sys from pathlib import Path from ..core import report as R from ..core import schedule as SCHED +from ..settings import ConfigError from . import _common as C from .judge import _load_rows @@ -42,8 +34,7 @@ def _spend_and_fallbacks(ns_dir: Path, modes: tuple[str, ...], k: int) -> tuple[ try: summary = json.loads(summary_path.read_text()) except (OSError, json.JSONDecodeError) as exc: - print(f" ! skipping unreadable {summary_path}: {exc}", flush=True) - continue + raise ConfigError(f"cannot read {summary_path}: {exc}") from exc records = SCHED._modes_of(summary) for mode in modes: key = (mode, k) @@ -55,60 +46,46 @@ def _spend_and_fallbacks(ns_dir: Path, modes: tuple[str, ...], k: int) -> tuple[ return fell_back, usd -def main(argv: list[str] | None = None) -> int: - p = argparse.ArgumentParser( - prog="harness2.cli.report", description=__doc__, formatter_class=C.HelpFormatter - ) - C.add_cell_args(p.add_argument_group("cell")) - g = p.add_argument_group("run identity (names the sweeps)") - g.add_argument("--k", default="1", help="comma list of scales to report, e.g. 1,2,3") - g.add_argument( - "--seed", - type=int, - default=None, - help="stream-order seed of the sweep; omit for the seedless namespace", +def add_parser(subparsers): + p = subparsers.add_parser( + "report", + help="summarise a run's grades", + description="Average the grades `harness2 judge` wrote, per recursion mode. Name several " + "runs to compare them.", ) - C.add_modes_arg(g) - args = p.parse_args(argv) - modes = C.parse_modes(args.modes) - try: - ks = sorted({int(s) for s in args.k.split(",") if s.strip()}) - except ValueError: - raise SystemExit(f"--k takes a comma list of ints, got {args.k!r}") - if not ks: - raise SystemExit("--k named no scale") + p.add_argument("names", nargs="*", help="run names (default: the most recent run)") + return p - adapter = C.make_adapter(args.bench, args.subset, args.substrate) - rows, fell_back, usd = [], {}, {} - for k in ks: - ns_dir = C.runs_root_for(adapter) / C.summary_namespace(adapter, k, seed=args.seed) - judged = ns_dir / "judged.json" - got = _load_rows(judged) - if not got: - print(f" ! no judged rows at {judged} — run cli.judge for k={k} first", flush=True) - continue - rows.extend(r for r in got if r.mode in modes) - fb, sp = _spend_and_fallbacks(ns_dir, modes, k) - fell_back.update(fb) - usd.update(sp) - if not rows: - raise SystemExit("nothing to report") - title = ( - f"{adapter.spec.name}-{adapter.substrate.name}" - + (f" seed={args.seed}" if args.seed is not None else "") - + f" modes={','.join(C.mode_label(m) for m in modes)}" - ) - summaries = R.summarise(rows, fell_back=fell_back, usd=usd) - print(R.render(summaries, title=title, bench=adapter.spec.name)) - spend = sum(s.usd for s in summaries) - if spend: - print( - f"\n improver (editors, composer, selection, reflection) spend across reported modes: " - f"${spend:.2f}" +def run(args, ctx) -> int: + reports, rows = [], [] + for name in args.names or [None]: + root = C.resolve_run(ctx, name) + record = C.load_run(root) + scored, fallbacks, spend = [], {}, {} + for path in sorted((root / "adapt").glob("*/judged.json")): + got = _load_rows(path) + scored.extend(got) + for k in {row.k for row in got}: + fb, usd = _spend_and_fallbacks( + path.parent, tuple({row.mode for row in got if row.k == k}), k + ) + for key, value in fb.items(): + fallbacks[key] = fallbacks.get(key, 0) + value + for key, value in usd.items(): + spend[key] = spend.get(key, 0) + value + rows.extend( + {"name": root.name, **row.__dict__, "frac": row.frac, "graded": row.graded} + for row in scored ) + reports.append( + R.render( + R.summarise(scored, fell_back=fallbacks, usd=spend), + title=root.name, + bench=record["suite"], + ) + ) + if not rows: + raise ConfigError("nothing to report") + ctx.out(rows, lambda _: "\n\n".join(reports)) return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/harness2/cli/run.py b/harness2/cli/run.py new file mode 100644 index 00000000..fac49701 --- /dev/null +++ b/harness2/cli/run.py @@ -0,0 +1,404 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Configure and execute a resumable task stream.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +from .. import core, registry +from ..core import events, ledger, schedule +from ..core.proposer import Proposer +from ..settings import ConfigError +from . import _common as C +from ._final import consolidate + +FLAGS = { + "suite": "suite.name", + "executor": "executor.name", + "k": "run.k", + "modes": "run.modes", + "seed": "run.seed", + "workers": "run.workers", + "budget_usd": "run.budget_usd", + "max_calls": "run.max_calls", + "selector": "selector.channel", + "steer": "improver.steer", +} + + +def add_parser(subparsers): + p = subparsers.add_parser( + "run", + help="refine the harness on the tasks", + description="Run the agent on every task, propose and probe harness edits, and adopt a " + "refined harness per task. Running the same command again resumes and reuses every " + "finished step. Model calls cost money: preview with --dry-run, cap with --budget-usd " + "or --max-calls.", + ) + p.add_argument("--suite", help="task suite (default: tasks, your own)") + p.add_argument("--bench", dest="suite", help=argparse.SUPPRESS) + p.add_argument("--executor", help="agent that runs the tasks: claude, codex or command") + p.add_argument("--name", help="run name (default: -k)") + p.add_argument("--k", type=int, help="recursion budget per task (default 1)") + p.add_argument( + "--mode", + "--modes", + dest="modes", + help="parallel (default), sequential, or both separated by a comma", + ) + p.add_argument("--seed", type=int, help="shuffle the task order within each domain") + p.add_argument( + "--split", choices=("all", "train", "test"), default="all", help="run one split only" + ) + p.add_argument("--split-file", type=Path, help="task partition file (benchmark suites)") + p.add_argument("--only-domains", default="", metavar="D,...", help="run only these domains") + p.add_argument("--only-tasks", default="", metavar="ID,...", help="run only these tasks") + p.add_argument("--max-tasks", type=int, metavar="N", help="run at most N tasks per domain") + p.add_argument("--workers", type=int, metavar="N", help="domains to run at the same time") + p.add_argument( + "--budget-usd", type=float, metavar="USD", help="stop once the run has spent this much" + ) + p.add_argument("--max-calls", type=int, metavar="N", help="stop after N model calls") + p.add_argument( + "--selector", choices=("human",), help="pick the final harness yourself (when k > 1)" + ) + p.add_argument("--steer", metavar="TEXT|@FILE", help="guidance added to every improver session") + p.add_argument( + "--dry-run", action="store_true", help="show the tasks and the model calls; run nothing" + ) + p.add_argument( + "--detach", action="store_true", help="start the run in the background and return at once" + ) + return p + + +def _csv(raw: str) -> tuple[str, ...]: + return tuple(s.strip() for s in raw.split(",") if s.strip()) + + +def _estimate(tasks: int, k: int, modes: tuple[str, ...], totals: dict) -> dict: + per_mode = { + "rollouts": ledger.expected_rollouts(k), + "improver": ledger.expected_sessions(k), + "selector": int(k > 1), + "reflector": 1, + } + # Reference rollouts are shared by modes; reflection also runs at k=1. + rollouts = tasks * (k + (ledger.expected_rollouts(k) - k) * len(modes)) + sessions = ( + tasks * len(modes) * sum(per_mode[key] for key in ("improver", "selector", "reflector")) + ) + len(modes) + return { + "per_task_per_mode": per_mode, + "rollouts": rollouts, + "sessions": sessions, + "calls": rollouts + sessions, + "average_cost_per_call": totals["spent_usd"] / totals["calls"] if totals["calls"] else None, + } + + +def _spend(totals: dict) -> str: + # Codex and subscriptions report no money; $0.00 would read as "free". + money = f"${totals['spent_usd']:.2f} spent" if totals["spent_usd"] else "spend not reported" + return f" · {money} · {totals['calls']} model calls" + + +def _stopped(name: str, reason: str, totals: dict) -> str: + why = { + "interrupted": "interrupted", + "budget_usd": f"reached --budget-usd {totals['budget_usd']:g}", + "max_calls": f"reached --max-calls {totals['max_calls']}", + "usage_limit": "reached the model provider's usage limit", + }.get(reason, reason) + advice = ( + "Run the same command to resume." + if reason == "interrupted" + else "Raise the limit and run the same command to resume." + ) + if reason == "usage_limit": + advice = "Run the same command once the limit resets to resume." + return f"Run {name} stopped: {why}{_spend(totals)}.\n{advice}" + + +def _print_stream(data) -> str: + domains, stream, est = data["domains"], data["stream"], data["estimate"] + lanes = data["workers"] + lines = [ + f"Run {data['name']} would refine {len(stream)} task(s) in {len(domains)} domain(s) " + f"(k={data['k']}, {', '.join(data['modes'])}), " + + ("one domain at a time." if lanes == 1 else f"{lanes} domains at a time."), + "", + ] + width = max(6, *(len(d) for d in domains)) + lines.append(f" {'#':>4} {'DOMAIN':<{width}} TASK") + lines.extend( + f" {i:>4} {domain:<{width}} {task}" for i, (domain, task) in enumerate(stream, 1) + ) + lines += [ + "", + f"Model calls: {est['rollouts']} agent runs + {est['sessions']} improver, selector and " + f"reflector sessions = {est['calls']} (retries can add more).", + ] + if est["average_cost_per_call"] is not None: + cost = est["average_cost_per_call"] * est["calls"] + lines.append( + f"At this run's average so far, ${est['average_cost_per_call']:.3f} per call, that " + f"is about ${cost:.2f} in total." + ) + lines.append(f"Stored in {data['run_dir']}") + return "\n".join(lines) + + +def _detach(args, ctx, name: str, root: Path) -> int: + root.mkdir(parents=True, exist_ok=True) + log = root / "run.log" + argv = [arg for arg in args.argv if arg != "--detach"] + if args.name is None: + argv.extend(["--name", name]) + with log.open("a", encoding="utf-8") as output: + child = subprocess.Popen( + [sys.executable, "-m", "harness2", *argv], + stdin=subprocess.DEVNULL, + stdout=output, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + try: + code = child.wait(timeout=2) + except subprocess.TimeoutExpired: + code = None + if code: + print( + "\n".join(log.read_text(encoding="utf-8", errors="replace").splitlines()[-20:]), + file=sys.stderr, + ) + return code + ctx.out( + {"name": name, "pid": child.pid, "log": log}, + lambda data: ( + f"Started run {name} in the background (pid {data['pid']}).\n" + f"Follow it: harness2 status {name} --watch\n" + f"Stop it: harness2 stop {name}\n" + f"Log: {log}" + ), + ) + return 0 + + +def run(args, ctx) -> int: + events.start(None) + status, reason, name = "error", "", args.name or "" + pid_path = None + try: + if args.detach and args.dry_run: + raise ConfigError("--detach starts a real run; drop --dry-run") + project, config = ctx.project, ctx.settings + k, seed = config.get("run.k"), config.get("run.seed") + modes = C.parse_modes(config.get("run.modes")) + name = args.name or C.default_name(config) + root = C.run_path(ctx, name) + if not args.dry_run and (process := C.run_process(root)) is not None: + raise ConfigError(f"run {name} is already running (pid {process['pid']})") + if args.detach: + return _detach(args, ctx, name, root) + suite = registry.load(config.get("suite.name")) + split_file = (project / args.split_file).resolve() if args.split_file else None + suite_ctx = registry.SuiteContext(config, project, root, split_file) + with ctx.progress(): + adapter = suite.make_adapter(suite_ctx) + problems = suite.check(suite_ctx) if hasattr(suite, "check") else [] + if problems: + raise ConfigError("\n".join(problems)) + filters = dict( + split=args.split, + seed=seed, + only_domains=_csv(args.only_domains), + only_tasks=_csv(args.only_tasks), + max_tasks=args.max_tasks, + ) + domains = schedule.task_stream(adapter, **filters) + stream = [(domain, task) for domain, tasks in domains.items() for task in tasks] + if not stream: + raise ConfigError("zero tasks scheduled") + if args.dry_run: + ctx.out( + { + "name": name, + "suite": adapter.spec.name, + "substrate": adapter.substrate.name, + "split": args.split, + "seed": seed, + "max_tasks": args.max_tasks, + "k": k, + "modes": [C.mode_label(m) for m in modes], + "namespaces": { + C.mode_label(m): schedule.ns( + adapter.spec.name, adapter.substrate.name, m, k, seed=seed + ) + for m in modes + }, + "summaries": C.runs_root_for(adapter) + / C.summary_namespace(adapter, k, seed=seed), + "library": C.library_root_for(adapter, k, seed=seed), + "workers": config.get("run.workers"), + "domains": domains, + "stream": stream, + "estimate": _estimate(len(stream), k, modes, C.event_totals(root)), + "run_dir": root, + }, + _print_stream, + ) + status = "ok" + return 0 + C.save_run( + root, + config, + stream, + { + "k": k, + "modes": config.get("run.modes"), + **filters, + "split_file": str(split_file) if split_file else None, + }, + ) + pid_path = root / "run.pid" + pid_path.write_text( + json.dumps( + { + "pid": os.getpid(), + "started": datetime.now(timezone.utc).isoformat(), + "argv": args.argv, + } + ) + + "\n", + encoding="utf-8", + ) + core.configure( + **{ + key: config.get(f"run.{key}") + for key in ("phase_width", "transport_attempts", "proposer_concurrency") + } + ) + events.start( + root / "events.jsonl", + budget_usd=config.get("run.budget_usd"), + max_calls=config.get("run.max_calls"), + ) + audit = root / "adapt/_audit" / C.summary_namespace(adapter, k, seed=seed) + prop = Proposer(adapter.spec.channels["proposer"], audit_dir=audit) + selector_spec = adapter.spec.channels["selector"] + if selector_spec.kind == "human": + from .human import HumanSelector + + selector = HumanSelector(selector_spec, audit_dir=audit) + else: + selector = Proposer(selector_spec, audit_dir=audit) + reflector = Proposer( + adapter.spec.channels.get( + "reflector", + adapter.spec.channels["proposer"] + if selector_spec.kind == "human" + else selector_spec, + ), + audit_dir=audit, + ) + events.emit("run_start", name=name) + with ctx.progress(): + result = schedule.run_sweep( + k, + adapter=adapter, + prop=prop, + selector_prop=selector, + reflector_prop=reflector, + runs_root=C.runs_root_for(adapter), + library_root=C.library_root_for(adapter, k, seed=seed), + modes=modes, + domain_parallel=config.get("run.workers"), + **filters, + ) + failed = sum(int(row.get("failed", 0)) for row in result["domains"]) + errors = [row["error"] for row in result["domains"] if row.get("error")] + status, reason = ( + ("error", "; ".join(errors)) + if errors + else (("failed", "failed tasks") if failed else ("ok", "")) + ) + done = sum(int(row.get("done", 0)) for row in result["domains"]) + final = {} + for mode in modes: + label = C.mode_label(mode) + try: + if not any( + path.parent.parent.name == audit.name + and (selection := (row.get("selections") or {}).get(mode) or {}).get("ok") + and selection.get("chosen_label") + for path, row in C.summaries(root) + ): + continue + final[label] = consolidate(ctx, adapter, root, label)["path"] + except Exception as exc: + print( + f"Could not build the final harness ({label}): {exc}. " + "Run the same command again to retry.", + file=sys.stderr, + ) + if errors and args.json: + raise ConfigError(reason) + ctx.out( + {"result": result, "totals": events.totals(), "final": final}, + lambda data: ( + f"Run {name}: {done} task(s) done, {failed} failed" + + _spend(data["totals"]) + + (f"\nErrors: {reason}" if errors else "") + + "".join(f"\nFinal harness ({mode}): {path}" for mode, path in final.items()) + + f"\nNext: harness2 diff {name} shows it; harness2 apply {name} writes it into " + f"the project; harness2 eval {name} measures it first." + ), + ) + return 1 if errors else 3 if failed else 0 + except events.Stopped as exc: + reason = str(exc) + status = "interrupted" if reason == "interrupted" else "stopped" + ctx.out( + {"error": "interrupted", "exit_code": 130} + if status == "interrupted" and args.json + else events.totals(), + lambda totals: _stopped(name, reason, totals), + ) + return 130 if status == "interrupted" else 4 + except KeyboardInterrupt: + status, reason = "interrupted", "interrupted" + events.request_stop(reason) + ctx.out( + {"error": "interrupted", "exit_code": 130} if args.json else events.totals(), + lambda totals: _stopped(name, reason, totals), + ) + return 130 + except Exception as exc: + reason = str(exc) + raise + finally: + try: + events.emit("run_end", status=status, reason=reason) + finally: + if pid_path is not None: + pid_path.unlink(missing_ok=True) diff --git a/harness2/cli/run_sweep.py b/harness2/cli/run_sweep.py deleted file mode 100644 index 0876f4e6..00000000 --- a/harness2/cli/run_sweep.py +++ /dev/null @@ -1,239 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Run one adaptation sweep: every task of a split, domains parallel, tasks in stream order. - -Thin argparse over core.schedule.run_sweep. Costs money when run for real: each task is -R=4k task-executor rollouts and P=3k improver sessions. `--dry-run` prints the exact -(domain, task) stream and namespaces without creating a Proposer, touching a substrate or -validating a model setting. - -A real run checks the judge before the first rollout, because the namespace records the -judge model and `harness2-judge` refuses to grade it under a different one: discovering a -missing judge after the sweep means paying for the sweep twice. The check is offline — a -judge model must be named and a benchmark that grades in its own environment must have it -installed — so it catches an unset or unbuilt judge, not one an account cannot serve. - - python3 -m harness2.cli.run_sweep --bench jb --substrate oc --k 2 --split all - python3 -m harness2.cli.run_sweep --bench wb --subset web --substrate cx \\ - --k 1 --split all --seed 7 --modes parallel --max-tasks 2 --dry-run -""" - -from __future__ import annotations - -import argparse -import json -import sys -from datetime import datetime, timezone -from pathlib import Path - -from .. import config -from ..core import schedule -from ..core.proposer import Proposer -from . import _common as C - - -def _build_parser() -> argparse.ArgumentParser: - p = argparse.ArgumentParser( - prog="harness2.cli.run_sweep", description=__doc__, formatter_class=C.HelpFormatter - ) - C.add_cell_args(p.add_argument_group("cell")) - C.add_run_identity_args(p.add_argument_group("run identity (enters the namespace)")) - g = p.add_argument_group("run control") - g.add_argument("--split", choices=("all", "train", "test"), default="all") - g.add_argument("--split-file", type=Path, help="optional partition from scripts/split_tasks.py") - g.add_argument("--workers", type=int, default=1, help="domains to run concurrently") - g.add_argument( - "--only-domains", default="", help="comma list restricting the sweep to these domains" - ) - g.add_argument( - "--max-tasks", - type=int, - default=None, - help="per-domain cap on the stream (the first N tasks of every domain); " - "drop it for the full run", - ) - g.add_argument( - "--dry-run", - action="store_true", - help="print the stream order, the namespaces and the split's domains, then " - "exit — no rollouts, no sessions, nothing written", - ) - return p - - -def _csv(raw: str) -> tuple[str, ...]: - return tuple(s.strip() for s in raw.split(",") if s.strip()) - - -def _stream_for(adapter, args) -> tuple[dict[str, list[str]], list[tuple[str, str]]]: - by_domain = schedule.task_stream( - adapter, - split=args.split, - seed=args.seed, - only_domains=_csv(args.only_domains), - max_tasks=args.max_tasks, - ) - return by_domain, [(domain, task) for domain, tasks in by_domain.items() for task in tasks] - - -def _print_stream(adapter, args, modes) -> int: - by_domain, stream = _stream_for(adapter, args) - print( - f"# {adapter.spec.name}-{adapter.substrate.name} split={args.split} " - f"seed={args.seed} k={args.k} modes={','.join(C.mode_label(m) for m in modes)}" - ) - for m in modes: - namespace = schedule.ns( - adapter.spec.name, adapter.substrate.name, m, args.k, seed=args.seed - ) - print(f"# namespace[{C.mode_label(m)}]: {namespace}") - print( - f"# summaries under: {C.runs_root_for(adapter)}/" - f"{C.summary_namespace(adapter, args.k, seed=args.seed)}" - ) - print(f"# library root: {C.library_root_for(adapter, args.k, seed=args.seed)}") - print(f"# domains parallel: {args.workers}") - for i, (domain, task) in enumerate(stream, 1): - print(f"{i:>4} {domain:<40} {task}") - print("# domains of this split, with task counts (the legal --only-domains names):") - for d, tasks in by_domain.items(): - print(f"{len(tasks):>4} {d}") - cap = f", max {args.max_tasks} per domain" if args.max_tasks is not None else "" - print( - f"# {len(stream)} tasks over {len(by_domain)} domains " - f"(domains parallel, tasks within a domain sequential{cap})" - ) - return 0 if stream else 1 - - -def main(argv: list[str] | None = None) -> int: - """Parse, then run `_main`, printing a misconfiguration as one remediation line.""" - args = _build_parser().parse_args(argv) - try: - return _main(args) - except (config.ConfigError, RuntimeError, ValueError, FileNotFoundError) as e: - print(f"error: {e}", file=sys.stderr) - return 1 - - -def _main(args: argparse.Namespace) -> int: - if args.k < 1 or args.workers < 1: - raise ValueError("--k and --workers must be positive") - modes = C.parse_modes(args.modes) - - if args.dry_run: - return _print_stream( - C.make_adapter(args.bench, args.subset, args.substrate, args.split_file), args, modes - ) - - adapter = C.make_adapter(args.bench, args.subset, args.substrate, args.split_file) - _stream_for(adapter, args) - from ..preflight import require_ready - - require_ready(adapter, judge=True) - - runs_root = C.runs_root_for(adapter) - library_root = C.library_root_for(adapter, args.k, seed=args.seed) - summary_ns = C.summary_namespace(adapter, args.k, seed=args.seed) - _check_identity(adapter, args, runs_root / summary_ns) - prop = Proposer(adapter.spec.channels["proposer"], audit_dir=runs_root / "_audit" / summary_ns) - - only = _csv(args.only_domains) - lanes = args.workers - print( - f"sweep {summary_ns}: {lanes} domain(s) in parallel, tasks within a domain in stream order", - flush=True, - ) - rep = schedule.run_sweep( - args.k, - adapter=adapter, - prop=prop, - runs_root=runs_root, - library_root=library_root, - split=args.split, - modes=modes, - seed=args.seed, - tag="", - domain_parallel=lanes, - only_domains=only, - max_tasks=args.max_tasks, - pool_policy="finals", - ) - - stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") - rec = runs_root / "_sweeps" / f"{summary_ns}-{stamp}.json" - rec.parent.mkdir(parents=True, exist_ok=True) - rec.write_text(json.dumps(rep, indent=2, default=str), encoding="utf-8") - - done = sum(int(d.get("done", 0)) for d in rep["domains"]) - failed = sum(int(d.get("failed", 0)) for d in rep["domains"]) - dead = [d["domain"] for d in rep["domains"] if d.get("error")] - if not rep["domains"]: - print( - f"\nsweep {summary_ns}: zero tasks scheduled (split={args.split!r}, " - f"only_domains={only!r}, max_tasks={args.max_tasks!r}); " - f"refusing to report success" - ) - print(f"record: {rec}") - return 1 - print( - f"\nsweep {summary_ns}: {done} tasks ok, {failed} failed, " - f"{len(dead)} domain(s) died{': ' + ', '.join(dead) if dead else ''}" - ) - print(f"record: {rec}") - print( - f"next: python3 -m harness2.cli.judge --bench {args.bench}" - + (f" --subset {args.subset}" if args.subset else "") - + f" --substrate {args.substrate} --k {args.k}" - + (f" --seed {args.seed}" if args.seed is not None else "") - + f" --modes {args.modes}" - ) - - if dead: - return 1 - return 2 if failed else 0 - - -def _check_identity(adapter, args, namespace_dir: Path) -> None: - """Prevent a resumed run or library from silently mixing models or task partitions.""" - identity = { - "method_version": config.METHOD_VERSION, - "executor": config.SOLVER_MODEL_OC if args.substrate == "oc" else config.SOLVER_MODEL_CX, - "improver": config.IMPROVER_MODEL, - "judge": config.JUDGE_MODEL, - "project": config.VERTEX_PROJECT, - "location": config.VERTEX_REGION, - "judge_endpoint": config.JB_JUDGE_API_BASE, - "endpoint": config.CODEX_BASE_URL if args.substrate == "cx" else "", - "split": args.split, - "tasks": _stream_for(adapter, args)[1], - "max_tasks": args.max_tasks, - "domains": sorted(_csv(args.only_domains)), - "seed": args.seed, - } - namespace_dir.mkdir(parents=True, exist_ok=True) - path = namespace_dir / "config.json" - if path.is_file(): - if json.loads(path.read_text()) != json.loads(json.dumps(identity)): - raise ValueError( - "This run directory belongs to different models or tasks. " - "Choose a new HARNESS2_RUNS_DIR for a new experiment." - ) - else: - path.write_text(json.dumps(identity, indent=2) + "\n", encoding="utf-8") - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/harness2/cli/runs.py b/harness2/cli/runs.py new file mode 100644 index 00000000..54673993 --- /dev/null +++ b/harness2/cli/runs.py @@ -0,0 +1,140 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""List persisted runs and task progress.""" + +from __future__ import annotations + +import shutil +import sys +from datetime import datetime, timezone + +from . import _common as C +from . import status as S + + +def add_parser(subparsers): + p = subparsers.add_parser("runs", help="list runs or show one") + commands = p.add_subparsers(dest="action", required=True) + commands.add_parser("ls", help="list runs with their progress and spend") + show = commands.add_parser("show", help="show a run's settings, totals and tasks") + show.add_argument("name", help="run name") + rm = commands.add_parser("rm", help="delete a run and everything it stored") + rm.add_argument("name", help="run name") + rm.add_argument("-y", "--yes", action="store_true", help="delete without asking") + return p + + +def describe(root) -> dict: + record = C.load_run(root) + plan = record["schedule"] + summaries = { + row["task_id"]: row + for _, row in C.summaries(root) + if row.get("k") == plan["k"] and row.get("seed") == plan.get("seed") + } + statuses = [ + {"task": task, "domain": domain, "status": summaries.get(task, {}).get("status", "pending")} + for domain, task in record["stream"] + ] + totals = C.event_totals(root) + return { + "run": record, + "totals": totals, + "tasks": statuses, + "done": sum(row["status"] in ("ok", "skipped") for row in statuses), + "total": len(statuses), + "last_activity": totals["last_activity"] + or datetime.fromtimestamp(root.stat().st_mtime, timezone.utc).isoformat(), + } + + +def run(args, ctx) -> int: + if args.action == "show": + root = C.resolve_run(ctx, args.name) + ctx.out(describe(root), lambda data: _show(root, data)) + elif args.action == "rm": + return _rm(ctx, C.resolve_run(ctx, args.name), args.yes) + else: + ctx.project + root = ctx.settings.path("paths.runs") + paths = ( + sorted( + (p for p in root.iterdir() if (p / "run.json").is_file()), + key=lambda p: p.stat().st_mtime_ns, + reverse=True, + ) + if root.is_dir() + else [] + ) + rows = [] + for path in paths: + detail = describe(path) + rows.append( + { + "name": path.name, + "suite": detail["run"]["suite"], + "executor": detail["run"]["executor"], + "done": detail["done"], + "total": detail["total"], + "spent_usd": detail["totals"]["spent_usd"], + "last_activity": detail["last_activity"], + } + ) + ctx.out(rows, _table) + return 0 + + +def _table(rows: list[dict]) -> str: + if not rows: + return "No runs yet; start one with `harness2 run`." + head = ("NAME", "SUITE", "EXECUTOR", "TASKS", "SPENT", "LAST ACTIVITY") + body = [ + ( + r["name"], + r["suite"], + r["executor"], + f"{r['done']}/{r['total']}", + f"${r['spent_usd']:.2f}", + S._when(r["last_activity"]), + ) + for r in rows + ] + widths = [max(len(row[i]) for row in (head, *body)) for i in range(len(head))] + return "\n".join( + " ".join(cell.ljust(w) for cell, w in zip(row, widths)).rstrip() for row in (head, *body) + ) + + +def _show(root, data: dict) -> str: + settings = data["run"]["settings"] + keys = ("executor.name", "executor.model", "improver.channel", "improver.model") + lines = [S.render(S.snapshot(root)), "", "Settings:"] + lines += [f" {key} = {settings[key]}" for key in keys if key in settings] + return "\n".join(lines) + + +def _rm(ctx, root, yes: bool) -> int: + size = sum(p.stat().st_size for p in root.rglob("*") if p.is_file()) + if not yes: + if not sys.stdin.isatty(): + print("Not a terminal: pass --yes to delete.", file=sys.stderr) + return 1 + question = f"Delete run {root.name} and everything it stored ({size / 1e6:.1f} MB)? [y/N] " + if input(question).strip().lower() != "y": + ctx.log("Nothing deleted.") + return 0 + shutil.rmtree(root) + ctx.log(f"Deleted run {root.name}.") + return 0 diff --git a/harness2/cli/setup.py b/harness2/cli/setup.py index 6c9afc35..01bf4236 100644 --- a/harness2/cli/setup.py +++ b/harness2/cli/setup.py @@ -1,4 +1,4 @@ -# Copyright 2026 Google LLC +# Copyright 2026 The Harness2 Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,109 +12,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Install a benchmark's pinned tools, runner overlays, dependencies, and data.""" +"""Dispatch setup arguments to an installed suite.""" from __future__ import annotations import argparse -import shutil -import subprocess -from harness2 import config, install +from .. import registry +from ..settings import ConfigError -def main(argv=None): - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("benchmark", choices=("jb", "lab", "wb")) - p.add_argument( - "--subset", - choices=("office", "web", "code"), - default=None, - help="wb only: one subset to install; omitted installs office, web, and code", +def add_parser(subparsers): + p = subparsers.add_parser( + "setup", + help="install what a plugin suite needs", + description="Run a plugin suite's installer, for example `harness2 setup lab`. " + "Arguments after the suite name are passed to the installer.", ) - p.add_argument("--substrate", choices=("oc", "cx"), default="oc") - p.add_argument("--dry-run", action="store_true") - args = p.parse_args(argv) - if args.subset and args.benchmark != "wb": - p.error("--subset only applies to wb") - dry = args.dry_run - missing = install.missing_runners() - if missing and not dry: - p.exit(1, f"setup failed: {missing}\n") - if missing: - print(f"Note: {missing}") - name = {"jb": "job-bench-eval", "lab": "harvey-labs", "wb": "workbuddy-bench"}[args.benchmark] - try: - if not dry: - for binary in ("git", "uv", "npm") + ( - ("docker", "curl") if args.benchmark == "wb" else () - ): - if not shutil.which(binary): - raise RuntimeError(f"Missing {binary}; see README.md") - install.clone("opencode", dry_run=dry) - install.patch("opencode", dry_run=dry) - install.run( - ["npm", "install", "--prefix", config.DEPENDENCIES / "tools", "bun@1.3.11"], dry_run=dry - ) - install.run( - [config.BUN, "install", "--frozen-lockfile", "--ignore-scripts"], - cwd=config.OPENCODE_DIR, - dry_run=dry, - ) - install.clone(name, dry_run=dry) - install.patch(name, dry_run=dry) - install.overlays(name, dry_run=dry) - repo = config.DEPENDENCIES / name - install.run(["uv", "sync", "--locked", "--python", "3.12"], cwd=repo, dry_run=dry) - if args.benchmark == "jb": - install.run( - ["uv", "pip", "install", "--python", repo / ".venv/bin/python", "python-docx"], - dry_run=dry, - ) - install.download_jobbench(dry_run=dry) - elif args.benchmark == "lab": - install.run( - [ - "uv", - "pip", - "install", - "--python", - repo / ".venv/bin/python", - "google-auth[requests]", - ], - dry_run=dry, - ) - else: - install.workbuddy_models(dry_run=dry) - subsets = [args.subset] if args.subset else ["office", "web", "code"] - install.run( - ["bash", "scripts/dataset/fetch-dataset.sh", *subsets], cwd=repo, dry_run=dry - ) - if args.substrate == "oc": - install.run( - [ - "uv", - "run", - "--no-sync", - "bash", - "scripts/harness/build-harness-mounts.sh", - "--harness", - "opencode/1.14.18", - ], - cwd=repo, - dry_run=dry, - ) - if args.substrate == "cx": - install.codex(dry_run=dry) - except (OSError, RuntimeError, ValueError, subprocess.CalledProcessError) as exc: - p.exit(1, f"setup failed: {exc}\n") - print( - "Setup plan complete." - if dry - else "Installation complete. Run harness2-check before an experiment." - ) - return 0 + p.add_argument("suite", help="suite name") + p.add_argument("args", nargs=argparse.REMAINDER, help="arguments for the installer") + return p -if __name__ == "__main__": - raise SystemExit(main()) +def run(args, ctx) -> int: + ctx.settings + suite = registry.load(args.suite) + if not hasattr(suite, "setup"): + raise ConfigError(f"suite {args.suite!r} has no setup command") + return suite.setup(args.args) diff --git a/harness2/cli/status.py b/harness2/cli/status.py new file mode 100644 index 00000000..5ccbbb27 --- /dev/null +++ b/harness2/cli/status.py @@ -0,0 +1,227 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Show a run's progress from its files on disk; `--watch` keeps the view refreshed. + +Reads run.json and events.jsonl only, so it works while the run is going, after it stopped, +and from another terminal. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +from . import _common as C + +_MODE = {"agg": "parallel", "seq": "sequential"} + + +def add_parser(sub) -> argparse.ArgumentParser: + p = sub.add_parser( + "status", + help="show a run's progress", + description="Show each task's state, the harness picked for it, and the run's spend. " + "NAME defaults to the most recent run.", + ) + p.add_argument("name", nargs="?", help="run name (default: the most recent run)") + p.add_argument( + "--watch", + nargs="?", + type=float, + const=5.0, + metavar="SECONDS", + help="redraw every SECONDS (default 5) until Ctrl-C", + ) + return p + + +def run(args, ctx) -> int: + run_dir = C.resolve_run(ctx, args.name) + if not args.watch: + snap = snapshot(run_dir) + ctx.out(snap, render) + return 0 + try: + while True: + text = render(snapshot(run_dir)) + sys.stdout.write("\033[H\033[2J" + text + "\n") + sys.stdout.flush() + time.sleep(max(args.watch, 0.5)) + except KeyboardInterrupt: + return 0 + + +def snapshot(run_dir: Path) -> dict: + """Everything the view shows, as data (this is also the --json output).""" + meta = C.load_run(run_dir) + events = _events(run_dir / "events.jsonl") + tasks: dict[str, dict] = {} + for domain, task in meta.get("stream", []): + tasks[task] = { + "task": task, + "area": domain, + "state": "pending", + "rollouts": 0, + "picked": {}, + } + spent = calls = 0.0 + ended = None + for ev in events: + kind, row = ev.get("kind"), tasks.get(ev.get("task", "")) + spent += float(ev.get("cost_usd") or 0) + if kind in ("session", "rollout") and not ev.get("cached"): + calls += 1 + if kind == "run_start": + ended = None + elif kind == "run_end": + ended = ev + if row is None: + continue + if kind == "task_start": + row["state"] = "running" + elif kind == "task_done": + row["state"] = "done" if ev.get("status") in ("ok", "skipped") else "failed" + elif kind == "rollout": + row["rollouts"] += 1 + elif kind == "selection" and ev.get("chosen"): + row["picked"][_MODE.get(ev.get("mode"), ev.get("mode"))] = ev["chosen"] + if ended is not None: + for row in tasks.values(): + if row["state"] == "running": + row["state"] = "stopped" + plan = meta.get("schedule", {}) + process = C.run_process(run_dir) + return { + "active": process is not None, + "pid": process["pid"] if process else None, + "run": meta.get("name", run_dir.name), + "suite": meta.get("suite", ""), + "executor": meta.get("executor", ""), + "k": plan.get("k"), + "modes": plan.get("modes", ""), + "created": meta.get("created", ""), + "spent_usd": round(spent, 4), + "calls": int(calls), + "ended": ended, + "tasks": list(tasks.values()), + "last_event": events[-1] if events else None, + } + + +def render(snap: dict) -> str: + rows = snap["tasks"] + head = f"{snap['run']} · {snap['suite']} suite · {snap['executor']}" + if snap["k"]: + head += f" · k={snap['k']} {snap['modes']}" + if snap.get("active"): + head += f" · running (pid {snap['pid']})" + last = snap["last_event"] + lines = [ + head, + f"started {_when(snap['created'])}" + + (f" · last activity {_ago(last.get('t'))}" if last else "") + + (f" · ${snap['spent_usd']:.2f} spent" if snap["spent_usd"] else " · spend not reported") + + f" · {snap['calls']} model calls", + "", + ] + if rows: + w = max(4, *(len(r["task"]) for r in rows)) + a = max(4, *(len(r["area"]) for r in rows)) + lines.append(f" {'TASK':<{w}} {'AREA':<{a}} {'STATE':<8} {'ROLLOUTS':>8} PICKED") + for r in rows: + picked = ", ".join(f"{m}: {c}" for m, c in r["picked"].items()) + lines.append( + f" {r['task']:<{w}} {r['area']:<{a}} {r['state']:<8} " + f"{r['rollouts'] or '':>8} {picked}" + ) + lines.append("") + states = ("done", "running", "stopped", "failed", "pending") + counts = [(sum(r["state"] == s for r in rows), s) for s in states] + lines.append(" · ".join(f"{n} {s}" for n, s in counts if n) or "no tasks") + end = snap["ended"] + if end: + status = end.get("status", "") + why = f" ({end['reason']})" if end.get("reason") else "" + tail = "" if status == "ok" else " · run the same command to resume" + lines.append(f"run {'finished' if status == 'ok' else 'ended: ' + status}{why}{tail}") + elif last: + lines.append(f"latest: {_describe(last)}") + if not snap.get("active"): + lines.append( + "not running: it stopped without finishing; run the same command to resume" + ) + return "\n".join(lines) + + +def _events(path: Path) -> list[dict]: + try: + text = path.read_text(encoding="utf-8") + except OSError: + return [] + out = [] + for line in text.splitlines(): + try: + ev = json.loads(line) + except json.JSONDecodeError: + continue # a line being written right now + if isinstance(ev, dict): + out.append(ev) + return out + + +def _describe(ev: dict) -> str: + kind = ev.get("kind", "") + where = f"{ev['task']} · " if ev.get("task") else "" + if kind == "rollout": + what = f"rollout {ev.get('label', '')} " + ( + "delivered" if ev.get("delivered") else "produced nothing" + ) + elif kind == "session": + what = f"{ev.get('label', 'session')} " + ("done" if ev.get("ok") else "failed") + elif kind == "selection": + what = f"picked {ev.get('chosen', '?')} ({_MODE.get(ev.get('mode'), ev.get('mode'))})" + else: + what = kind.replace("_", " ") + return f"{where}{what} · {_ago(ev.get('t'))}" + + +def _parse(ts: str | None) -> datetime | None: + try: + return datetime.fromisoformat(str(ts).replace("Z", "+00:00")) + except ValueError: + return None + + +def _when(ts: str | None) -> str: + t = _parse(ts) + return t.astimezone().strftime("%Y-%m-%d %H:%M") if t else "?" + + +def _ago(ts: str | None) -> str: + t = _parse(ts) + if t is None: + return "?" + s = int((datetime.now(timezone.utc) - t).total_seconds()) + if s < 60: + return f"{s}s ago" + if s < 3600: + return f"{s // 60}m ago" + if s < 86400: + return f"{s // 3600}h ago" + return f"{s // 86400}d ago" diff --git a/harness2/cli/stop.py b/harness2/cli/stop.py new file mode 100644 index 00000000..4490d0fa --- /dev/null +++ b/harness2/cli/stop.py @@ -0,0 +1,59 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Ask a run and its agent processes to stop.""" + +from __future__ import annotations + +import os +import signal + +from . import _common as C + + +def add_parser(subparsers): + p = subparsers.add_parser("stop", help="stop a running run; rerunning it resumes") + p.add_argument("name", nargs="?", help="run name (default: the most recent run)") + return p + + +def run(args, ctx) -> int: + root = C.resolve_run(ctx, args.name) + process = C.run_process(root) + if process is not None: + pid = process["pid"] + try: + # A detached run leads its own process group, so the agent CLIs it started stop + # with it. Otherwise the group can include whatever launched the run (a shell, an + # agent), so only the run itself is signalled. + if os.getpgid(pid) == pid: + os.killpg(pid, signal.SIGINT) + else: + os.kill(pid, signal.SIGINT) + except ProcessLookupError: + process = None + ctx.out( + { + "name": root.name, + "pid": process["pid"] if process else None, + "signalled": process is not None, + }, + lambda data: ( + f"Asked run {root.name} (pid {data['pid']}) to stop. It ends the calls in flight, " + "then exits; run the same command again to resume." + if data["signalled"] + else f"Run {root.name} is not running." + ), + ) + return 0 diff --git a/harness2/cli/task.py b/harness2/cli/task.py new file mode 100644 index 00000000..593395bd --- /dev/null +++ b/harness2/cli/task.py @@ -0,0 +1,83 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Manage task inputs for the built-in suite.""" + +from __future__ import annotations + +from pathlib import Path + + +def add_parser(subparsers): + p = subparsers.add_parser("task", help="create, list and import tasks") + commands = p.add_subparsers(dest="action", required=True) + new = commands.add_parser("new", help="create a task directory") + new.add_argument("id", help="task id: lowercase letters, digits, '.', '_' and '-'") + new.add_argument( + "--files", + action="store_true", + help="a files task: inputs in files/, results in output/ (default: a project task)", + ) + new.add_argument("--domain", default="default", help="domain the task belongs to") + new.add_argument("--check", help="shell command that grades a result") + commands.add_parser("ls", help="list tasks") + import_parser = commands.add_parser("import", help="create tasks from a JSONL or CSV file") + import_parser.add_argument("file", type=Path, help="the .jsonl or .csv file") + import_parser.add_argument( + "--force", action="store_true", help="overwrite tasks that already exist" + ) + return p + + +def run(args, ctx) -> int: + ctx.project + from .. import suite + + tasks_dir = ctx.settings.path("paths.tasks") + if args.action == "new": + result = suite.new_task( + tasks_dir, args.id, files=args.files, domain=args.domain, check=args.check + ) + ctx.out( + {"path": result}, + lambda data: ( + f"Created {data['path']}. Write the request in its task.md" + + (" and put the input files in files/." if args.files else ".") + ), + ) + elif args.action == "ls": + ctx.out( + suite.list_tasks(tasks_dir), + _table, + ) + else: + ctx.out( + suite.import_tasks(tasks_dir, args.file, force=args.force), lambda rows: "\n".join(rows) + ) + return 0 + + +def _table(rows: list[dict]) -> str: + if not rows: + return "No tasks yet; create one with `harness2 task new `." + head = ("ID", "DOMAIN", "SPLIT", "WORKSPACE", "GRADED") + body = [ + (r["id"], r["domain"], r["split"] or "-", r["workspace"], "yes" if r["has_judge"] else "no") + for r in rows + ] + widths = [max(len(str(row[i])) for row in (head, *body)) for i in range(len(head))] + return "\n".join( + " ".join(str(cell).ljust(w) for cell, w in zip(row, widths)).rstrip() + for row in (head, *body) + ) diff --git a/harness2/core/__init__.py b/harness2/core/__init__.py index d59d80d5..2138c7af 100644 --- a/harness2/core/__init__.py +++ b/harness2/core/__init__.py @@ -17,3 +17,28 @@ Per-task harness adaptation over a c1..c8 component tree, two recursion modes at an identical budget, blind answer-free selection, and a per-domain experience library. """ + +from __future__ import annotations + +import threading + + +def configure(*, phase_width=None, transport_attempts=None, proposer_concurrency=None) -> None: + """Apply runtime scheduling settings before starting work.""" + from . import proposer, recursion, schedule + + values = { + "phase_width": phase_width, + "transport_attempts": transport_attempts, + "proposer_concurrency": proposer_concurrency, + } + for name, value in values.items(): + if value is not None and (type(value) is not int or value < 1): + raise ValueError(f"{name} must be a positive integer") + if phase_width is not None: + recursion.PHASE_WIDTH = schedule.ROLLOUT_PARALLEL = phase_width + if transport_attempts is not None: + recursion.TRANSPORT_ATTEMPTS = transport_attempts + if proposer_concurrency is not None: + proposer.MAX_CONCURRENCY = proposer_concurrency + proposer._SEM = threading.BoundedSemaphore(proposer_concurrency) diff --git a/harness2/core/events.py b/harness2/core/events.py new file mode 100644 index 00000000..0188925f --- /dev/null +++ b/harness2/core/events.py @@ -0,0 +1,112 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Thread-safe runtime events and resumable spending limits.""" + +from __future__ import annotations + +import json +import threading +from datetime import datetime, timezone +from pathlib import Path + + +class Stopped(RuntimeError): + """Execution stopped at a user request or a spending limit.""" + + +_lock = threading.Lock() +_active = False +_path: Path | None = None +_spent_usd = 0.0 +_calls = 0 +_budget_usd = 0.0 +_max_calls = 0 +_stopped: str | None = None + + +def _accumulate(record: dict) -> None: + global _spent_usd, _calls + _spent_usd += record.get("cost_usd", 0) or 0 + if record["kind"] == "rollout" or (record["kind"] == "session" and not record.get("cached")): + _calls += 1 + + +def start(path: Path | None, *, budget_usd: float = 0.0, max_calls: int = 0) -> None: + """Reset limits and restore accounting from a previous event stream.""" + if budget_usd < 0 or max_calls < 0: + raise ValueError("budget_usd and max_calls must be non-negative") + global _active, _path, _spent_usd, _calls, _budget_usd, _max_calls, _stopped + with _lock: + _active = True + _path = Path(path) if path is not None else None + _spent_usd, _calls = 0.0, 0 + _budget_usd, _max_calls, _stopped = budget_usd, max_calls, None + if _path is not None: + _path.parent.mkdir(parents=True, exist_ok=True) + if _path.is_file(): + for line in _path.read_text(encoding="utf-8").splitlines(): + if line.strip(): + _accumulate(json.loads(line)) + + +def emit(kind: str, **fields) -> None: + """Append one event and count completed work, under the same lock.""" + with _lock: + if not _active: + return + record = { + "t": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "kind": kind, + **fields, + } + if _path is not None: + with _path.open("a", encoding="utf-8") as stream: + stream.write(json.dumps(record) + "\n") + _accumulate(record) + + +def _reason() -> str | None: + if _stopped: + return _stopped + if _active and _budget_usd and _spent_usd >= _budget_usd: + return "budget_usd" + if _active and _max_calls and _calls >= _max_calls: + return "max_calls" + return None + + +def check() -> None: + """Raise before starting more work when a limit or stop request applies.""" + with _lock: + reason = _reason() + if reason is not None: + raise Stopped(reason) + + +def request_stop(reason: str) -> None: + global _stopped + with _lock: + _stopped = reason + + +def totals() -> dict: + with _lock: + return { + "spent_usd": round(_spent_usd, 6), + "calls": _calls, + "budget_usd": _budget_usd, + "max_calls": _max_calls, + "stopped": _reason(), + } diff --git a/harness2/core/judge.py b/harness2/core/judge.py index cf4e7b9f..d28cc857 100644 --- a/harness2/core/judge.py +++ b/harness2/core/judge.py @@ -26,6 +26,7 @@ from pathlib import Path from typing import Sequence +from . import runmark from .spec import BenchmarkAdapter @@ -120,7 +121,7 @@ def judge_selections( sel["k"], sel["chosen_label"], sel.get("chosen_run_id", ""), - Path(rd), + runmark.resolve_path(rd, adapter.spec.runs_root), chosen_is_base=bool(sel.get("chosen_is_base")), ) ] = sel diff --git a/harness2/core/prompts.py b/harness2/core/prompts.py index 9ccacf23..d553df72 100644 --- a/harness2/core/prompts.py +++ b/harness2/core/prompts.py @@ -22,7 +22,7 @@ from __future__ import annotations import re -from dataclasses import dataclass +from dataclasses import dataclass, replace from .spec import BenchPrompts from .tree import SurfaceSpec @@ -204,7 +204,7 @@ def _tidy(text: str) -> str: | `.agents/skills//SKILL.md` | a reusable workflow | name+description advertised; the BODY is read ON DEMAND | | `.agents/skills//scripts/*` | a helper carried with its skill | run via bash | | `sub_agents//AGENT.md` | a delegated agent role | spawned in its OWN context, returns text | -| `scripts/.py` | a deterministic helper | run on demand via bash | +| `.agents/scripts/.py` | a deterministic helper | run on demand via bash | | `.codex/hooks.json` | lifecycle hooks | fire INSIDE the tool loop, on every matching tool call | The three that are easy to get wrong: @@ -229,7 +229,8 @@ def _tidy(text: str) -> str: "reason" sends that reason back to the model; `PreToolUse` can deny a call the same way. The only component that acts in the tool loop rather than in the prompt -- the only way to enforce something the model cannot skip. Prose can be stopped reading; a hook cannot. -Keep commands fast and non-interactive; put real logic in a `scripts/` helper the hook calls. +Keep commands fast and non-interactive; put real logic in a `.agents/scripts/` helper the hook +calls. Change only what a trajectory shows needs changing -- but WHERE you change something, the FORM matters as much as the content, and the cheap form is not the obvious one. @@ -352,10 +353,9 @@ def _lens(bp: BenchPrompts) -> str: - TOOL ERROR: distinguish bad input from timeout, missing capability, permission, or unread output. Give one bounded fallback and stop condition; never prescribe blind repetition. -- MECHANICAL REPETITION: use `scripts/.py` for repeated deterministic extraction, +- MECHANICAL REPETITION: use __SCRIPT_LEVER__ for repeated deterministic extraction, conversion or checking, with clear inputs, outputs and failure behavior. -- TRIGGERED WORKFLOW: use an on-demand skill for a narrow multi-step procedure, name its trigger - in always-on text, and carry any helper it requires. +- TRIGGERED WORKFLOW: __SKILL_LEVER__ - IGNORED INVARIANT: only repeated evidence that clear prose is skipped can justify a defensive __LOOP_LEVER__. It must not block valid completion. - ORDINARY SEQUENCING: prefer concise always-on guidance. Do not add code for a one-off mistake @@ -476,7 +476,7 @@ def _surface_part(surface_md: str, start: str, end: str | None = None) -> str: | `.agents/skills//SKILL.md` | c6 reusable workflow | name+description advertised; the BODY is read ON DEMAND | | `.agents/skills//scripts/*` | helper carried with its skill | run via Bash | | `sub_agents//AGENT.md` | c5 delegated agent role | spawned in its OWN context, returns text | -| `scripts/.py` | c7 deterministic helper | run ON DEMAND via Bash as `python scripts/.py` | +| `.agents/scripts/.py` | c7 deterministic helper | run ON DEMAND via Bash as `python .agents/scripts/.py` | | `.codex/hooks.json` | c8 lifecycle hooks | fire INSIDE the tool loop, on every matching tool call | ## The two on-demand levers that are easy to get wrong @@ -494,8 +494,8 @@ def _surface_part(surface_md: str, start: str, end: str | None = None) -> str: the event as JSON on stdin; a `PostToolUse` command printing a JSON "decision" of "block" with a "reason" sends that reason back to the model. Unlike everything else here it acts in the tool loop rather than in the prompt, which is the only way to enforce something the model cannot -skip. Keep commands fast and non-interactive; put real logic in a `scripts/` helper the hook -calls. +skip. Keep commands fast and non-interactive; put real logic in a `.agents/scripts/` helper +the hook calls. Both levers are rare in the harnesses adopted so far. Read that as a gap in what has been tried, not as a verdict -- but still add one only where a trajectory shows the failure it would fix. @@ -508,7 +508,7 @@ def _surface_part(surface_md: str, start: str, end: str | None = None) -> str: one is a harness that does not exist. - A skill's literal directory name must appear in `AGENTS.md` as a trigger, or the skill is never opened. Scripts run via Bash. -- `scripts/*.py` filenames must be snake_case and must compile under py_compile. +- `.agents/scripts/*.py` filenames must be snake_case and must compile under py_compile. - Never write `harness.yaml` or other runtime-owned files: they are not on the editable surface.""" @@ -516,7 +516,7 @@ def _surface_part(surface_md: str, start: str, end: str | None = None) -> str: _TTS_INV = """\ # YOUR DIMENSION: INVARIANT -- how the agent works, independent of the subject -You own the machinery: tool guidance, `scripts/*.py`, `sub_agents/*/AGENT.md`, the tool-loop +You own the machinery: tool guidance, `__SCRIPT_DIR__/*.py`, `sub_agents/*/AGENT.md`, the tool-loop lever, and SHORT imperative always-on rules that hold for essentially EVERY task in this {domain} (output discipline, verification order, format mechanics, budget and finalization). @@ -526,7 +526,7 @@ def _surface_part(surface_md: str, start: str, end: str | None = None) -> str: Pick the cheapest mechanism that fixes what the trajectory shows: - the model fumbled a format or a library repeatedly -> a tool-guidance note, or a small - deterministic `scripts/*.py` helper it can call instead of improvising + deterministic `__SCRIPT_DIR__/*.py` helper it can call instead of improvising - a required step is skipped under time pressure -> ONE always-on rule, stated in an imperative line (every token here competes with the {deliverable} -- if you add, cut) - a bounded sub-task drowned the context and returned @@ -604,6 +604,132 @@ def _surface_part(surface_md: str, start: str, end: str | None = None) -> str: context for nothing.""" +_SURFACE_CLAUDE_HEAD = """\ +# WHAT YOU CAN CHANGE + +The harness is a tree of components. Each has a different runtime cost and a different reach: + +| path | what it is | when it is in context | +|---|---|---| +| `CLAUDE.md` | role, procedure, rules, durable facts | ALWAYS ON -- loaded whole into every session; every token competes with the agent's own work | +| `.claude/skills//SKILL.md` | a reusable workflow | name+description always listed; the BODY is read ON DEMAND when the agent invokes the skill | +| `.claude/skills//scripts/*` | a helper carried with its skill | run via Bash | +| `sub_agents//AGENT.md` | a delegated subagent, installed as `.claude/agents/.md` | runs in its OWN context, returns text | +| `.claude/scripts/.py` | a deterministic helper | run on demand via Bash | +| `.claude/hooks.json` | lifecycle hooks, installed into `.claude/settings.json` | fire INSIDE the tool loop, on every matching event | + +The three that are easy to get wrong: + +`.claude/skills/` -- YAML frontmatter with a `description:` is REQUIRED; a `name:` line is +optional and must match the directory. Only the name and description are shown up front, and +the agent decides from the description alone whether to open the skill -- so the description IS +the trigger: name the situation the skill handles, precisely and narrowly. If a skill calls a +helper, the helper must travel with it. + +`sub_agents/` -- frontmatter needs a `description:` saying when to delegate; an optional +`tools:` line (comma-separated tool names) narrows what it may use. It sees only what its own +file and the delegation message say, never the main conversation, and returns text. Worth it +for exactly one shape of problem: a bounded sub-task whose INPUT is large and whose ANSWER is +small. A bad trade when the output is as big as the input. + +`.claude/hooks.json` -- ONE JSON file: a top-level "hooks" object maps an event -- +`PreToolUse`, `PostToolUse`, `Stop`, `SessionStart` -- to matcher entries (a "matcher" such as +"Bash" or "Edit|Write", plus a "hooks" list of "command"-type entries). Each hook is a shell +command receiving the event as JSON on stdin (`tool_name`, `tool_input`; PostToolUse adds +`tool_response`). A command that exits with status 2 blocks the step and its stderr is shown to +the model: a `PreToolUse` hook refuses the call, a `PostToolUse` hook reports a problem with +what just happened, a `Stop` hook sends the agent back to work. The only component that acts in +the tool loop rather than in the prompt -- the only way to enforce something the model cannot +skip. Prose can be stopped reading; a hook cannot. Keep commands fast and non-interactive; put +real logic in a `.claude/scripts/` helper the hook calls.""" + + +_SURFACE_CLAUDE_PREFER = """\ +So prefer, in this order: + - a SKILL for anything with a recognisable trigger -- a procedure for one kind of change or + document, a recurring situation, a tool the work keeps needing. This is the default form + for new instruction. + - a SCRIPT for a deterministic operation the runs hand-rolled and got wrong, or did slowly. + - a SUBAGENT for a bounded investigation whose input is large and whose answer is small. + - a HOOK when a recurring, mechanical problem in the tool loop is better handled there than + in prose. It acts inside the loop, so it cannot be ignored AND it costs nothing in always-on + budget -- which makes it the cheap way to carry knowledge that prose would have to repeat. + The recurring shapes: running a fast check after an edit and reporting what broke; blocking + a known-bad command with a reason that teaches the fix; and refusing to stop while a + required step is still missing. + - ALWAYS-ON TEXT last, and only for a short rule that genuinely applies to every run. If you + add always-on text, cut something else to pay for it. + +EVERY SCRIPT YOU NAME MUST EXIST. A `.claude/scripts/.py` invoked from `CLAUDE.md`, a +skill or a hook must ship in the same proposal, and a skill's own helper goes at +`.claude/skills//scripts/.py`. A helper placed directly in the skill +directory is OFF THE EDITABLE SURFACE and silently dropped, leaving the skill invoking a file +that does not exist. Never write `.claude/settings.json` or `.claude/agents/` directly: the +runner builds them from `.claude/hooks.json` and `sub_agents/`.""" + + +_TTS_SURFACE_HEAD_CLAUDE = """\ +# EDITABLE HARNESS SURFACE + +You may write ONLY these paths. Anything else is off the editable surface: it is dropped before +your work is used, however good it is. + +| path | component | when it is in context | +|---|---|---| +| `CLAUDE.md` | c1/c3/c4/c6 persona, rules, durable facts, tool prose | ALWAYS ON -- loaded whole into every session | +| `.claude/skills//SKILL.md` | c6 reusable workflow | name+description listed; the BODY is read ON DEMAND | +| `.claude/skills//scripts/*` | helper carried with its skill | run via Bash | +| `sub_agents//AGENT.md` | c5 delegated subagent | run ON DEMAND in its OWN context, returns text | +| `.claude/scripts/.py` | c7 deterministic helper | run ON DEMAND via Bash as `python3 .claude/scripts/.py` | +| `.claude/hooks.json` | c8 lifecycle hooks | fire INSIDE the tool loop, on every matching event | + +## The two on-demand levers that are easy to get wrong + +`sub_agents//AGENT.md` -- frontmatter needs a `description:` saying when to delegate. It +runs in a SEPARATE context: it sees only what its own file and the delegation message say, +never the main conversation, and it returns text. Worth it for exactly one shape of problem -- +a bounded sub-task whose inputs are large and whose ANSWER is small. A bad trade when the +output is as big as the input. + +`.claude/hooks.json` -- ONE JSON file: a top-level "hooks" object maps an event (`PreToolUse`, +`PostToolUse`, `Stop`, `SessionStart`) to matcher entries -- a "matcher" such as "Bash" plus a +"hooks" list of "command"-type entries. Each hook is a shell command receiving the event as +JSON on stdin; one that exits with status 2 blocks the step and shows its stderr to the model. +Unlike everything else here it acts in the tool loop rather than in the prompt, which is the +only way to enforce something the model cannot skip. Keep commands fast and non-interactive; +put real logic in a `.claude/scripts/` helper the hook calls. + +Both levers are rare in the harnesses adopted so far. Read that as a gap in what has been tried, +not as a verdict -- but still add one only where a trajectory shows the failure it would fix. + +Hard rules, each one learned from a previous failure: +- The single most common failure of this harness family is producing NO deliverable because the + model exhausted its budget on reasoning. Always-on prose makes that WORSE. Prefer an + on-demand skill over new always-on prose; if you add always-on text, cut something else -- + `CLAUDE.md` is capped. +- A skill's description is its trigger: the agent opens a skill only when the description + matches the situation in front of it. Scripts run via Bash. +- `.claude/scripts/*.py` filenames must be snake_case and must compile under py_compile.""" + + +_SURFACE_COMMAND_HEAD = """\ +# WHAT YOU CAN CHANGE + +The harness is the set of files in `context/harness/`: whatever the agent being tuned reads at +run time -- prompt text, configuration, helper code. These path patterns are the whole editable +surface: + +__EDITABLE__ + +A file you create counts only if its path matches one of those patterns; anything else is +dropped before your work is used, however good it is. Where other parts of these instructions +name skills, scripts, subagents or hooks, those levers exist here only as files the patterns +allow. + +Change only what a trajectory shows needs changing. Text the agent reads on every run is paid +for on every run, competing with the agent's own work: if you add some, cut something else.""" + + @dataclass(frozen=True) class _SurfaceText: """The per-substrate prompt fragments, resolved once at module load.""" @@ -612,6 +738,20 @@ class _SurfaceText: inv: str dom: str tts_surface: str + script_dir: str = "scripts" + + +def _levers(template: str, **values: str) -> str: + """Fill the `__NAME__` slots of a shared INV/DOM template for one surface.""" + for name, value in values.items(): + template = template.replace(f"__{name.upper()}__", value) + return template + + +_SKILL_TRIGGER_LEVER = ( + "use an on-demand skill for a narrow multi-step procedure, name its trigger\n" + " in always-on text, and carry any helper it requires." +) def _build_surface_texts() -> dict[str, _SurfaceText]: @@ -629,34 +769,91 @@ def _build_surface_texts() -> dict[str, _SurfaceText]: _surface_part(_SURFACE_CODEX, "# HOW TO MAKE A CHANGE"), ] ) + shape_and_how = _surface_part(_SURFACE_CODEX, "THE DELIVERABLE'S SHAPE") + claude_md = "\n\n".join( + [ + _SURFACE_CLAUDE_HEAD, + _surface_part(_SURFACE_CODEX, "Change only what a trajectory", "So prefer, in this"), + _SURFACE_CLAUDE_PREFER, + shape_and_how, + ] + ) + claude_tts = "\n\n".join( + [ + _TTS_SURFACE_HEAD_CLAUDE, + _surface_part(claude_md, "EVERY SCRIPT YOU NAME", "THE DELIVERABLE'S SHAPE"), + _surface_part(claude_md, "# HOW TO MAKE A CHANGE"), + ] + ) + command_md = "\n\n".join([_SURFACE_COMMAND_HEAD, shape_and_how]) return { "opencode": _SurfaceText( surface_md=_SURFACE_OPENCODE, - inv=_INV_TEMPLATE.replace( - "__LOOP_LEVER__", - "`plugin/.ts` hook -- for example, preventing exit before a " + inv=_levers( + _INV_TEMPLATE, + script_lever="`scripts/.py`", + skill_lever=_SKILL_TRIGGER_LEVER, + loop_lever="`plugin/.ts` hook -- for example, preventing exit before a " "{deliverable} exists", ), - dom=_DOM_TEMPLATE.replace( - "__TRIGGER_SENTENCE__", - "A skill exists only once its name appears in the always-on text: write that " - "trigger line in the same edit.", + dom=_levers( + _DOM_TEMPLATE, + trigger_sentence="A skill exists only once its name appears in the always-on " + "text: write that trigger line in the same edit.", ), tts_surface=oc_tts, ), "codex": _SurfaceText( surface_md=_SURFACE_CODEX, - inv=_INV_TEMPLATE.replace( - "__LOOP_LEVER__", - "hook in `.codex/hooks.json` -- for example, blocking a finish before a " - "{deliverable} exists, with a corrective reason", + inv=_levers( + _INV_TEMPLATE, + script_lever="`.agents/scripts/.py`", + skill_lever=_SKILL_TRIGGER_LEVER, + loop_lever="hook in `.codex/hooks.json` -- for example, blocking a finish before " + "a {deliverable} exists, with a corrective reason", ), - dom=_DOM_TEMPLATE.replace( - "__TRIGGER_SENTENCE__", - "A skill exists only once its name appears in `AGENTS.md` as a trigger: write " - "that trigger line in the same edit.", + dom=_levers( + _DOM_TEMPLATE, + trigger_sentence="A skill exists only once its name appears in `AGENTS.md` as a " + "trigger: write that trigger line in the same edit.", ), tts_surface=cx_tts, + script_dir=".agents/scripts", + ), + "claude": _SurfaceText( + surface_md=claude_md, + inv=_levers( + _INV_TEMPLATE, + script_lever="`.claude/scripts/.py`", + skill_lever="use an on-demand skill for a narrow multi-step procedure, write its\n" + " description as its trigger, and carry any helper it requires.", + loop_lever="hook in `.claude/hooks.json` -- for example, a `Stop` hook that " + "refuses to finish before a {deliverable} exists, with a corrective reason", + ), + dom=_levers( + _DOM_TEMPLATE, + trigger_sentence="The agent finds a skill by its description alone: write the " + "description as the situation the skill handles.", + ), + tts_surface=claude_tts, + script_dir=".claude/scripts", + ), + "command": _SurfaceText( + surface_md=command_md, + inv=_levers( + _INV_TEMPLATE, + script_lever="a helper file, where the editable patterns allow one,", + skill_lever="keep a narrow multi-step procedure in a file the agent opens\n" + " only when it applies, if the patterns allow one, and say in always-on text " + "when to open it.", + loop_lever="check in whatever runtime configuration the editable patterns include", + ), + dom=_levers( + _DOM_TEMPLATE, + trigger_sentence="Knowledge in a file the agent opens on demand is read only " + "when always-on text says when to open it: write that line in the same edit.", + ), + tts_surface=command_md, ), } @@ -669,12 +866,21 @@ def _st(surface: SurfaceSpec | str) -> _SurfaceText: got = _SURFACE_TEXTS.get(name) if got is None: raise ValueError(f"no prompt surface for {name!r}; known: {sorted(_SURFACE_TEXTS)}") + globs = getattr(surface, "editable_globs", ()) + if name == "command" and globs: + listing = "\n".join(f"- `{g}`" for g in globs) + got = replace( + got, + surface_md=got.surface_md.replace("__EDITABLE__", listing), + tts_surface=got.tts_surface.replace("__EDITABLE__", listing), + ) return got def _tts_propose(bp: BenchPrompts, dim: str, st: _SurfaceText, *, k: int, index: int) -> str: """propose(), in the tts pack's words.""" - block = (_TTS_INV if dim == "INV" else _TTS_DOM).format( + template = (_TTS_INV if dim == "INV" else _TTS_DOM).replace("__SCRIPT_DIR__", st.script_dir) + block = template.format( deliverable=bp.deliverable_word, domain=bp.domain_word, levers=(bp.inv_levers if dim == "INV" else bp.dom_levers).strip(), @@ -1161,3 +1367,40 @@ def experience(bp: BenchPrompts) -> str: ] ) ) + + +CONSOLIDATE_MISSION = """\ +You are consolidating an autonomous agent's harness for the project it works on. + +The agent has already been run on {n} task(s) from this project. For each task, its harness +was refined on evidence from that task's own runs, and the best result was adopted. Those +adopted harnesses are task-specific: each was shaped for one request. Your job is to compose +ONE harness for the project's next tasks -- tasks nobody has seen yet. + +# YOUR EVIDENCE + +context/harness/ the harness the runs started from -- edit THIS one +context/tasks//task.md the request each task made +context/tasks//harness/ the harness adopted for that task +context/tasks//EXPERIENCE.md which edits made a difference and the lesson drawn, when + one was recorded +context/INDEX.md the tasks, their domains, and which adopted harnesses + differ from the start + +Read INDEX.md, then compare each adopted harness with context/harness/ and read the lessons. + +# WHAT TO KEEP + +- A component or rule that helped on several tasks, or that a lesson supports as general, + belongs in the consolidated harness. +- A workspace fact tied to one task -- a file it touched, a value it computed, a mapping it + verified -- does not. Keep the procedure that found it, if that procedure generalizes. +- Where two tasks' edits conflict, keep the one with the stronger evidence, or neither. +- When the adopted harnesses mostly left the start unchanged, so should you: an edit no task + supported costs context on every future run and buys nothing. +- Knowledge that applies only in some situations belongs in an on-demand component, not in + always-on text. Keep always-on text short. + +{surface} + +In your ``, name for each change the tasks and lessons that support it.""" diff --git a/harness2/core/proposer.py b/harness2/core/proposer.py index 4a235075..5c2d23a4 100644 --- a/harness2/core/proposer.py +++ b/harness2/core/proposer.py @@ -25,6 +25,7 @@ import json import os import re +import shlex import shutil import subprocess import tempfile @@ -33,23 +34,16 @@ from dataclasses import dataclass, field from pathlib import Path -from .. import config +from .. import providers +from ..substrate.codex import CodexSubstrate, parse_items_text +from . import events from .spec import ChannelSpec SYSTEM_ARG_MAX = 120_000 -MAX_CONCURRENCY = int(config.PROPOSER_CONCURRENCY) -_SEM = threading.BoundedSemaphore(MAX_CONCURRENCY) if MAX_CONCURRENCY > 0 else None - - -_DROP_ENV = ( - "ANTHROPIC_API_KEY", - "ANTHROPIC_AUTH_TOKEN", - "ANTHROPIC_BASE_URL", - "ANTHROPIC_BEDROCK_BASE_URL", - "CLAUDE_CODE_USE_BEDROCK", -) +MAX_CONCURRENCY = 2 +_SEM = threading.BoundedSemaphore(MAX_CONCURRENCY) def _obj(d: dict, key: str) -> dict: @@ -94,35 +88,39 @@ class Proposer: spec: ChannelSpec audit_dir: Path | None = None attempts: int = 2 - project: str = field(default_factory=lambda: config.VERTEX_PROJECT) - region: str = field(default_factory=lambda: config.VERTEX_REGION) _calls: list[Reply] = field(default_factory=list) def __post_init__(self) -> None: - if not self.spec.model.strip(): - raise ValueError("Set HARNESS2_IMPROVER_MODEL before starting model sessions") + if self.spec.kind != "human" and not self.spec.model.strip(): + raise ValueError("model must not be empty for model sessions") def env(self) -> dict[str, str]: - e = {k: v for k, v in os.environ.items() if k not in _DROP_ENV} - if self.spec.kind == "claude_cli": - e.update( - { - "CLAUDE_CODE_USE_VERTEX": "1", - "ANTHROPIC_VERTEX_PROJECT_ID": self.project, - "CLOUD_ML_REGION": self.region, - "ANTHROPIC_MODEL": self.spec.model, - } - ) - else: - e.update( - { - "GOOGLE_VERTEX_PROJECT": self.project, - "GOOGLE_VERTEX_LOCATION": self.region, - "VERTEXAI_LOCATION": self.region, - } - ) + e = {k: v for k, v in os.environ.items() if k not in providers.STRIP_ENV} + if self.spec.provider is not None: + if self.spec.kind == "claude": + e.update(providers.claude_env(self.spec.provider)) + elif self.spec.kind == "opencode": + e.update(providers.opencode_config(self.spec.provider, self._model())[1]) return e + def _model(self) -> str: + return self.spec.model.split("/", 1)[-1] + + def _bin(self) -> list[str]: + return shlex.split(self.spec.bin) if self.spec.bin else [self.spec.kind] + + def _emit(self, reply: Reply) -> None: + events.emit( + "session", + label=reply.label, + channel=self.spec.kind, + model=self.spec.model, + ok=reply.ok, + cached=reply.cached, + cost_usd=reply.cost_usd, + wall_s=reply.wall_s, + ) + def call( self, mission: str, @@ -140,10 +138,13 @@ def call( directory is destroyed -- e.g. "context/harness", which is how an edited tree comes back. """ - cache_key = _call_fingerprint(label, mission, system, context_dir) + if self.spec.kind == "human": + raise ProposerError("the human channel needs an interactive selector") + cache_key = _call_fingerprint(label, mission, system, context_dir, self.audit_dir) cached = self._cached(label, cache_key, require_workspace=bool(harvest)) if cached is not None: self._calls.append(cached) + self._emit(cached) return cached wall = int(timeout or self.spec.timeout) @@ -154,17 +155,21 @@ def call( for attempt in range(1, tries + 1): if attempt > 1: time.sleep(10 * (attempt - 1)) - tmp = Path(tempfile.mkdtemp(prefix="harness2-prop-")) + tmp = Path(tempfile.mkdtemp(prefix="harness2-prop-")).resolve() try: if context_dir and Path(context_dir).is_dir(): shutil.copytree(context_dir, tmp / "context", dirs_exist_ok=True) with _SEM if _SEM is not None else contextlib.nullcontext(): - text, cost, err = ( - self._claude_cli if self.spec.kind == "claude_cli" else self._opencode - )(tmp, mission, system, wall) + events.check() + text, cost, err = getattr(self, f"_{self.spec.kind}")( + tmp, mission, system, wall + ) spent_usd += cost + if err and providers.USAGE_LIMIT.search(err): + events.request_stop("usage_limit") + raise events.Stopped("usage_limit") if not text.strip(): - last = err or "empty reply" + last = err[-300:] or "empty reply" continue r = Reply( label=label, @@ -181,13 +186,19 @@ def call( _drop_workspace(self._paths(label, cache_key)) self._write_audit(r, mission, system, cache_key) self._calls.append(r) + self._emit(r) return r + except events.Stopped: + raise except subprocess.TimeoutExpired: last = f"timeout after {wall}s" except Exception as exc: last = f"{type(exc).__name__}: {exc}" finally: shutil.rmtree(tmp, ignore_errors=True) + if providers.USAGE_LIMIT.search(last): + events.request_stop("usage_limit") + raise events.Stopped("usage_limit") r = Reply( label=label, @@ -201,24 +212,33 @@ def call( ) self._calls.append(r) self._write_audit(r, mission, system, cache_key) + self._emit(r) raise ProposerError( f"[{label}] {self.spec.kind}/{self.spec.model}: {last}", attempts=tries, cost_usd=spent_usd, ) - def _claude_cli( - self, cwd: Path, mission: str, system: str, wall: int - ) -> tuple[str, float, str]: - cli = shutil.which("claude") or "claude" + def _claude(self, cwd: Path, mission: str, system: str, wall: int) -> tuple[str, float, str]: + login = self.spec.provider is not None and self.spec.provider.type == "claude-login" + env = self.env() + if not login: + home = cwd / ".claude-home" + home.mkdir() + env["CLAUDE_CONFIG_DIR"] = str(home) cmd = [ - cli, + *self._bin(), "--dangerously-skip-permissions", "-p", "--output-format", "json", "--effort", self.spec.effort, + "--model", + self._model(), + "--setting-sources", + "project,local", + *(["--strict-mcp-config", "--no-session-persistence"] if login else []), ] stdin = mission if system: @@ -234,7 +254,7 @@ def _claude_cli( cmd, input=stdin, cwd=cwd, - env=self.env(), + env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -243,13 +263,61 @@ def _claude_cli( payload = _parse_json(p.stdout) text = str(payload.get("result") or payload.get("text") or "") err = "" - if payload.get("is_error") or payload.get("subtype") not in (None, "success"): - err = f"CLI error: {payload.get('subtype')} {str(text)[:200]}" + if ( + p.returncode + or payload.get("is_error") + or payload.get("subtype") not in (None, "success") + ): + err = f"rc={p.returncode}: {p.stdout} {p.stderr}" text = "" - elif p.returncode != 0 and not text.strip(): - err = f"rc={p.returncode}: {(p.stderr or '')[-300:]}" return text, float(payload.get("total_cost_usd") or 0.0), err + def _codex(self, cwd: Path, mission: str, system: str, wall: int) -> tuple[str, float, str]: + model = self._model() + home = None + if self.spec.provider is None or self.spec.provider.type != "chatgpt-login": + home = cwd / ".codex-home" + home.mkdir() + config = ( + providers.codex_config(self.spec.provider, model, self.spec.effort) + if self.spec.provider is not None + else f"model = {json.dumps(model)}\n" + f"model_reasoning_effort = {json.dumps(self.spec.effort)}\n" + ) + (home / "config.toml").write_text(config, encoding="utf-8") + p = subprocess.run( + [ + *self._bin(), + "exec", + "--json", + "--skip-git-repo-check", + "--dangerously-bypass-approvals-and-sandbox", + "--ephemeral", + *providers.codex_args(self.spec.provider, model, self.spec.effort), + "-C", + str(cwd), + "-", + ], + input=f"# SYSTEM INSTRUCTIONS\n\n{system}\n\n# END\n\n{mission}" if system else mission, + cwd=cwd, + env=CodexSubstrate().env(self.env(), home, provider=self.spec.provider), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=wall, + ) + text, errors = "", [] + for event in parse_items_text(p.stdout): + if event.get("type") in ("turn.failed", "error"): + error = event.get("error") or event.get("message") or event["type"] + errors.append(str(error)) + item = _obj(event, "item") + if item.get("type") == "agent_message": + text = str(item.get("text") or "") + if errors or p.returncode: + return "", 0.0, f"rc={p.returncode}: {p.stdout} {p.stderr}" + return text, 0.0, "" + def _opencode(self, cwd: Path, mission: str, system: str, wall: int) -> tuple[str, float, str]: (cwd / "MISSION.md").write_text( (f"{system}\n\n---\n\n" if system else "") + mission, encoding="utf-8" @@ -257,14 +325,14 @@ def _opencode(self, cwd: Path, mission: str, system: str, wall: int) -> tuple[st out = cwd / "output" out.mkdir(exist_ok=True) - (cwd / "opencode.json").write_text( - json.dumps({"agent": {"build": {"steps": self.spec.steps}}}), encoding="utf-8" - ) + config = {"agent": {"build": {"steps": self.spec.steps}}} + if self.spec.provider is not None: + config.update(providers.opencode_config(self.spec.provider, self._model())[0]) + (cwd / "opencode.json").write_text(json.dumps(config), encoding="utf-8") prompt = ( "Read MISSION.md first. Do what it asks. Reply with the requested output as " "your final message AND save it verbatim to output/answer.md." ) - oc = _opencode_dir() env = self.env() env["OPENCODE_PERMISSION"] = json.dumps( @@ -274,12 +342,7 @@ def _opencode(self, cwd: Path, mission: str, system: str, wall: int) -> tuple[st env["XDG_STATE_HOME"] = str(cwd / ".ocstate") cmd = [ - str(config.BUN), - "run", - "--cwd", - str(oc / "packages" / "opencode"), - "--conditions=browser", - "src/index.ts", + *self._bin(), "run", prompt, "--dir", @@ -301,10 +364,12 @@ def _opencode(self, cwd: Path, mission: str, system: str, wall: int) -> tuple[st timeout=wall, ) cost = _opencode_cost(p.stdout) + text, err = _opencode_stdout(p.stdout) + if (p.returncode or err) and providers.USAGE_LIMIT.search(p.stdout + p.stderr): + return "", cost, p.stdout + p.stderr ans = out / "answer.md" if ans.is_file() and ans.read_text(errors="replace").strip(): return ans.read_text(errors="replace"), cost, "" - text, err = _opencode_stdout(p.stdout) if text.strip(): return text, cost, "" tail = _strip_oc_banner(p.stderr or "").strip()[-300:] @@ -393,17 +458,6 @@ def _drop_workspace(paths: dict[str, Path] | None) -> None: shutil.rmtree(paths["workspace"], ignore_errors=True) -def _opencode_dir() -> Path: - """The upstream OpenCode checkout to run through bun: config.OPENCODE_DIR.""" - oc = config.OPENCODE_DIR - if not (oc / "packages" / "opencode" / "src" / "index.ts").is_file(): - raise FileNotFoundError( - f"no OpenCode checkout at {oc} (config.OPENCODE_DIR); run " - f"scripts/setup_benchmark.py to clone the pinned commit there" - ) - return oc - - def _parse_json(stdout: str) -> dict: """The claude CLI's one JSON object, tolerating banners and trailing noise around it. @@ -504,7 +558,13 @@ def _opencode_cost(stdout: str) -> float: return round(total, 6) -def _call_fingerprint(label: str, mission: str, system: str, context_dir: Path | str | None) -> str: +def _call_fingerprint( + label: str, + mission: str, + system: str, + context_dir: Path | str | None, + anchor: Path | None = None, +) -> str: """Content-address one call, including its evidence and its mode/task workspace path. The resolved path is part of the key: the two modes can have byte-identical k=1 evidence @@ -516,7 +576,12 @@ def _call_fingerprint(label: str, mission: str, system: str, context_dir: Path | h.update(b"\0") if context_dir: root = Path(context_dir) - h.update(str(root.resolve()).encode("utf-8", errors="replace")) + location = ( + os.path.relpath(root.resolve(), Path(anchor).resolve()) + if anchor + else str(root.resolve()) + ) + h.update(location.encode("utf-8", errors="replace")) h.update(b"\0") if root.is_dir(): for p in sorted(root.rglob("*")): diff --git a/harness2/core/recursion.py b/harness2/core/recursion.py index dda6b98c..99154ce1 100644 --- a/harness2/core/recursion.py +++ b/harness2/core/recursion.py @@ -21,21 +21,21 @@ from __future__ import annotations import re +import time from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from pathlib import Path from typing import Callable, Sequence -from .. import config -from . import context, prompts +from . import context, events, prompts, runmark from . import tree as harness_tree from .ledger import Ledger from .proposer import Proposer, ProposerError from .spec import BenchmarkAdapter, RunView, resolve_prompts STAGE_ATTEMPTS = 2 -TRANSPORT_ATTEMPTS = config.TRANSPORT_ATTEMPTS -PHASE_WIDTH = config.PHASE_WIDTH +TRANSPORT_ATTEMPTS = 2 +PHASE_WIDTH = 2 @dataclass @@ -149,6 +149,7 @@ def _run_stage( decisions: dict[str, str] | None = None, ) -> Stage: """Build evidence, propose an edit, and retry an invalid proposal once.""" + events.check() surface = adapter.substrate.surface() try: @@ -163,6 +164,8 @@ def _run_stage( decisions=decisions, ) mission = context.mission(tree, job=job) + except events.Stopped: + raise except Exception as exc: return Stage( label=label, @@ -257,9 +260,53 @@ def _record_sessions(ledger: Ledger, stages: Sequence[Stage]) -> None: ledger.session(stage.label, stage.usd, cached=stage.cached, attempts=stage.attempts) +def _cached_rollout(adapter, work_dir, label, run_id, harness_dir) -> RunView | None: + """Reuse completed rollouts without executing or charging for them again.""" + cache = work_dir / "rollouts" / label + if not runmark.is_complete(cache, harness_dir=harness_dir): + return None + record = runmark.read_mark(cache) + if record.get("run_id") != run_id: + return None + data = dict(record["rollout"]) + data["run_dir"] = runmark.resolve_path(data["run_dir"], adapter.spec.runs_root) + if not data["run_dir"].is_dir(): + return None + if data["delivered"] and not adapter.delivered(data["run_dir"]): + return None + intact = getattr(adapter, "artifacts_intact", None) + if callable(intact) and not intact(data["run_dir"]): + return None + return RunView(**data) + + +def _finish_rollout(adapter, work_dir, task_id, rv, harness_dir, started) -> None: + rv.wall_s = round(time.monotonic() - started, 3) + cost = getattr(adapter, "rollout_cost", None) + rv.cost_usd = float(cost(rv.run_dir)) if callable(cost) and rv.run_dir.is_dir() else 0.0 + if rv.ok: + data = vars(rv) | {"run_dir": runmark.relative_path(rv.run_dir, adapter.spec.runs_root)} + runmark.mark_complete( + work_dir / "rollouts" / rv.label, + run_id=rv.run_id, + harness_dir=harness_dir, + extra={"rollout": data}, + ) + events.emit( + "rollout", + task=task_id, + label=rv.label, + run_id=rv.run_id, + ok=rv.ok, + delivered=rv.delivered, + cost_usd=rv.cost_usd, + wall_s=rv.wall_s, + ) + + def _roll( adapter: BenchmarkAdapter, - ledger: Ledger, + ledger: Ledger | None, task_id: str, label: str, tree: harness_tree.Tree | None, @@ -275,9 +322,16 @@ def _roll( adapter.substrate.surface(), tree, work_dir / "harnesses" / label ) run_id = run_id_of(label) + cached = _cached_rollout(adapter, work_dir, label, run_id, hdir) + if cached is not None: + if record and ledger is not None: + ledger.rollout(label, cached=True) + return cached + events.check() + started = time.monotonic() try: - rd = adapter.execute(task_id, hdir, run_id) - exists = Path(rd).exists() + rd = Path(adapter.execute(task_id, hdir, run_id)) + exists = rd.exists() rv = RunView( label=label, run_id=run_id, @@ -286,6 +340,8 @@ def _roll( delivered=exists and adapter.delivered(rd), error="" if exists else "adapter returned a missing run directory", ) + except events.Stopped: + raise except Exception as exc: rv = RunView( label=label, @@ -295,8 +351,9 @@ def _roll( delivered=False, error=f"{type(exc).__name__}: {exc}", ) - if record: - ledger.rollout(label) + _finish_rollout(adapter, work_dir, task_id, rv, hdir, started) + if record and ledger is not None: + ledger.rollout(label, rv.cost_usd) return rv @@ -335,7 +392,7 @@ def _roll_many( ) ) for (label, _tree), rv in zip(items, got): - ledger.rollout(label) + ledger.rollout(label, rv.cost_usd) out[label] = rv return out @@ -351,14 +408,16 @@ def run_parallel( run_id_of: Callable[[str], str], base_tree: harness_tree.Tree, library_dir: Path | None = None, - propose_workers: int = PHASE_WIDTH, - rollout_workers: int = PHASE_WIDTH, + propose_workers: int | None = None, + rollout_workers: int | None = None, ) -> ModeResult: """WIDTH: 2k iid candidates from shared evidence, then k INDEPENDENT compositions.""" + propose_workers = PHASE_WIDTH if propose_workers is None else propose_workers + rollout_workers = PHASE_WIDTH if rollout_workers is None else rollout_workers surface = adapter.substrate.surface() ledger = Ledger(mode="agg", task_id=task_id, k=k) for r in base: - ledger.rollout(r.label) + ledger.rollout(r.label, r.cost_usd) result = ModeResult(mode="agg", task_id=task_id, k=k, ledger=ledger) result.rollouts.update({r.label: r for r in base}) ctx_dir = work_dir / "context" @@ -460,14 +519,16 @@ def run_sequential( run_id_of: Callable[[str], str], base_tree: harness_tree.Tree, library_dir: Path | None = None, - propose_workers: int = 2, - rollout_workers: int = PHASE_WIDTH, + propose_workers: int | None = None, + rollout_workers: int | None = None, ) -> ModeResult: """DEPTH: k nested passes over the same shared base evidence.""" + propose_workers = PHASE_WIDTH if propose_workers is None else propose_workers + rollout_workers = PHASE_WIDTH if rollout_workers is None else rollout_workers surface = adapter.substrate.surface() ledger = Ledger(mode="seq", task_id=task_id, k=k) for r in base: - ledger.rollout(r.label) + ledger.rollout(r.label, r.cost_usd) result = ModeResult(mode="seq", task_id=task_id, k=k, ledger=ledger) result.rollouts.update({r.label: r for r in base}) ctx_dir = work_dir / "context" diff --git a/harness2/core/runmark.py b/harness2/core/runmark.py index 1de5794b..b4d58ed2 100644 --- a/harness2/core/runmark.py +++ b/harness2/core/runmark.py @@ -33,6 +33,21 @@ VERSION = 1 +def relative_path(path: Path | str, root: Path) -> str: + """Persist paths inside the run root without tying them to its current location.""" + resolved = Path(path).resolve() + return str( + resolved.relative_to(root.resolve()) + if resolved.is_relative_to(root.resolve()) + else resolved + ) + + +def resolve_path(path: Path | str, root: Path) -> Path: + """Read both relocatable records and legacy absolute paths.""" + return root / Path(path) + + def _ignored(p: Path) -> bool: """Byproducts that are not part of the harness and must not change its identity.""" return "__pycache__" in p.parts or p.suffix == ".pyc" or p.name == ".DS_Store" diff --git a/harness2/core/schedule.py b/harness2/core/schedule.py index f91f450b..87d853d8 100644 --- a/harness2/core/schedule.py +++ b/harness2/core/schedule.py @@ -24,6 +24,7 @@ import os import random import re +import time import traceback from collections import defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed @@ -32,8 +33,8 @@ from pathlib import Path from typing import Mapping, Sequence +from . import events, recursion, runmark, selection from . import library as library_store -from . import recursion, runmark, selection from . import tree as harness_tree from .proposer import Proposer from .spec import BenchmarkAdapter, RunView @@ -188,9 +189,7 @@ def _selector_for( if prop.spec == spec: return prop audit = Path(prop.audit_dir) / "selector" if prop.audit_dir else None - return Proposer( - spec, audit_dir=audit, attempts=prop.attempts, project=prop.project, region=prop.region - ) + return Proposer(spec, audit_dir=audit, attempts=prop.attempts) def _tree_is_base(surface, base_tree: harness_tree.Tree, tree: harness_tree.Tree) -> bool: @@ -391,7 +390,7 @@ def _resume( sel = cached_sels[mode] namespace = Namespace(adapter.spec.name, adapter.substrate.name, mode, k, seed, tag) harness_dir = namespace.task_dir(runs_root, task_id) / "harnesses" / sel["chosen_label"] - run_dir = Path(sel.get("chosen_run_dir") or "") + run_dir = runmark.resolve_path(sel.get("chosen_run_dir") or "", adapter.spec.runs_root) try: if not sel.get("chosen_run_dir") or not run_dir.is_dir(): raise ValueError("adopted run directory is missing") @@ -434,8 +433,19 @@ def _roll_bases( if not pending: return {} rolled: dict[str, RunView] = {} + work_dir = dead_dir.parent + for label, rid in pending: + cached = recursion._cached_rollout(adapter, work_dir, label, rid, None) + if cached is not None: + rolled[label] = cached + pending = [(label, rid) for label, rid in pending if label not in rolled] + if not pending: + return rolled execute_many = getattr(adapter, "execute_many", None) if len(pending) > 1 and callable(execute_many): + for _ in pending: + events.check() + started = time.monotonic() try: dirs = list(execute_many(task_id, None, [rid for _lbl, rid in pending])) for i, (label, rid) in enumerate(pending): @@ -454,27 +464,29 @@ def _roll_bases( delivered=exists and adapter.delivered(rd), error="" if exists else "adapter returned a missing run directory", ) + except events.Stopped: + raise except Exception as exc: for label, rid in pending: rolled[label] = _dead(label, rid, dead_dir, f"{type(exc).__name__}: {exc}") + for label, _rid in pending: + recursion._finish_rollout(adapter, work_dir, task_id, rolled[label], None, started) return rolled def _one(label_rid: tuple[str, str]) -> RunView: label, rid = label_rid - try: - rd = adapter.execute(task_id, None, rid) - return RunView( - label=label, run_id=rid, run_dir=rd, ok=True, delivered=adapter.delivered(rd) - ) - except Exception as exc: - return _dead(label, rid, dead_dir, f"{type(exc).__name__}: {exc}") + return recursion._roll( + adapter, None, task_id, label, None, lambda _label: rid, work_dir, record=False + ) if len(pending) == 1: rv = _one(pending[0]) - return {rv.label: rv} - width = min(len(pending), ROLLOUT_PARALLEL) - with ThreadPoolExecutor(max_workers=max(1, width)) as pool_r: - return {rv.label: rv for rv in pool_r.map(_one, pending)} + rolled[rv.label] = rv + else: + width = min(len(pending), ROLLOUT_PARALLEL) + with ThreadPoolExecutor(max_workers=max(1, width)) as pool_r: + rolled.update({rv.label: rv for rv in pool_r.map(_one, pending)}) + return rolled def _run_one_mode( @@ -493,6 +505,7 @@ def _run_one_mode( surface, pool_policy: str, res: TaskResult, + reflector_prop: Proposer | None = None, ) -> None: """Run one recursion mode over the shared bases, select a final, and record it on `res`.""" work_dir = n.task_dir(runs_root, task_id) @@ -510,6 +523,7 @@ def _run_one_mode( library_dir=lib_dir, ) res.modes[mode] = r.to_dict() + events.check() candidates = selection.selection_pool(mode, k, r.rollouts, base) if pool_policy == "finals" and k == 1: sel = selection.trivial_selection( @@ -526,19 +540,23 @@ def _run_one_mode( work_dir=work_dir, pool_kind="finals", ) + events.emit("selection", task=task_id, mode=mode, chosen=sel.chosen_label) if mode_lib and sel.ok: + events.check() selection.reflect_adoption( task_id, sel, r.rollouts, adapter=adapter, - prop=selector, + prop=reflector_prop if reflector_prop is not None else selector, work_dir=work_dir, decisions={stage.label: stage.notes or stage.error for stage in r.stages}, ) d = sel.to_dict() - d["chosen_run_dir"] = str( - candidates[sel.chosen_label].run_dir if sel.chosen_label in candidates else "" + d["chosen_run_dir"] = ( + runmark.relative_path(candidates[sel.chosen_label].run_dir, adapter.spec.runs_root) + if sel.chosen_label in candidates + else "" ) chosen_tree = r.trees.get(sel.chosen_label, base_tree) @@ -576,6 +594,7 @@ def run_task( library: library_store.Library | None = None, libraries: dict[str, library_store.Library] | None = None, selector_prop: Proposer | None = None, + reflector_prop: Proposer | None = None, pool_policy: str = "finals", ) -> TaskResult: """k shared reference executions -> every mode -> blind selection per mode -> summary.json. @@ -613,6 +632,8 @@ def run_task( ): return res + events.check() + events.emit("task_start", task=task_id, domain=domain, status="running") surface = adapter.substrate.surface() base_tree = harness_tree.read_tree(surface, adapter.base_harness()) @@ -625,11 +646,19 @@ def run_task( rolled = _roll_bases(task_id, adapter, pending, dead_dir) base: list[RunView] = [rolled[label] for label in labels] - res.base = [{"label": b.label, "run_id": b.run_id, "run_dir": str(b.run_dir)} for b in base] + res.base = [ + { + "label": b.label, + "run_id": b.run_id, + "run_dir": runmark.relative_path(b.run_dir, adapter.spec.runs_root), + } + for b in base + ] if not any(b.ok for b in base): res.status = "base-failed" res.errors.append("no base rollout produced anything") _write_summary(summary_path, res) + events.emit("task_done", task=task_id, domain=domain, status=res.status) return res for mode in modes: @@ -643,6 +672,7 @@ def run_task( adapter=adapter, prop=prop, selector=selector, + reflector_prop=reflector_prop, n=n, runs_root=runs_root, mode_lib=mode_libs.get(mode), @@ -651,6 +681,8 @@ def run_task( pool_policy=pool_policy, res=res, ) + except events.Stopped: + raise except Exception as exc: res.modes[mode] = { "ok": False, @@ -663,6 +695,7 @@ def run_task( res.status = _merged_status(res.to_dict()) _write_summary(summary_path, res) + events.emit("task_done", task=task_id, domain=domain, status=res.status) return res @@ -679,6 +712,7 @@ def run_domain( seed: int | None = None, tag: str = "", selector_prop: Proposer | None = None, + reflector_prop: Proposer | None = None, pool_policy: str = "finals", ) -> dict: """Every task in one domain, in `stream_order()` order, threading the library forward.""" @@ -711,6 +745,7 @@ def run_domain( tag=tag, libraries=libs, selector_prop=selector_prop, + reflector_prop=reflector_prop, pool_policy=pool_policy, ) if r.status in ("ok", "skipped"): @@ -724,6 +759,8 @@ def run_domain( "error": "; ".join(r.errors) if r.errors else r.status, "library_entries_before": before, } + except events.Stopped: + raise except Exception as exc: out["failed"] += 1 row = {"task": tid, "status": "exception", "error": f"{type(exc).__name__}: {exc}"} @@ -791,6 +828,7 @@ def run_sweep( domain_parallel: int = 1, only_domains: Sequence[str] = (), selector_prop: Proposer | None = None, + reflector_prop: Proposer | None = None, pool_policy: str = "finals", max_tasks: int | None = None, only_tasks: Sequence[str] = (), @@ -824,15 +862,25 @@ def run_sweep( seed=seed, tag=tag, selector_prop=selector, + reflector_prop=reflector_prop, pool_policy=pool_policy, ): d for d in order } - for f in as_completed(futs): - try: - rows.append(f.result()) - except Exception as exc: - rows.append({"domain": futs[f], "error": f"{type(exc).__name__}: {exc}"}) + stopped = None + try: + for f in as_completed(futs): + try: + rows.append(f.result()) + except events.Stopped as exc: + stopped = exc + except Exception as exc: + rows.append({"domain": futs[f], "error": f"{type(exc).__name__}: {exc}"}) + except KeyboardInterrupt: + events.request_stop("interrupted") + raise + if stopped is not None: + raise stopped return { "benchmark": adapter.spec.name, "substrate": adapter.substrate.name, diff --git a/harness2/core/selection.py b/harness2/core/selection.py index 2cf04f50..6d88bdb7 100644 --- a/harness2/core/selection.py +++ b/harness2/core/selection.py @@ -25,7 +25,7 @@ from pathlib import Path from typing import Callable, Sequence, TypeVar -from . import context +from . import context, runmark from . import prompts as P from .proposer import Proposer, ProposerError from .spec import BenchmarkAdapter, BenchPrompts, RunView, resolve_prompts @@ -352,7 +352,9 @@ def run_selection( sel.chosen_run_id = pool[sel.chosen_label].run_id sel.chosen_is_base = _is_base(sel.chosen_label) - _persist(sel, root, str(pool[sel.chosen_label].run_dir if sel.chosen_label in pool else "")) + _persist( + sel, root, runmark.relative_path(pool[sel.chosen_label].run_dir, adapter.spec.runs_root) + ) return sel @@ -395,7 +397,7 @@ def trivial_selection( root = Path(work_dir) / "selection" / f"{mode}{k}" shutil.rmtree(root, ignore_errors=True) root.mkdir(parents=True, exist_ok=True) - _persist(sel, root, str(pool[label].run_dir)) + _persist(sel, root, runmark.relative_path(pool[label].run_dir, adapter.spec.runs_root)) return sel @@ -447,7 +449,11 @@ def reflect_adoption( task_id, sel, candidates, adapter=adapter, prop=prop, root=root, layout=evidence.layout ) sel.experience = experience - _persist(sel, Path(work_dir) / "selection" / f"{sel.mode}{sel.k}", str(chosen.run_dir)) + _persist( + sel, + Path(work_dir) / "selection" / f"{sel.mode}{sel.k}", + runmark.relative_path(chosen.run_dir, adapter.spec.runs_root), + ) return experience diff --git a/harness2/core/spec.py b/harness2/core/spec.py index e58d1485..fc4d8af2 100644 --- a/harness2/core/spec.py +++ b/harness2/core/spec.py @@ -25,12 +25,12 @@ from pathlib import Path from typing import Protocol, runtime_checkable -from .. import config +from ..providers import Provider from .tree import SurfaceSpec ModelId = str -CHANNELS = ("claude_cli", "opencode") +CHANNELS = ("claude", "codex", "opencode", "human") @dataclass(frozen=True) @@ -42,50 +42,16 @@ class ChannelSpec: effort: str = "high" steps: int = 120 timeout: int = 3600 + bin: str = "" + provider: Provider | None = None def __post_init__(self) -> None: + if self.kind == "claude_cli": + object.__setattr__(self, "kind", "claude") if self.kind not in CHANNELS: raise ValueError(f"unknown channel {self.kind!r}; expected one of {CHANNELS}") -def resolve_channel_model( - kind: str | None, - model: str | None, - *, - effort: str = "high", - steps: int = 120, - timeout: int = 3600, -) -> ChannelSpec: - """The one channel->model mapping; a lone override of either raises.""" - use_default = kind is None and model is None - if (kind is None) != (model is None): - raise ValueError( - "channel and model must be overridden TOGETHER (a bare-id/provider-prefixed " - f"mismatch fails silently downstream); got kind={kind!r}, model={model!r}" - ) - if kind is None: - kind = config.DEFAULT_PROPOSER_CHANNEL - if kind not in CHANNELS: - raise ValueError(f"unknown channel {kind!r}; expected one of {CHANNELS}") - if model is None: - model = config.PROPOSER_MODEL_BY_CHANNEL[kind] - if not model.strip(): - if use_default: - return ChannelSpec(kind, "", effort=effort, steps=steps, timeout=timeout) - raise ValueError("model override must not be empty") - if kind == "opencode" and "/" not in model: - raise ValueError( - f"opencode channel needs a provider-prefixed model id " - f"in HARNESS2_IMPROVER_MODEL or HARNESS2_SOLVER_MODEL, got {model!r}" - ) - if kind == "claude_cli" and "/" in model: - raise ValueError( - f"claude_cli channel takes a bare Anthropic id " - f"(e.g. {config.PROPOSER_MODEL_BY_CHANNEL['claude_cli']!r}), got {model!r}" - ) - return ChannelSpec(kind, model, effort=effort, steps=steps, timeout=timeout) - - @dataclass(frozen=True) class BenchPrompts: """The benchmark-shaped text the shared prompts interpolate.""" @@ -126,7 +92,14 @@ def __post_init__(self) -> None: ) -INSTALL_MODES = ("workspace_agents_md", "home_dir_jinja", "workspace_config", "codex_root") +INSTALL_MODES = ( + "workspace_agents_md", + "home_dir_jinja", + "workspace_config", + "codex_root", + "claude_root", + "command_files", +) @dataclass(frozen=True) @@ -271,7 +244,6 @@ def resolve_prompts(adapter: BenchmarkAdapter, task_id: str) -> BenchPrompts: "RunView", "ModelId", "resolve_prompts", - "resolve_channel_model", "CHANNELS", "INSTALL_MODES", ] diff --git a/harness2/core/tree.py b/harness2/core/tree.py index c9bf1341..45cc7268 100644 --- a/harness2/core/tree.py +++ b/harness2/core/tree.py @@ -25,6 +25,7 @@ import json import re from dataclasses import dataclass +from fnmatch import fnmatchcase from functools import lru_cache from pathlib import Path @@ -56,19 +57,27 @@ class SurfaceSpec: hook_events: frozenset[str] = frozenset() compile_skill_scripts: bool = False reject_empty_always_on: bool = False + script_dir: str = "scripts" + skills_need_trigger: bool = True + editable_globs: tuple[str, ...] = () stubs: tuple[tuple[str, str], ...] = () @lru_cache(maxsize=8) -def _res(surface_name: str, skill_dir: str, plugin_dir: str | None) -> dict[str, re.Pattern]: +def _res( + surface_name: str, skill_dir: str, plugin_dir: str | None, script_dir: str +) -> dict[str, re.Pattern]: sd = re.escape(skill_dir) + scripts = re.escape(script_dir) out = { "skill": re.compile(rf"{sd}/({_NAME})/SKILL\.md"), "skill_script": re.compile(rf"{sd}/{_NAME}/scripts/[A-Za-z0-9_.\-]+"), "subagent": re.compile(rf"sub_agents/({_NAME})/AGENT\.md"), - "script": re.compile(r"scripts/[a-z0-9_][a-z0-9_]*\.py"), - "script_ref": re.compile(rf"(?:python3?\s+)((?:{sd}/[\w\-]+/)?scripts/[\w.\-]+\.py)"), + "script": re.compile(rf"{scripts}/[a-z0-9_][a-z0-9_]*\.py"), + "script_ref": re.compile( + rf"(?:python3?\s+)((?:{sd}/[\w\-]+/scripts|{scripts})/[\w.\-]+\.py)" + ), } if plugin_dir is not None: out["plugin"] = re.compile(rf"{re.escape(plugin_dir)}/({_NAME})\.ts") @@ -76,7 +85,7 @@ def _res(surface_name: str, skill_dir: str, plugin_dir: str | None) -> dict[str, def _rx(surface: SurfaceSpec) -> dict[str, re.Pattern]: - return _res(surface.name, surface.skill_dir, surface.plugin_dir) + return _res(surface.name, surface.skill_dir, surface.plugin_dir, surface.script_dir) def normalize_rel(rel: str) -> str: @@ -95,6 +104,8 @@ def is_allowed(surface: SurfaceSpec, rel: str) -> bool: rel = normalize_rel(rel) except ValueError: return False + if surface.editable_globs: + return any(fnmatchcase(rel, g) for g in surface.editable_globs) if rel in surface.always_on: return True if surface.hooks_file is not None and rel == surface.hooks_file: @@ -150,7 +161,9 @@ def read_tree(surface: SurfaceSpec, d: Path | str, *, rejected: list[str] | None def is_tree(surface: SurfaceSpec, d: Path | str) -> bool: d = Path(d) - return d.is_dir() and any((d / a).is_file() for a in surface.always_on) + return d.is_dir() and ( + bool(surface.editable_globs) or any((d / a).is_file() for a in surface.always_on) + ) def write_tree(surface: SurfaceSpec, tree: Tree, dest: Path | str, *, stubs: bool = True) -> Path: @@ -194,6 +207,8 @@ def autotrigger(surface: SurfaceSpec, tree: Tree, *, always_on_max: int | None = A line that would push the body over the always-on cap is skipped instead. """ + if not surface.skills_need_trigger: + return [] cap = surface.always_on_max if always_on_max is None else always_on_max body = always_on_text(surface, tree) fixed: list[str] = [] @@ -289,7 +304,7 @@ def _check_skills(surface: SurfaceSpec, tree: Tree, body: str, errs: list[str]) f"trigger line naming `{name}` stops matching; drop the `name:` line " f"or make it `name: {name}`" ) - if name not in body: + if surface.skills_need_trigger and name not in body: errs.append( f"skill '{name}' is never named in the always-on text, so the agent " f"will not know to invoke it — an untriggered skill is dead weight. " @@ -358,21 +373,25 @@ def validate( f"this framework, so an empty one is a harness that does not exist" ) - skills = _check_skills(surface, tree, body, errs) - subs = _check_subagents(surface, tree, allow_subagents, errs) + skills = [] if surface.editable_globs else _check_skills(surface, tree, body, errs) + subs = [] if surface.editable_globs else _check_subagents(surface, tree, allow_subagents, errs) for rel, src in tree.items(): - if r["script"].fullmatch(rel) or ( + if surface.editable_globs: + if rel.endswith(".py") and is_allowed(surface, rel): + _py_compiles(rel, src, errs) + elif r["script"].fullmatch(rel) or ( surface.compile_skill_scripts and r["skill_script"].fullmatch(rel) and rel.endswith(".py") ): _py_compiles(rel, src, errs) - if surface.hooks_file is not None and surface.hooks_file in tree: + if not surface.editable_globs and surface.hooks_file is not None and surface.hooks_file in tree: check_hooks(surface, tree[surface.hooks_file], errs) - _check_script_refs(surface, tree, r, reserved_scripts, errs) + if not surface.editable_globs: + _check_script_refs(surface, tree, r, reserved_scripts, errs) if errs: raise ValueError(" | ".join(errs)) diff --git a/harness2/demo.py b/harness2/demo.py index 487de02d..01c7c9e8 100644 --- a/harness2/demo.py +++ b/harness2/demo.py @@ -22,8 +22,8 @@ from decimal import Decimal from pathlib import Path -from .core import tree -from .core.proposer import Proposer, Reply +from .core import runmark, tree +from .core.proposer import Proposer from .core.spec import BenchPrompts, BenchSpec, ChannelSpec from .substrate.opencode import OpencodeSubstrate @@ -86,8 +86,14 @@ def validate_tree(self, harness: tree.Tree) -> None: def execute( self, task_id: str, harness_dir: Path | None, run_id: str, *, timeout: int | None = None ) -> Path: + run_dir = self.spec.runs_root / "executions" / run_id + if ( + runmark.is_complete(run_dir, harness_dir=harness_dir) + and self.delivered(run_dir) + and (run_dir / "trajectory.md").is_file() + ): + return run_dir self.execution_count += 1 - run_dir = self.root / "executions" / run_id run_dir.mkdir(parents=True, exist_ok=True) script = (harness_dir or self.base_harness()) / "scripts/summarize.py" result = subprocess.run( @@ -102,6 +108,7 @@ def execute( "[ACTION] Run scripts/summarize.py on ledger.csv\n[OBSERVATION]\n" + result.stdout, encoding="utf-8", ) + runmark.mark_complete(run_dir, run_id=run_id, harness_dir=harness_dir) return run_dir def delivered(self, run_dir: Path) -> bool: @@ -120,10 +127,9 @@ def deliverable_files(self, run_dir: Path) -> dict[str, str]: class DemoProposer(Proposer): """Scripted edits and selection; no model service or benchmark judge is used.""" - def call(self, mission, context_dir, label, **kwargs) -> Reply: - context = Path(context_dir) - workspace = None - if label.startswith("select."): + def _opencode(self, cwd, mission, system, wall): + context = cwd / "context" + if (context / "candidates").is_dir(): with (context / "source/ledger.csv").open() as source: expected = sum( (Decimal(row["amount"]) for row in csv.DictReader(source)), Decimal(0) @@ -135,7 +141,7 @@ def call(self, mission, context_dir, label, **kwargs) -> Reply: text = ( f"The signed total matches the source ledger.\n```choice\n{chosen}\n```" ) - elif label.startswith("experience."): + elif (context / "adoption.json").is_file(): changed = (context / "harness/scripts/summarize.py").read_text() != ( context / "base_harness/scripts/summarize.py" ).read_text() @@ -158,6 +164,4 @@ def call(self, mission, context_dir, label, **kwargs) -> Reply: script = workspace / "scripts/summarize.py" script.write_text(script.read_text().replace(" if amount > 0", ""), encoding="utf-8") text = "Keep signed amounts: the base total omitted credits in the source ledger." - reply = Reply(label=label, text=text, ok=True, attempts=1, workspace=workspace) - self._calls.append(reply) - return reply + return text, 0.0, "" diff --git a/harness2/providers.py b/harness2/providers.py new file mode 100644 index 00000000..f17b7ba3 --- /dev/null +++ b/harness2/providers.py @@ -0,0 +1,252 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Translate provider profiles into each model client's configuration.""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import urlsplit + +from .settings import ConfigError, state_dir + +PROVIDER_TYPES = ("anthropic", "openai", "vertex", "bedrock", "claude-login", "chatgpt-login") +USAGE_LIMIT = re.compile( + r"(?i)usage limit|rate[ _-]?limit|too many requests|\b429\b|quota|credit balance is too low" +) +STRIP_ENV = ( + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_BASE_URL", + "ANTHROPIC_MODEL", + "ANTHROPIC_BEDROCK_BASE_URL", + "ANTHROPIC_VERTEX_PROJECT_ID", + "CLAUDE_CODE_USE_BEDROCK", + "CLAUDE_CODE_USE_VERTEX", + "CLOUD_ML_REGION", + "OPENAI_BASE_URL", +) + + +@dataclass(frozen=True) +class Provider: + """A named endpoint with credentials referenced by environment variable.""" + + name: str + type: str + base_url: str = "" + api_key_env: str = "" + project: str = "" + region: str = "" + home: str = "" + + +def codex_login_home(p: Provider) -> Path: + return Path(p.home).expanduser() if p.home else state_dir() / "codex-home" + + +def codex_logged_in(home: Path) -> bool: + try: + return ( + subprocess.run( + ["codex", "login", "status"], + env={ + **{ + k: v + for k, v in os.environ.items() + if k not in (*STRIP_ENV, "OPENAI_API_KEY", "CODEX_API_KEY") + }, + "CODEX_HOME": str(home), + }, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=10, + ).returncode + == 0 + ) + except Exception: + return False + + +def split_model(ref: str) -> tuple[str, str]: + provider, separator, model = ref.partition("/") + if not separator or not provider or not model: + raise ConfigError( + f"model {ref!r} must be written provider/model, e.g. anthropic/claude-sonnet-5-5" + ) + return provider, model + + +def claude_env(p: Provider) -> dict[str, str]: + if p.type == "claude-login": + return {} + if p.type == "chatgpt-login": + raise ConfigError("Claude Code cannot use a ChatGPT login.") + if p.type == "anthropic": + env = {} + if p.api_key_env: + env["ANTHROPIC_API_KEY"] = os.environ.get(p.api_key_env, "") + if p.base_url: + env["ANTHROPIC_BASE_URL"] = p.base_url + return env + if p.type == "vertex": + return { + "CLAUDE_CODE_USE_VERTEX": "1", + "ANTHROPIC_VERTEX_PROJECT_ID": p.project, + "CLOUD_ML_REGION": p.region or "global", + } + if p.type == "bedrock": + return {"CLAUDE_CODE_USE_BEDROCK": "1", **({"AWS_REGION": p.region} if p.region else {})} + if p.type == "openai": + raise ConfigError("Claude Code needs an Anthropic-compatible provider.") + raise ConfigError(f"unknown provider type {p.type!r}; use one of {PROVIDER_TYPES}") + + +def codex_config(p: Provider, model: str, effort: str) -> str: + if p.type != "openai": + raise ConfigError( + "Codex needs an OpenAI Responses-compatible provider; serve one with " + "`harness2 bridge --model /` and point an openai-type provider at it." + ) + + def quote(value): + return json.dumps(value, ensure_ascii=False).replace("\x7f", "\\u007f") + + # Codex refuses to redefine its built-in provider ids (e.g. "openai"), so namespace ours. + pid = f"harness2-{p.name}" + return ( + f"model = {quote(model)}\n" + f"model_provider = {quote(pid)}\n" + f"model_reasoning_effort = {quote(effort)}\n\n" + f"[model_providers.{quote(pid)}]\n" + f"name = {quote(p.name)}\n" + f"base_url = {quote(p.base_url)}\n" + f"env_key = {quote(p.api_key_env)}\n" + 'wire_api = "responses"\n' + ) + + +def codex_args(p: Provider | None, model: str, effort: str) -> list[str]: + if p is not None and p.type == "chatgpt-login": + return [ + "--ignore-user-config", + "--ignore-rules", + "-m", + model, + "-c", + f"model_reasoning_effort={json.dumps(effort)}", + ] + if p is not None: + codex_config(p, model, effort) + return [] + + +def opencode_config(p: Provider, model: str) -> tuple[dict, dict[str, str]]: + if p.type in ("claude-login", "chatgpt-login"): + raise ConfigError("OpenCode needs an API provider; subscription logins are not supported.") + if p.type == "vertex": + region = p.region or "global" + return {}, { + "GOOGLE_VERTEX_PROJECT": p.project, + "GOOGLE_VERTEX_LOCATION": region, + "VERTEXAI_LOCATION": region, + } + if p.type == "bedrock": + raise ConfigError("Bedrock is not supported on the opencode channel.") + if p.type not in ("openai", "anthropic"): + raise ConfigError(f"unknown provider type {p.type!r}; use one of {PROVIDER_TYPES}") + options = {} + if p.api_key_env or p.type == "openai": + options["apiKey"] = "{env:" + p.api_key_env + "}" + if p.base_url or p.type == "openai": + options["baseURL"] = p.base_url + npm = "@ai-sdk/openai-compatible" if p.type == "openai" else "@ai-sdk/anthropic" + return { + "provider": { + p.name: { + "npm": npm, + "name": p.name, + "options": options, + "models": {model: {}}, + } + } + }, {} + + +def litellm_params(p: Provider, model: str) -> dict: + if p.type in ("claude-login", "chatgpt-login"): + raise ConfigError( + "The bridge needs an API provider; subscription logins are not supported." + ) + if p.type in ("anthropic", "openai"): + params = {"model": f"{p.type}/{model}", "api_key": f"os.environ/{p.api_key_env}"} + if p.base_url or p.type == "openai": + params["api_base"] = p.base_url + return params + if p.type == "vertex": + return { + "model": f"vertex_ai/{model}", + "vertex_project": p.project, + "vertex_location": p.region or "global", + } + if p.type == "bedrock": + return {"model": f"bedrock/{model}", "aws_region_name": p.region} + raise ConfigError(f"unknown provider type {p.type!r}; use one of {PROVIDER_TYPES}") + + +def check(p: Provider) -> list[str]: + if p.type == "chatgpt-login": + home = codex_login_home(p) + if codex_logged_in(home): + return [] + if p.home: + return [ + f"chatgpt-login: not logged in at {home}; run `codex login` with CODEX_HOME={home}" + ] + return ["chatgpt-login: not logged in for harness2; run `harness2 login codex`"] + if p.type == "claude-login": + try: + result = subprocess.run( + ["claude", "auth", "status"], + env={k: v for k, v in os.environ.items() if k not in STRIP_ENV}, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=10, + ) + if result.returncode == 0: + return [] + except (OSError, subprocess.SubprocessError): + pass + return ["claude-login: not logged in; run `claude auth login`"] + problems = [] + if p.api_key_env and not os.environ.get(p.api_key_env): + problems.append(f"{p.name}: set {p.api_key_env}") + if p.type == "vertex" and not p.project: + problems.append(f"{p.name}: set project") + if p.base_url: + try: + url = urlsplit(p.base_url) + valid = url.scheme in ("http", "https") and bool(url.netloc) + except ValueError: + valid = False + if not valid: + problems.append(f"{p.name}: base_url must be an HTTP(S) URL") + return problems diff --git a/harness2/registry.py b/harness2/registry.py new file mode 100644 index 00000000..62f8abd8 --- /dev/null +++ b/harness2/registry.py @@ -0,0 +1,49 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Discover suite modules without importing them until selected.""" + +from __future__ import annotations + +import importlib +import importlib.metadata +from dataclasses import dataclass +from pathlib import Path +from types import ModuleType + +from .settings import ConfigError, Settings + + +@dataclass(frozen=True) +class SuiteContext: + """The project and run settings supplied to a suite's adapter factory.""" + + settings: Settings + project: Path + run_root: Path + split_file: Path | None = None + + +def names() -> list[str]: + installed = importlib.metadata.entry_points(group="harness2.suites") + return ["tasks", *sorted({entry.name for entry in installed} - {"tasks"})] + + +def load(name: str) -> ModuleType: + if name == "tasks": + return importlib.import_module("harness2.suite") + for entry in importlib.metadata.entry_points(group="harness2.suites"): + if entry.name == name: + return entry.load() + raise ConfigError(f"unknown suite {name!r}; available suites: {', '.join(names())}") diff --git a/harness2/settings.py b/harness2/settings.py new file mode 100644 index 00000000..5aa93fe0 --- /dev/null +++ b/harness2/settings.py @@ -0,0 +1,443 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Layered project settings, provenance, and TOML edits.""" + +from __future__ import annotations + +import json +import os +import re +import tomllib +from collections.abc import Mapping +from copy import deepcopy +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from . import providers + +DEFAULTS = { + "suite": {"name": "tasks", "work": ""}, + "paths": {"harness": ".", "tasks": ".harness2/tasks", "runs": ".harness2/runs"}, + "executor": { + "name": "claude", + "model": "", + "bin": "", + "effort": "high", + "timeout": 1800, + "command": "", + "editable": [], + }, + "improver": { + "channel": "claude", + "model": "", + "bin": "", + "effort": "high", + "timeout": 3600, + "steps": 120, + "steer": "", + }, + "selector": {}, + "reflector": {}, + "judge": {}, + "run": { + "k": 1, + "modes": "parallel", + "seed": None, + "workers": 1, + "phase_width": 2, + "proposer_concurrency": 2, + "transport_attempts": 2, + "obs_trunc": 50000, + "budget_usd": 0.0, + "max_calls": 0, + }, + "providers": { + "anthropic": {"type": "anthropic", "base_url": "", "api_key_env": "ANTHROPIC_API_KEY"}, + "openai": { + "type": "openai", + "base_url": "https://api.openai.com/v1", + "api_key_env": "OPENAI_API_KEY", + }, + "claude-login": {"type": "claude-login"}, + "chatgpt-login": {"type": "chatgpt-login"}, + }, +} +_MISSING = object() + + +class ConfigError(RuntimeError): + """Invalid configuration; each message line carries the remediation.""" + + +def find_project(start: Path) -> Path | None: + """Find the nearest ancestor, including start, with a .harness2 directory.""" + start = start.resolve() + return next((p for p in (start, *start.parents) if (p / ".harness2").is_dir()), None) + + +def global_path() -> Path: + return ( + Path(os.environ.get("XDG_CONFIG_HOME") or Path.home() / ".config") / "harness2/config.toml" + ) + + +def project_path(project: Path) -> Path: + return project / ".harness2/config.toml" + + +def cache_dir() -> Path: + return Path(os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache") / "harness2" + + +def state_dir() -> Path: + return Path(os.environ.get("XDG_STATE_HOME") or Path.home() / ".local/state") / "harness2" + + +def parse_value(raw: str) -> Any: + """Parse a TOML literal, falling back to an unquoted string.""" + try: + return tomllib.loads("v = " + raw)["v"] + except (tomllib.TOMLDecodeError, KeyError): + return raw + + +def _merge(data: dict, values: Mapping, origins: dict[str, str], origin: str, prefix="") -> None: + for key, value in values.items(): + dotted = f"{prefix}.{key}" if prefix else key + if isinstance(value, dict): + if not isinstance(data.get(key), dict): + data[key] = {} + _merge(data[key], value, origins, origin, dotted) + else: + data[key] = deepcopy(value) + origins[dotted] = origin + + +def _put(data: dict, key: str, value: Any, origins: dict[str, str], origin: str) -> None: + parts = key.split(".") + if not all(parts): + raise ConfigError(f"invalid setting key {key!r}; use a dotted key") + nested = value + for part in reversed(parts): + nested = {part: nested} + _merge(data, nested, origins, origin) + + +def _validate(data: dict) -> None: + from . import providers # providers imports ConfigError from here + + problems = [] + + def secrets(value): + if isinstance(value, dict): + if "api_key" in value: + problems.append( + "store secrets in environment variables and reference them with api_key_env" + ) + for child in value.values(): + secrets(child) + elif isinstance(value, list): + for child in value: + secrets(child) + + secrets(data) + for name, value in data.items(): + if name not in DEFAULTS and name != "profiles": + problems.append( + f"unknown section {name!r}; use one of {', '.join(DEFAULTS)} or profiles" + ) + elif not isinstance(value, dict): + problems.append(f"{name} must be a table") + run = data.get("run", {}) + if isinstance(run, dict): + for key in ("k", "workers", "phase_width", "proposer_concurrency", "transport_attempts"): + value = run.get(key) + if type(value) is not int or value <= 0: + problems.append(f"run.{key} must be a positive integer") + for key in ("budget_usd", "max_calls"): + value = run.get(key) + if type(value) not in (int, float) or not value >= 0: + problems.append(f"run.{key} must be a non-negative number") + configured = data.get("providers", {}) + if isinstance(configured, dict): + for name, value in configured.items(): + if not isinstance(value, dict): + problems.append(f"providers.{name} must be a table") + elif value.get("type") not in providers.PROVIDER_TYPES: + problems.append( + f"providers.{name}.type must be one of {', '.join(providers.PROVIDER_TYPES)}" + ) + if problems: + raise ConfigError("\n".join(dict.fromkeys(problems))) + + +@dataclass +class Settings: + """Resolved settings and the source of each stored value.""" + + project: Path | None + _data: dict = field(repr=False) + _origins: dict[str, str] = field(repr=False) + + def get(self, key: str, default=_MISSING) -> Any: + value = self._data + for part in key.split("."): + if not isinstance(value, dict) or part not in value: + if default is _MISSING: + raise KeyError(key) + return default + value = value[part] + return deepcopy(value) + + def origin(self, key: str) -> str: + self.get(key) + return self._origins[key] + + def flat(self) -> dict[str, Any]: + values = {} + + def visit(data, prefix=""): + for key, value in data.items(): + dotted = f"{prefix}.{key}" if prefix else key + if isinstance(value, dict): + visit(value, dotted) + else: + values[dotted] = deepcopy(value) + + visit(self._data) + return values + + def section(self, name: str) -> dict: + return self.get(name) + + def role(self, name: str) -> dict: + if name not in ("improver", "selector", "reflector", "judge"): + raise ConfigError(f"unknown role {name!r}; use improver, selector, reflector or judge") + values = self.section("improver") + if name == "reflector": + selector = self.role("selector") + if selector["channel"] != "human": + values.update(selector) + if name != "improver": + values.update(self.section(name)) + keys = ("channel", "model", "bin", "effort", "timeout", "steps") + if name == "improver": + keys += ("steer",) + return {key: values[key] for key in keys} + + def provider(self, name: str) -> providers.Provider: + from . import providers + + configured = self.section("providers") + if name not in configured: + raise ConfigError(f"unknown provider {name!r}; configure providers.{name}") + fields = ("type", "base_url", "api_key_env", "project", "region", "home") + return providers.Provider( + name, **{k: v for k, v in configured[name].items() if k in fields} + ) + + def path(self, key: str) -> Path: + path = Path(self.get(key)).expanduser() + return ((self.project or Path.cwd()) / path).resolve() + + +def load( + project: Path | None, + *, + profile: str | None = None, + overrides: Mapping[str, tuple[Any, str]] = {}, +) -> Settings: + """Apply defaults, files, a profile, environment, and explicit overrides in order.""" + if project is not None: + project = project.resolve() + load_dotenv(project / ".harness2/.env") + data, origins = {}, {} + _merge(data, DEFAULTS, origins, "default") + paths = [global_path()] + if project is not None: + paths.append(project_path(project)) + for path in paths: + try: + content = path.read_text(encoding="utf-8") + except FileNotFoundError: + continue + except (OSError, UnicodeError) as e: + raise ConfigError(f"cannot read {path}: {e}") from e + try: + values = tomllib.loads(content) + except tomllib.TOMLDecodeError as e: + raise ConfigError(f"invalid TOML in {path}: {e}") from e + _merge(data, values, origins, str(path)) + profile = profile if profile is not None else os.environ.get("HARNESS2_PROFILE") + if profile: + profiles = data.get("profiles", {}) + if not isinstance(profiles, dict) or profile not in profiles: + raise ConfigError(f"unknown profile {profile!r}; configure profiles.{profile}") + if not isinstance(profiles[profile], dict): + raise ConfigError(f"profiles.{profile} must be a table") + _merge(data, deepcopy(profiles[profile]), origins, f"profile {profile}") + for name, raw in os.environ.items(): + if name.startswith("HARNESS2_") and "__" in name: + key = name.removeprefix("HARNESS2_").lower().replace("__", ".") + _put(data, key, parse_value(raw), origins, f"env {name}") + for key, (value, origin) in overrides.items(): + _put(data, key, value, origins, origin) + _validate(data) + return Settings(project, data, origins) + + +def _literal(value: Any) -> str: + if isinstance(value, str): + return json.dumps(value, ensure_ascii=False).replace("\x7f", "\\u007f") + if isinstance(value, bool): + return str(value).lower() + if type(value) in (int, float): + return str(value) + if isinstance(value, list) and all(not isinstance(v, list) for v in value): + return "[" + ", ".join(_literal(v) for v in value) + "]" + raise ConfigError("config values must be strings, booleans, numbers or lists of scalars") + + +def _toml_key(key: str) -> str: + return key if re.fullmatch(r"[A-Za-z0-9_-]+", key) else _literal(key) + + +def _comments(text: str) -> list[str]: + comments = [] + quote = "" + i = 0 + while i < len(text): + if quote: + if text[i] == "\\" and quote.startswith('"'): + i += 2 + continue + if text.startswith(quote, i): + i += len(quote) + if len(quote) == 3: + while i < len(text) and text[i] == quote[0]: + i += 1 + quote = "" + continue + elif text[i] in "\"'": + quote = text[i] * (3 if text.startswith(text[i] * 3, i) else 1) + i += len(quote) + continue + elif text[i] == "#": + end = text.find("\n", i) + end = len(text) if end == -1 else end + comments.append(text[i:end].rstrip("\r")) + i = end + i += 1 + return comments + + +def set_value(path: Path, key: str, value: Any) -> None: + """Replace or append a TOML assignment while retaining unrelated lines and comments.""" + parts = key.split(".") + if not all(parts): + raise ConfigError(f"invalid setting key {key!r}; use a dotted key") + literal = _literal(value) + try: + with path.open(encoding="utf-8", newline="") as source: + text = source.read() + except FileNotFoundError: + text = "" + except (OSError, UnicodeError) as e: + raise ConfigError(f"cannot read {path}: {e}") from e + try: + tomllib.loads(text) + except tomllib.TOMLDecodeError as e: + raise ConfigError(f"invalid TOML in {path}: {e}") from e + lines = text.splitlines(keepends=True) + newline = "\r\n" if "\r\n" in text else "\n" + table, leaf = tuple(parts[:-1]), parts[-1] + current = () + insert = 0 if not table else None + start = 0 + while start < len(lines): + end = start + 1 + while True: + try: + parsed = tomllib.loads("".join(lines[start:end])) + break + except tomllib.TOMLDecodeError: + end += 1 + if end > len(lines): + raise ConfigError(f"cannot edit {key!r} in {path}") from None + if lines[start].lstrip().startswith("["): + names = [] + node = parsed + while isinstance(node, dict) and len(node) == 1: + name, node = next(iter(node.items())) + names.append(name) + current = tuple(names) if isinstance(node, dict) else None + if current == table: + insert = end + if leaf in parsed and not lines[start].lstrip().startswith("["): + comments = _comments("".join(lines[start:end])) + indent = lines[start][: len(lines[start]) - len(lines[start].lstrip())] + replacement = f"{indent}{_toml_key(leaf)} = {literal}" + if comments: + replacement += " " + comments.pop() + replacement += newline + lines[start:end] = [*(c + newline for c in comments), replacement] + break + start = end + else: + assignment = f"{_toml_key(leaf)} = {literal}{newline}" + if insert is None: + header = ".".join(_toml_key(part) for part in table) + lines.append(f"{newline}[{header}]{newline}{assignment}") + else: + if insert and not lines[insert - 1].endswith("\n"): + assignment = newline + assignment + lines.insert(insert, assignment) + result = "".join(lines) + try: + tomllib.loads(result) + except tomllib.TOMLDecodeError as e: + raise ConfigError(f"cannot set {key!r} in {path}: {e}") from e + try: + encoded = result.encode("utf-8") + except UnicodeError as e: + raise ConfigError("config strings must contain valid Unicode") from e + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(encoded) + + +def load_dotenv(path: Path) -> None: + """Load KEY=VALUE lines with setdefault, using the original config module's parser.""" + try: + text = Path(path).read_text(encoding="utf-8") + except OSError: + return + for raw in text.splitlines(): + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + if line.startswith("export "): + line = line[len("export ") :].lstrip() + key, _, value = line.partition("=") + key = key.strip() + if not key or not all(c.isalnum() or c == "_" for c in key): + continue + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'): + value = value[1:-1] + elif " #" in value: + value = value.split(" #", 1)[0].rstrip() + os.environ.setdefault(key, value) diff --git a/harness2/substrate/__init__.py b/harness2/substrate/__init__.py index 3fa7f651..e048142f 100644 --- a/harness2/substrate/__init__.py +++ b/harness2/substrate/__init__.py @@ -12,14 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""The two solver substrates behind one interface (substrate/base.py). - -`OpencodeSubstrate` carries OPENCODE_SURFACE and three install modes; `CodexSubstrate` -carries CODEX_SURFACE and one. Benchmark adapters supply the launcher body. -""" +"""Agent-framework and command executors behind the substrate interface.""" from .base import Install, Launcher, RunRequest, Substrate +from .claude import CLAUDE_SURFACE, ClaudeSubstrate from .codex import CODEX_SURFACE, CodexSubstrate +from .command import CommandSubstrate from .opencode import OPENCODE_SURFACE, OpencodeSubstrate __all__ = [ @@ -31,4 +29,7 @@ "OPENCODE_SURFACE", "CodexSubstrate", "CODEX_SURFACE", + "ClaudeSubstrate", + "CLAUDE_SURFACE", + "CommandSubstrate", ] diff --git a/harness2/substrate/_native.py b/harness2/substrate/_native.py new file mode 100644 index 00000000..19966640 --- /dev/null +++ b/harness2/substrate/_native.py @@ -0,0 +1,109 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared CLI resolution and bounded native-file operations for executors.""" + +from __future__ import annotations + +import shlex +import shutil +import subprocess +from pathlib import Path + +from ..core import tree as HT +from ..settings import ConfigError + + +def cli_argv(binary: str, default: str) -> list[str]: + argv = shlex.split(binary or default) + resolved = shutil.which(argv[0]) if argv else None + if resolved is None: + raise ConfigError(f"executor binary not found: {binary or default}") + return [resolved, *argv[1:]] + + +def check_cli(binary: str, default: str) -> None: + argv = cli_argv(binary, default) + try: + subprocess.run([*argv, "--version"], check=True, capture_output=True, timeout=10) + except (OSError, subprocess.SubprocessError) as exc: + raise ConfigError(f"executor version check failed: {binary or default}") from exc + + +def checked_path(root: Path, rel: str) -> Path: + rel = HT.normalize_rel(rel) + if not rel or rel == ".": + raise ValueError("managed file path must not be empty") + path = root + for part in Path(rel).parts: + path = path / part + if path.is_symlink(): + raise ConfigError(f"managed path is a symlink: {rel}") + return path + + +def surface_files(surface: HT.SurfaceSpec, tree: HT.Tree) -> HT.Tree: + stubs = {rel for rel, _ in surface.stubs} + out = {} + for rel, content in tree.items(): + rel = HT.normalize_rel(rel) + if rel in stubs: + continue + if not HT.is_allowed(surface, rel): + raise ValueError(f"file is outside the {surface.name} surface: {rel}") + out[rel] = content + return out + + +def native_tree( + surface: HT.SurfaceSpec, + root: Path, + entries: tuple[str, ...], + rejected: list[str] | None = None, +) -> HT.Tree: + out = {} + stubs = {rel for rel, _ in surface.stubs} + for entry in entries: + path = root if entry == "." else checked_path(root, entry) + candidates = sorted(path.rglob("*")) if path.is_dir() else [path] + for candidate in candidates: + if not candidate.is_file() or "__pycache__" in candidate.parts: + continue + rel = candidate.relative_to(root).as_posix() + if rel in stubs: + continue + if HT.is_allowed(surface, rel): + out[rel] = checked_path(root, rel).read_text(encoding="utf-8", errors="replace") + elif rejected is not None: + rejected.append(rel) + return out + + +def export_files(root: Path, files: HT.Tree, base: HT.Tree) -> dict[str, list[str]]: + paths = {rel: checked_path(root, rel) for rel in files.keys() | base.keys()} + written = [] + deleted = [] + for rel, content in sorted(files.items()): + path = paths[rel] + if path.is_file() and path.read_text(encoding="utf-8", errors="replace") == content: + continue + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + written.append(rel) + for rel in sorted(base.keys() - files.keys()): + path = paths[rel] + if path.is_file(): + path.unlink() + deleted.append(rel) + return {"written": written, "deleted": deleted} diff --git a/harness2/substrate/claude.py b/harness2/substrate/claude.py new file mode 100644 index 00000000..01f2574b --- /dev/null +++ b/harness2/substrate/claude.py @@ -0,0 +1,341 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Claude Code's editable surface, native project files and stream-json renderer.""" + +from __future__ import annotations + +import json +import re +import shutil +import tempfile +from pathlib import Path +from typing import Mapping + +from .. import providers +from ..core import runmark as RM +from ..core import tree as HT +from ..core.tree import SurfaceSpec +from ..settings import ConfigError +from ._native import check_cli, checked_path, cli_argv, export_files, native_tree, surface_files +from .base import Install +from .codex import _TRUNC_FMT, _clip, _text_of + +CLAUDE_SURFACE = SurfaceSpec( + name="claude", + always_on=("CLAUDE.md",), + always_on_max=40_000, + always_on_cap_reason="CLAUDE.md is loaded whole into every session, and Claude Code " + "warns above 40,000 characters", + skill_dir=".claude/skills", + skill_name_rule="overrides_dir", + trigger_hint="", + autotrigger_file="CLAUDE.md", + trigger_line="", + subagent_requires_mode=False, + plugin_dir=None, + hooks_file=".claude/hooks.json", + hook_events=frozenset( + { + "PreToolUse", + "PostToolUse", + "Notification", + "UserPromptSubmit", + "Stop", + "SubagentStop", + "PreCompact", + "SessionStart", + "SessionEnd", + } + ), + compile_skill_scripts=True, + reject_empty_always_on=False, + script_dir=".claude/scripts", + skills_need_trigger=False, + stubs=(), +) + +NATIVE_PATHS = ( + "CLAUDE.md", + ".claude/skills", + ".claude/agents", + ".claude/scripts", + ".claude/settings.json", +) +_SETTINGS = ".claude/settings.json" + + +def _settings(root: Path) -> dict: + path = checked_path(root, _SETTINGS) + if not path.exists(): + return {} + try: + doc = json.loads(path.read_text(encoding="utf-8")) + except (ValueError, UnicodeError) as exc: + raise ConfigError(f"{path} is not valid JSON: {exc}") from exc + if not isinstance(doc, dict): + raise ConfigError(f"{path} must contain a JSON object") + return doc + + +def _agent_body(name: str, body: str) -> str: + frontmatter = re.match(r"\A---\r?\n(.*?)^---(?:\r?\n|$)", body, re.S | re.M) + if frontmatter: + name_line = re.search(r"^name:[ \t]*(.*)$", frontmatter.group(1), re.M) + if name_line and name_line.group(1).strip(): + return body + if name_line: + start = frontmatter.start(1) + name_line.start() + end = frontmatter.start(1) + name_line.end() + return body[:start] + f"name: {name}" + body[end:] + return body[: frontmatter.start(1)] + f"name: {name}\n" + body[frontmatter.start(1) :] + return f"---\nname: {name}\n---\n{body}" + + +def _native_files(tree: HT.Tree) -> tuple[HT.Tree, dict | None]: + files = {} + hooks = None + for rel, content in surface_files(CLAUDE_SURFACE, tree).items(): + if rel == CLAUDE_SURFACE.hooks_file: + try: + doc = json.loads(content) + except ValueError as exc: + raise ConfigError(f"{rel} is not valid JSON: {exc}") from exc + if not isinstance(doc, dict) or not isinstance(doc.get("hooks"), dict): + raise ConfigError(f'{rel} needs a top-level "hooks" object') + hooks = doc["hooks"] + elif rel.startswith("sub_agents/"): + name = rel.split("/")[1] + files[f".claude/agents/{name}.md"] = _agent_body(name, content) + else: + files[rel] = content + return files, hooks + + +class ClaudeSubstrate: + """Claude Code with isolated per-run global configuration.""" + + name = "claude" + NATIVE_PATHS = NATIVE_PATHS + + def __init__(self, bin: str = ""): + self.bin = bin + + def surface(self) -> SurfaceSpec: + return CLAUDE_SURFACE + + def materialise(self, tree_dir: Path, spec, ctx: Mapping[str, str]) -> Install: + if spec.install_mode != "claude_root": + raise ValueError( + f"ClaudeSubstrate needs install_mode 'claude_root', got {spec.install_mode!r}" + ) + staged = Path(tempfile.mkdtemp(prefix="harness2-claude-")) + try: + self.export_tree(HT.read_tree(CLAUDE_SURFACE, tree_dir), staged, base={}) + return Install(staged, RM.fingerprint(tree_dir), {"install_mode": spec.install_mode}) + except BaseException: + shutil.rmtree(staged, ignore_errors=True) + raise + + def import_tree(self, root: Path, *, rejected: list[str] | None = None) -> HT.Tree: + tree = native_tree( + CLAUDE_SURFACE, root, ("CLAUDE.md", ".claude/skills", ".claude/scripts"), rejected + ) + agents = checked_path(root, ".claude/agents") + for path in sorted(agents.rglob("*")): + if not path.is_file(): + continue + rel = path.relative_to(root).as_posix() + canonical = f"sub_agents/{path.stem}/AGENT.md" + if ( + path.parent == agents + and path.suffix == ".md" + and HT.is_allowed(CLAUDE_SURFACE, canonical) + ): + tree[canonical] = checked_path(root, rel).read_text( + encoding="utf-8", errors="replace" + ) + elif rejected is not None: + rejected.append(rel) + settings = _settings(root) + if "hooks" in settings: + if not isinstance(settings["hooks"], dict): + raise ConfigError(f'{_SETTINGS} needs a "hooks" object') + tree[CLAUDE_SURFACE.hooks_file] = ( + json.dumps({"hooks": settings["hooks"]}, indent=2) + "\n" + ) + return tree + + def export_tree(self, tree: HT.Tree, root: Path, *, base: HT.Tree) -> dict[str, list[str]]: + files, hooks = _native_files(tree) + old, old_hooks = _native_files(base) + if hooks is not None or old_hooks is not None: + settings = _settings(root) + merged = dict(settings) + if hooks is None: + merged.pop("hooks", None) + else: + merged["hooks"] = hooks + if merged != settings: + files[_SETTINGS] = json.dumps(merged, indent=2) + "\n" + return export_files(root, files, old) + + def strip_native(self, workspace: Path) -> None: + self.export_tree({}, workspace, base=self.import_tree(workspace)) + + def install_into(self, install: Install, workspace: Path) -> None: + self.export_tree(self.import_tree(install.staging_dir), workspace, base={}) + + def argv( + self, + prompt_file: Path, + workspace: Path, + model: str, + effort: str, + *, + provider: providers.Provider | None = None, + ) -> list[str]: + return [ + *cli_argv(self.bin, "claude"), + "-p", + "--output-format", + "stream-json", + "--verbose", + "--dangerously-skip-permissions", + "--setting-sources", + "project,local", + "--model", + model, + "--effort", + effort, + *( + ["--strict-mcp-config", "--no-session-persistence"] + if provider is not None and provider.type == "claude-login" + else [] + ), + ] + + def env( + self, base_env: Mapping[str, str], provider: providers.Provider, config_dir: Path + ) -> dict[str, str]: + login = provider.type == "claude-login" + if not login: + config_dir.mkdir(parents=True, exist_ok=True) + return { + **{k: v for k, v in base_env.items() if k not in providers.STRIP_ENV}, + **providers.claude_env(provider), + **({} if login else {"CLAUDE_CONFIG_DIR": str(config_dir)}), + } + + def read_trajectory( + self, stream_path: Path, *, obs_trunc: int, max_chars: int, args_trunc: int = 2000 + ) -> str: + return render_events( + stream_path, obs_trunc=obs_trunc, max_chars=max_chars, args_trunc=args_trunc + ) + + def final_message(self, stream_path: Path) -> str: + return final_message(stream_path) + + def cost_usd(self, stream_path: Path) -> float: + return cost_usd(stream_path) + + def preflight(self) -> None: + check_cli(self.bin, "claude") + + +def parse_events(path: Path) -> list[dict]: + try: + raw = path.read_text(encoding="utf-8", errors="replace").replace("\x00", "") + except OSError as exc: + return [{"type": "_read_error", "message": str(exc)}] + events = [] + for line in raw.splitlines(): + try: + event = json.loads(line) + except ValueError: + continue + if isinstance(event, dict): + events.append(event) + return events + + +def render_events(path: Path, *, obs_trunc: int, max_chars: int, args_trunc: int = 2000) -> str: + lines = [] + for event in parse_events(path): + kind = event.get("type") + if kind == "_read_error": + return f"(failed to read trajectory: {event.get('message')})" + if kind == "system": + if event.get("subtype") == "init": + lines.append(f"[HARNESS] init | model={event.get('model')} cwd={event.get('cwd')}") + continue + if kind == "result": + if event.get("is_error"): + text = event.get("result") or _text_of(event.get("errors")) or event.get("subtype") + lines.append(f"[ERROR] {_clip(text, obs_trunc)}") + elif event.get("result"): + lines.append(f"[ASSISTANT] {_clip(event['result'], obs_trunc)}") + continue + message = event.get("message") + if kind not in ("assistant", "user") or not isinstance(message, dict): + continue + content = message.get("content", []) + if isinstance(content, str): + content = [{"type": "text", "text": content}] + if not isinstance(content, list): + continue + for block in content: + if not isinstance(block, dict): + continue + block_type = block.get("type") + if kind == "assistant" and block_type in ("text", "thinking"): + tag = "ASSISTANT" if block_type == "text" else "REASONING" + text = block.get("text" if block_type == "text" else "thinking") + lines.append(f"[{tag}] {_clip(text, obs_trunc)}") + elif kind == "assistant" and block_type == "tool_use": + args = json.dumps(block.get("input", {}), ensure_ascii=False) + lines.append(f"[ACTION] {block.get('name', 'tool')}: {_clip(args, args_trunc)}") + elif kind == "user" and block_type == "tool_result": + tag = "OBSERVATION ERROR" if block.get("is_error") else "OBSERVATION" + lines.append(f"[{tag}] {_clip(_text_of(block.get('content')), obs_trunc)}") + elif kind == "user" and block_type == "text": + lines.append(f"[USER] {_clip(block.get('text'), obs_trunc)}") + rendered = "\n".join(lines).strip() + if not rendered: + return "(no parseable events in trajectory)" + if len(rendered) > max_chars: + return rendered[:max_chars] + "\n" + _TRUNC_FMT.format(n=len(rendered) - max_chars) + return rendered + + +def final_message(path: Path) -> str: + last = "" + result = None + for event in parse_events(path): + if event.get("type") == "result" and isinstance(event.get("result"), str): + result = event["result"] + elif event.get("type") == "assistant" and isinstance(event.get("message"), dict): + text = _text_of(event["message"].get("content")) + if text.strip(): + last = text + return result if result is not None else last + + +def cost_usd(path: Path) -> float: + for event in reversed(parse_events(path)): + if event.get("type") == "result": + value = event.get("total_cost_usd") + return float(value) if isinstance(value, (int, float)) else 0.0 + return 0.0 diff --git a/harness2/substrate/codex.py b/harness2/substrate/codex.py index fd34eacd..591d6254 100644 --- a/harness2/substrate/codex.py +++ b/harness2/substrate/codex.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""CodexSubstrate: codex-cli (config.CODEX_VERSION) as a solver substrate. +"""CodexSubstrate: codex-cli as a solver substrate. Carries the c1..c8 surface (CODEX_SURFACE), the one install mode (codex_root: AGENTS.md, -.agents, scripts and .codex copied to the workspace root, where `codex exec -C ` finds +.agents and .codex copied to the workspace root, where `codex exec -C ` finds them), the `codex exec` flag contract, and the codex thread-item trajectory renderer. Per-deployment differences are values in Install.extras, never subclasses. """ @@ -23,17 +23,16 @@ from __future__ import annotations import json -import os import shutil -import socket import tempfile from pathlib import Path from typing import Mapping -from .. import config +from .. import providers from ..core import runmark as RM from ..core import tree as HT from ..core.tree import SurfaceSpec +from ._native import check_cli, cli_argv, export_files, native_tree, surface_files from .base import Install, Launcher, RunRequest CODEX_SURFACE = SurfaceSpec( @@ -72,11 +71,12 @@ ), compile_skill_scripts=True, reject_empty_always_on=True, + script_dir=".agents/scripts", stubs=( ( - "scripts/README.md", - "\n", + ".agents/scripts/README.md", + "\n", ), ( "sub_agents/README.md", @@ -103,16 +103,19 @@ ) -CODEX_ROOT_ENTRIES = ("AGENTS.md", ".agents", "scripts", ".codex") +CODEX_ROOT_ENTRIES = ("AGENTS.md", ".agents", ".codex") +NATIVE_PATHS = ("AGENTS.md", ".agents/skills", ".agents/scripts", ".codex/hooks.json") class CodexSubstrate: """codex-cli behind the one Substrate interface.""" name = "cx" + NATIVE_PATHS = NATIVE_PATHS - def __init__(self, launcher: Launcher | None = None): + def __init__(self, launcher: Launcher | None = None, *, bin: str = ""): self._launcher = launcher + self.bin = bin def surface(self) -> SurfaceSpec: return CODEX_SURFACE @@ -129,14 +132,11 @@ def materialise(self, tree_dir: Path, spec, ctx: Mapping[str, str]) -> Install: if not HT.is_tree(CODEX_SURFACE, tree_dir): raise ValueError(f"{tree_dir} is not a codex harness tree (no AGENTS.md)") staged = Path(tempfile.mkdtemp(prefix="adapt-cx-root-")) - for entry in CODEX_ROOT_ENTRIES: - src = tree_dir / entry - if not src.exists(): - continue - if src.is_dir(): - shutil.copytree(src, staged / entry) - else: - shutil.copy2(src, staged / entry) + try: + self.export_tree(HT.read_tree(CODEX_SURFACE, tree_dir), staged, base={}) + except BaseException: + shutil.rmtree(staged, ignore_errors=True) + raise return Install( staging_dir=staged, fingerprint=RM.fingerprint(tree_dir), @@ -147,7 +147,13 @@ def materialise(self, tree_dir: Path, spec, ctx: Mapping[str, str]) -> Install: ) def exec_argv( - self, prompt: str, workspace: Path, model: str, *, reasoning_effort: str = "high" + self, + prompt: str, + workspace: Path, + model: str, + *, + reasoning_effort: str = "high", + bin: str = "", ) -> list[str]: """The reference `codex exec` argv; the real launchers build their own. @@ -155,7 +161,7 @@ def exec_argv( Callers use stdin=DEVNULL and `exec_env`. """ return [ - str(Path(config.CODEX_DIST) / "codex"), + *cli_argv(bin or self.bin, "codex"), "exec", prompt, "--model", @@ -173,16 +179,86 @@ def exec_argv( str(workspace), ] - def exec_env(self, base_env: Mapping[str, str], *, codex_home: Path) -> dict[str, str]: + def exec_env( + self, base_env: Mapping[str, str], *, codex_home: Path, bin: str = "" + ) -> dict[str, str]: """The reference `codex exec` environment; the real launchers build their own. The custom provider comes from CODEX_HOME, not a global OpenAI URL override. """ - env = dict(base_env) - env.pop("OPENAI_BASE_URL", None) - env["CODEX_HOME"] = str(codex_home) - env["PATH"] = str(Path(config.CODEX_DIST) / "codex-path") + os.pathsep + env.get("PATH", "") - return env + return self.env(base_env, codex_home) + + def import_tree(self, root: Path, *, rejected: list[str] | None = None) -> HT.Tree: + return native_tree(CODEX_SURFACE, root, NATIVE_PATHS, rejected) + + def export_tree(self, tree: HT.Tree, root: Path, *, base: HT.Tree) -> dict[str, list[str]]: + files = { + k: v + for k, v in surface_files(CODEX_SURFACE, tree).items() + if not k.startswith("sub_agents/") + } + old = { + k: v + for k, v in surface_files(CODEX_SURFACE, base).items() + if not k.startswith("sub_agents/") + } + return export_files(root, files, old) + + def strip_native(self, workspace: Path) -> None: + self.export_tree({}, workspace, base=self.import_tree(workspace)) + + def install_into(self, install: Install, workspace: Path) -> None: + self.export_tree(self.import_tree(install.staging_dir), workspace, base={}) + + def argv( + self, + prompt_file: Path, + workspace: Path, + model: str, + effort: str, + *, + provider: providers.Provider | None = None, + ) -> list[str]: + return [ + *cli_argv(self.bin, "codex"), + "exec", + "--json", + "--skip-git-repo-check", + "--dangerously-bypass-approvals-and-sandbox", + "--dangerously-bypass-hook-trust", + "--ephemeral", + *providers.codex_args(provider, model, effort), + "-C", + str(workspace), + "-", + ] + + def write_home(self, home: Path, provider: providers.Provider, model: str, effort: str) -> None: + content = providers.codex_config(provider, model, effort) + home.mkdir(parents=True, exist_ok=True) + (home / "config.toml").write_text(content, encoding="utf-8") + + def env( + self, + base_env: Mapping[str, str], + home: Path | None, + *, + provider: providers.Provider | None = None, + ) -> dict[str, str]: + stripped = providers.STRIP_ENV + if provider is not None and provider.type == "chatgpt-login": + stripped += ("OPENAI_API_KEY", "CODEX_API_KEY") + home = providers.codex_login_home(provider) + return { + **{k: v for k, v in base_env.items() if k not in stripped}, + **({"CODEX_HOME": str(home)} if home is not None else {}), + } + + def final_message(self, stream_path: Path) -> str: + return final_message(stream_path) + + def cost_usd(self, stream_path: Path) -> float: + return 0.0 def execute( self, task_ref: str, install: Install, run_ids: list[str], timeout: int @@ -204,28 +280,8 @@ def read_trajectory( Path(stream_path), obs_trunc=obs_trunc, max_chars=max_chars, args_trunc=args_trunc ) - def preflight(self, *, require_dist: bool = False, require_bridge: bool = False) -> None: - """Check the CLI and Responses endpoint, including container configuration.""" - from urllib.parse import urlparse - - config.require("HARNESS2_CODEX_BASE_URL", needed_by="Codex Responses endpoint") - config.require("HARNESS2_CODEX_API_KEY", needed_by="Codex endpoint authentication") - url = urlparse(config.CODEX_BASE_URL) - if url.scheme not in ("http", "https") or not url.hostname: - raise config.ConfigError("HARNESS2_CODEX_BASE_URL must be an HTTP(S) URL") - binary = Path(config.CODEX_DIST) / "codex" - if not binary.is_file(): - raise RuntimeError( - "Codex is not installed; run scripts/setup_benchmark.py with --substrate cx" - ) - if require_bridge and not config.CODEX_CONTAINER_URL: - raise RuntimeError( - "Set HARNESS2_CODEX_CONTAINER_URL to a container-accessible Responses endpoint" - ) - with socket.create_connection( - (url.hostname, url.port or (443 if url.scheme == "https" else 80)), timeout=5 - ): - pass + def preflight(self) -> None: + check_cli(self.bin, "codex") _TRUNC_FMT = "…(+{n} chars — render truncation)" @@ -268,7 +324,12 @@ def parse_items(path: Path) -> list[dict]: raw = Path(path).read_text(encoding="utf-8", errors="replace") except OSError as e: return [{"type": "_read_error", "message": str(e)}] - raw = raw.replace("\x00", "") + return parse_items_text(raw) + + +def parse_items_text(text: str) -> list[dict]: + """Parse an exec JSONL string, keeping the last state of each thread item.""" + raw = text.replace("\x00", "") events: list[dict] = [] by_id: dict[str, int] = {} diff --git a/harness2/substrate/command.py b/harness2/substrate/command.py new file mode 100644 index 00000000..80fe7e25 --- /dev/null +++ b/harness2/substrate/command.py @@ -0,0 +1,128 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A shell-command executor with an explicitly editable set of project files.""" + +from __future__ import annotations + +import shlex +import shutil +import subprocess +import tempfile +from pathlib import Path +from typing import Mapping + +from ..core import runmark as RM +from ..core import tree as HT +from ..core.tree import SurfaceSpec +from ._native import export_files, native_tree, surface_files +from .base import Install +from .codex import _clip + + +class CommandSubstrate: + """Run a command template inside the task workspace.""" + + name = "command" + + def __init__(self, template: str, editable: tuple[str, ...]): + if not editable: + raise ValueError("command executor needs non-empty editable globs") + self.template = template + self.editable = editable + self.NATIVE_PATHS = tuple(f":(glob){g}" for g in editable) + + def surface(self) -> SurfaceSpec: + return SurfaceSpec( + name="command", + always_on=(), + always_on_max=200_000, + always_on_cap_reason="", + skill_dir=".skills-unused", + skill_name_rule="overrides_dir", + trigger_hint="", + autotrigger_file="", + trigger_line="", + subagent_requires_mode=False, + plugin_dir=None, + hooks_file=None, + editable_globs=self.editable, + skills_need_trigger=False, + ) + + def materialise(self, tree_dir: Path, spec, ctx: Mapping[str, str]) -> Install: + if spec.install_mode != "command_files": + raise ValueError( + f"CommandSubstrate needs install_mode 'command_files', got {spec.install_mode!r}" + ) + staged = Path(tempfile.mkdtemp(prefix="harness2-command-")) + try: + self.export_tree(self.import_tree(tree_dir), staged, base={}) + return Install(staged, RM.fingerprint(tree_dir), {"install_mode": spec.install_mode}) + except BaseException: + shutil.rmtree(staged, ignore_errors=True) + raise + + def import_tree(self, root: Path, *, rejected: list[str] | None = None) -> HT.Tree: + return native_tree(self.surface(), root, (".",), rejected) + + def export_tree(self, tree: HT.Tree, root: Path, *, base: HT.Tree) -> dict[str, list[str]]: + surface = self.surface() + return export_files(root, surface_files(surface, tree), surface_files(surface, base)) + + def strip_native(self, workspace: Path) -> None: + self.export_tree({}, workspace, base=self.import_tree(workspace)) + + def install_into(self, install: Install, workspace: Path) -> None: + self.export_tree(self.import_tree(install.staging_dir), workspace, base={}) + + def run( + self, + prompt_file: Path, + workspace: Path, + run_dir: Path, + timeout: int, + env: Mapping[str, str], + ) -> int: + command = self.template.format( + workspace=shlex.quote(str(workspace)), + prompt_file=shlex.quote(str(prompt_file)), + output=shlex.quote(str(workspace / "output")), + ) + run_dir.mkdir(parents=True, exist_ok=True) + with (run_dir / "stream.log").open("w", encoding="utf-8") as stream: + return subprocess.run( + command, + shell=True, + cwd=workspace, + stdout=stream, + stderr=subprocess.STDOUT, + timeout=timeout, + env=dict(env), + ).returncode + + def read_trajectory( + self, stream_path: Path, *, obs_trunc: int, max_chars: int, args_trunc: int = 2000 + ) -> str: + text = stream_path.with_name("stream.log").read_text(encoding="utf-8", errors="replace") + tail = text[-obs_trunc:] if obs_trunc > 0 else text + return _clip(f"[OBSERVATION] {tail}", max_chars) + + def final_message(self, stream_path: Path) -> str: + return stream_path.with_name("stream.log").read_text(encoding="utf-8", errors="replace")[ + -4000: + ] + + def cost_usd(self, stream_path: Path) -> float: + return 0.0 diff --git a/harness2/substrate/opencode.py b/harness2/substrate/opencode.py index ad0a2ea4..29b1b0b6 100644 --- a/harness2/substrate/opencode.py +++ b/harness2/substrate/opencode.py @@ -14,7 +14,7 @@ """OpencodeSubstrate: the OpenCode agent framework as a solver substrate. -Carries the c1..c8 surface (OPENCODE_SURFACE), the three install modes, the bun-run CLI +Carries the c1..c8 surface (OPENCODE_SURFACE), the three install modes, the CLI contract, and the one opencode event-stream trajectory renderer. The install modes are different delivery channels for the always-on text and are not interchangeable: changing the one a benchmark uses moves that benchmark's baseline. @@ -33,10 +33,10 @@ from pathlib import Path from typing import Mapping -from .. import config from ..core import runmark as RM from ..core import tree as HT from ..core.tree import SurfaceSpec +from ._native import check_cli, cli_argv from .base import Install, Launcher, RunRequest OPENCODE_SURFACE = SurfaceSpec( @@ -175,8 +175,9 @@ class OpencodeSubstrate: name = "oc" - def __init__(self, launcher: Launcher | None = None): + def __init__(self, launcher: Launcher | None = None, *, bin: str = ""): self._launcher = launcher + self.bin = bin def surface(self) -> SurfaceSpec: return OPENCODE_SURFACE @@ -233,19 +234,14 @@ def run_argv( variant: str | None = None, agent: str = "", session: str | None = None, + bin: str = "", ) -> list[str]: """The reference argv; the real launchers build their own. - The upstream checkout runs from source through bun: there is no `opencode` binary in - the contract. With `agent=""` the rendered AGENTS.md appends to the provider prompt. + With `agent=""` the rendered AGENTS.md appends to the provider prompt. """ argv = [ - str(config.BUN), - "run", - "--cwd", - str(config.OPENCODE_DIR / "packages" / "opencode"), - "--conditions=browser", - "src/index.ts", + *cli_argv(bin or self.bin, "opencode"), "run", prompt, "--dir", @@ -305,19 +301,7 @@ def read_trajectory( ) def preflight(self) -> None: - """Checks the OpenCode checkout and bun exist; starts nothing.""" - oc = config.OPENCODE_DIR - if not (oc / "packages" / "opencode" / "src" / "index.ts").is_file(): - raise RuntimeError( - f"OpenCode checkout missing at {oc} (no packages/opencode/src/index.ts): " - f"run scripts/setup_benchmark.py to fetch the pinned checkout " - f"at {config.DEPENDENCY_PINS['opencode'][:12]} (v1.14.18)." - ) - if not config.BUN.is_file(): - raise RuntimeError( - f"bun not found at {config.BUN}: install bun on PATH or at ~/.bun/bin/bun " - f"(the upstream OpenCode is run from source through bun)." - ) + check_cli(self.bin, "opencode") _TRUNC_FMT = "…(+{n} chars — render truncation)" diff --git a/harness2/suite.py b/harness2/suite.py new file mode 100644 index 00000000..99af3eab --- /dev/null +++ b/harness2/suite.py @@ -0,0 +1,740 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The built-in suite for tasks in the user's project or supplied files.""" + +from __future__ import annotations + +import csv +import hashlib +import json +import os +import re +import shutil +import subprocess +import tempfile +import threading +import time +import tomllib +import warnings +from dataclasses import replace +from pathlib import Path + +from . import providers, workspace +from .core import events, runmark, tree +from .core.library import slug +from .core.proposer import Proposer, ProposerError, _parse_json +from .core.spec import BenchPrompts, BenchSpec, ChannelSpec +from .registry import SuiteContext +from .settings import ConfigError, Settings +from .substrate._native import checked_path, cli_argv +from .substrate.claude import ClaudeSubstrate, parse_events +from .substrate.codex import CodexSubstrate +from .substrate.command import CommandSubstrate + +DESCRIPTION = "Tasks" + +PROJECT_PROMPTS = BenchPrompts( + work=( + "Software engineering in this repository: the changes, fixes and investigations a " + "maintainer asks for. The request controls. The result must be a change a maintainer " + "could merge: complete, consistent with the codebase's conventions, and checked by " + "running it." + ), + domain_word="area", + deliverable_word="change", + inv_levers=( + "- locate the code, tests and conventions that govern the request before editing; read " + "before writing\n" + "- find and use the project's own way to build, run and test; run it before finishing " + "and read what it reports\n" + "- keep the change scoped to the request; do not reformat, rename or refactor what the " + "request did not ask for\n" + "- when a command fails, read the error and change approach instead of repeating it" + ), + dom_levers=( + "- the codebase's architecture, module boundaries and the patterns existing code follows\n" + "- the invariants a change in this area must keep: interfaces, data formats, error " + "handling, compatibility\n" + "- where the tests for this area live and what a convincing test for this kind of " + "change looks like" + ), + yardstick=( + "Did the run find the code that governs the request, make the change the request asked " + "for and nothing else, follow the codebase's conventions, and verify the change by " + "running it before stopping?" + ), + selection_checks=( + "Prefer the change that does what the request asks, completely; fits the surrounding " + "code's conventions; keeps the behaviour the request did not ask to change; and shows " + "it was verified. A smaller change that fully solves the request beats a larger one." + ), +) + +FILES_PROMPTS = BenchPrompts( + work=( + "Producing the documents, analyses or data a request asks for from the supplied files. " + "The request and the source files control. The result must be ready to use: complete, " + "accurate to the sources, and in exactly the files and format requested." + ), + domain_word="area", + deliverable_word="deliverable", + inv_levers=( + "- inventory the supplied files and map each requested output to the sources that " + "control it\n" + "- write every requested file early enough that it exists before the budget runs out, " + "then reopen it and check that it is complete and parses\n" + "- use code for counting, arithmetic and format conversion instead of doing it by hand\n" + "- when a tool fails on a file, read the error and try one bounded alternative" + ), + dom_levers=( + "- what a complete deliverable of this kind contains and how its audience will use it\n" + "- which source governs when two sources disagree, and how to state what the sources " + "do not say\n" + "- the checks a practitioner runs on this kind of work before handing it over" + ), + yardstick=( + "Did the run read the sources that control the request, produce every requested file in " + "usable form, keep every figure and claim traceable to a source, and check the finished " + "files before stopping?" + ), + selection_checks=( + "Prefer the deliverable that answers every part of the request, stays accurate to the " + "supplied files, is in exactly the requested files and format, and needs no repair " + "before use." + ), +) + +FILES_PROMPT_SUFFIX = "\n\nSave your deliverables in the output/ directory." + +RUBRIC_MISSION = """\ +You are grading one result of a task against a rubric. + +context/task.md is the request the agent received. context/rubric.md lists the criteria. +context/deliverables/ holds the result: the files it produced or its patch, and its final +message. + +Read the request, the rubric and the whole result. For each criterion, decide whether the +result meets it, judging only what the result actually contains -- not what it claims or +promises. A criterion the result does not address is not met. + +Give your reasoning per criterion, then end with exactly one JSON object on the last line: +{"n_passed": , "n_criteria": , "notes": ""} +""" + +TASK_TEMPLATE = "TODO: describe the task the way you would ask the agent to do it.\n" +# suite.check reports a task whose task.md still starts with "TODO:": +# "task {id}: task.md still holds the template text" + +BASE_DRIFT_WARNING = ( + "the project's harness has changed since this run started; the run keeps using the copy " + "it started with in {path}" +) + +_TASK_KEYS = {"domain", "split", "workspace", "ref", "deliverable", "check", "timeout"} + + +def _task_id(task_id: str) -> str: + if not isinstance(task_id, str) or not re.fullmatch(r"[a-z0-9][a-z0-9._-]*", task_id): + raise ConfigError(f"invalid task id {task_id!r}; use [a-z0-9][a-z0-9._-]*") + return task_id + + +def _task_config(task_id: str, values: dict, *, files: bool) -> dict: + _task_id(task_id) + for key in values: + if key not in _TASK_KEYS: + raise ConfigError(f"task {task_id}: unknown task.toml key {key!r}") + for key, value in values.items(): + if key == "timeout": + if type(value) is not int or value <= 0: + raise ConfigError(f"task {task_id}: timeout must be a positive integer") + elif not isinstance(value, str) or not value.strip(): + raise ConfigError(f"task {task_id}: {key} must be a non-empty string") + config = {"domain": "default", "workspace": "files" if files else "project", "ref": "HEAD"} + config.update(values) + config.setdefault("deliverable", "diff" if config["workspace"] == "project" else "files") + for key, choices in ( + ("split", ("train", "test")), + ("workspace", ("project", "files")), + ("deliverable", ("diff", "files", "message")), + ): + if key in config and config[key] not in choices: + raise ConfigError(f"task {task_id}: {key} must be one of {', '.join(choices)}") + return config + + +def _read_task(tasks_dir: Path, task_id: str) -> dict: + _task_id(task_id) + task_dir = checked_path(tasks_dir, task_id) + try: + checked_path(task_dir, "task.md").read_text(encoding="utf-8") + config_path = checked_path(task_dir, "task.toml") + values = ( + tomllib.loads(config_path.read_text(encoding="utf-8")) if config_path.exists() else {} + ) + except (OSError, ValueError, UnicodeError) as exc: + raise ConfigError(f"task {task_id}: cannot read task: {exc}") from exc + return _task_config(task_id, values, files=(task_dir / "files").is_dir()) + + +def list_tasks(tasks_dir: Path) -> list[dict]: + if not tasks_dir.exists(): + return [] + rows = [] + for path in sorted(tasks_dir.iterdir()): + if not path.is_dir(): + continue + config = _read_task(tasks_dir, path.name) + rows.append( + { + "id": path.name, + "domain": config["domain"], + "split": config.get("split"), + "workspace": config["workspace"], + "has_judge": bool(config.get("check") or (path / "rubric.md").is_file()), + } + ) + return rows + + +def _write_task(task_dir: Path, prompt: str, values: dict) -> None: + prompt_path = checked_path(task_dir, "task.md") + config_path = checked_path(task_dir, "task.toml") + files_path = checked_path(task_dir, "files") + task_dir.mkdir(parents=True, exist_ok=True) + prompt_path.write_text(prompt, encoding="utf-8") + config_path.write_text( + "".join( + f"{key} = {json.dumps(value, ensure_ascii=False)}\n" for key, value in values.items() + ), + encoding="utf-8", + ) + if values.get("workspace") == "files": + files_path.mkdir(exist_ok=True) + + +def new_task(tasks_dir: Path, task_id: str, *, files: bool, domain: str, check: str | None) -> Path: + values = {"domain": domain, "workspace": "files" if files else "project"} + if check is not None: + values["check"] = check + _task_config(task_id, values, files=files) + task_dir = checked_path(tasks_dir, task_id) + if task_dir.exists(): + raise ConfigError(f"task {task_id} already exists") + _write_task(task_dir, TASK_TEMPLATE, values) + return task_dir + + +def import_tasks(tasks_dir: Path, source: Path, *, force: bool) -> list[str]: + try: + with source.open(encoding="utf-8", newline="") as stream: + if source.suffix.lower() == ".jsonl": + records = [json.loads(line) for line in stream if line.strip()] + elif source.suffix.lower() == ".csv": + records = list(csv.DictReader(stream)) + else: + raise ConfigError("task imports must use .jsonl or .csv") + except (OSError, ValueError, UnicodeError, csv.Error) as exc: + raise ConfigError(f"cannot import {source}: {exc}") from exc + pending = {} + for record in records: + if not isinstance(record, dict): + raise ConfigError("each imported task must be an object") + task_id = _task_id(record.get("id")) + prompt = record.get("prompt") + if not isinstance(prompt, str) or not prompt.strip(): + raise ConfigError(f"task {task_id}: prompt is required") + values = {k: v for k, v in record.items() if k not in ("id", "prompt") and v != ""} + if source.suffix.lower() == ".csv" and "timeout" in values: + try: + values["timeout"] = int(values["timeout"]) + except (TypeError, ValueError) as exc: + raise ConfigError(f"task {task_id}: timeout must be a positive integer") from exc + _task_config(task_id, values, files=(tasks_dir / task_id / "files").is_dir()) + if task_id in pending: + raise ConfigError(f"duplicate task id {task_id!r} in import") + task_dir = checked_path(tasks_dir, task_id) + if task_dir.exists() and not force: + raise ConfigError(f"task {task_id} already exists; use --force to replace it") + pending[task_id] = (prompt, values) + for task_id, (prompt, values) in pending.items(): + _write_task(tasks_dir / task_id, prompt, values) + return list(pending) + + +def _channel(settings: Settings, role: str) -> ChannelSpec: + values = settings.role(role) + kind = values["channel"] + if kind == "human" and role != "selector": + raise ConfigError("the human channel is only allowed for selector") + provider = None + if kind != "human": + provider_name, _ = providers.split_model(values["model"]) + provider = settings.provider(provider_name) + try: + return ChannelSpec( + kind=kind, + model=values["model"], + bin=values["bin"], + effort=values["effort"], + timeout=values["timeout"], + steps=values["steps"], + provider=provider, + ) + except ValueError as exc: + raise ConfigError(str(exc)) from exc + + +class GenericAdapter: + """Run local task definitions through the shared adaptation method.""" + + def __init__(self, ctx: SuiteContext): + self.ctx = ctx + self.settings = ctx.settings + self.tasks_dir = self.settings.path("paths.tasks") + self.executor = self.settings.section("executor") + name = self.executor["name"] + if name == "claude": + self.substrate = ClaudeSubstrate(bin=self.executor["bin"]) + install_mode = "claude_root" + elif name == "codex": + self.substrate = CodexSubstrate(bin=self.executor["bin"]) + install_mode = "codex_root" + elif name == "command": + try: + self.substrate = CommandSubstrate( + self.executor["command"], tuple(self.executor["editable"]) + ) + except ValueError as exc: + raise ConfigError(str(exc)) from exc + install_mode = "command_files" + else: + raise ConfigError("the tasks suite runs on claude, codex or command") + self.spec = BenchSpec( + name="tasks", + runs_root=ctx.run_root, + install_mode=install_mode, + prompts=PROJECT_PROMPTS, + channels={ + "proposer": _channel(self.settings, "improver"), + "selector": _channel(self.settings, "selector"), + "reflector": _channel(self.settings, "reflector"), + }, + domain_word="area", + leak_globs=("rubric.md", "check/**"), + allow_subagents=name == "claude", + obs_trunc=self.settings.get("run.obs_trunc"), + ) + self._warned_drift = False + self._base_lock = threading.Lock() + + def tasks(self, split: str = "all") -> list[str]: + if split not in ("all", "train", "test"): + raise ConfigError(f"unknown task split {split!r}; use all, train or test") + return [ + row["id"] + for row in list_tasks(self.tasks_dir) + if split == "all" or row["split"] == split + ] + + def domain_of(self, task_id: str) -> str: + return _read_task(self.tasks_dir, task_id)["domain"] + + def task_prompt(self, task_id: str) -> str: + _read_task(self.tasks_dir, task_id) + return (self.tasks_dir / task_id / "task.md").read_text(encoding="utf-8") + + def task_files(self, task_id: str) -> Path | None: + _read_task(self.tasks_dir, task_id) + path = checked_path(self.tasks_dir / task_id, "files") + return path if path.is_dir() else None + + def prompts_for(self, task_id: str) -> BenchPrompts: + config = _read_task(self.tasks_dir, task_id) + prompts = PROJECT_PROMPTS if config["workspace"] == "project" else FILES_PROMPTS + steer = self.settings.get("improver.steer") + if steer.startswith("@"): + path = self.ctx.project / Path(steer[1:]).expanduser() + try: + steer = path.read_text(encoding="utf-8") + except (OSError, UnicodeError) as exc: + raise ConfigError(f"cannot read improver.steer file {path}: {exc}") from exc + try: + return replace( + prompts, + work=self.settings.get("suite.work") or prompts.work, + deliverable_word="answer" + if config["deliverable"] == "message" + else prompts.deliverable_word, + domain_guidance=steer, + ) + except ValueError as exc: + raise ConfigError("improver.steer / suite.work: " + str(exc)) from exc + + def base_harness(self) -> Path: + dest = self.spec.runs_root / "base_harness" + surface = self.substrate.surface() + with self._base_lock: + current = self.substrate.import_tree(self.settings.path("paths.harness")) + if not dest.exists(): + tree.write_tree(surface, current, dest, stubs=True) + elif ( + not self._warned_drift + and tree.read_tree(surface, dest) != {a: "" for a in surface.always_on} | current + ): + warnings.warn(BASE_DRIFT_WARNING.format(path=dest), stacklevel=2) + self._warned_drift = True + return dest + + def _workspace(self, task_id: str, config: dict): + if config["workspace"] == "project": + return workspace.project_workspace(self.ctx.project, config["ref"]) + return workspace.files_workspace(self.task_files(task_id)) + + def execute( + self, task_id: str, harness_dir: Path | None, run_id: str, *, timeout: int | None = None + ) -> Path: + config = _read_task(self.tasks_dir, task_id) + harness_dir = harness_dir if harness_dir is not None else self.base_harness() + run_name = slug(run_id) + if not run_name or run_name in (".", "..") or "\\" in run_name: + raise ConfigError(f"invalid rollout id {run_id!r}") + run_dir = self.spec.runs_root / "rollouts" / run_name + if runmark.is_complete(run_dir, harness_dir=harness_dir): + return run_dir + if run_dir.exists(): + shutil.rmtree(run_dir) + run_dir.mkdir(parents=True) + prompt_file = run_dir / "prompt.md" + prompt = self.task_prompt(task_id) + if config["workspace"] == "files": + prompt += FILES_PROMPT_SUFFIX + prompt_file.write_text(prompt, encoding="utf-8") + wall = config.get("timeout") or timeout or self.executor["timeout"] + started = time.monotonic() + timed_out = False + with self._workspace(task_id, config) as ws: + self.substrate.strip_native(ws) + if config["workspace"] == "project": + state = ws / ".harness2" + if state.is_symlink() or state.is_file(): + state.unlink() + else: + shutil.rmtree(state, ignore_errors=True) + install = self.substrate.materialise(harness_dir, self.spec, {}) + try: + self.substrate.install_into(install, ws) + finally: + shutil.rmtree(install.staging_dir, ignore_errors=True) + if config["workspace"] == "project": + workspace.commit_all(ws, "harness2: candidate harness") + base = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=ws, text=True + ).strip() + env = dict(os.environ) + if self.executor["name"] != "command": + provider_name, model = providers.split_model(self.executor["model"]) + provider = self.settings.provider(provider_name) + if self.executor["name"] == "claude": + env = self.substrate.env(env, provider, run_dir / "claude-home") + else: + home = None + if provider.type != "chatgpt-login": + home = run_dir / "codex-home" + self.substrate.write_home(home, provider, model, self.executor["effort"]) + env = self.substrate.env(env, home, provider=provider) + argv = self.substrate.argv( + prompt_file, ws, model, self.executor["effort"], provider=provider + ) + try: + if self.executor["name"] == "command": + exit_code = self.substrate.run(prompt_file, ws, run_dir, wall, env) + else: + with ( + prompt_file.open("rb") as stdin, + (run_dir / "stream.jsonl").open("wb") as stdout, + (run_dir / "stderr.log").open("wb") as stderr, + ): + exit_code = subprocess.run( + argv, + cwd=ws, + env=env, + stdin=stdin, + stdout=stdout, + stderr=stderr, + timeout=wall, + ).returncode + except subprocess.TimeoutExpired: + exit_code, timed_out = -1, True + except OSError as exc: + (run_dir / "stderr.log").write_text(str(exc), encoding="utf-8") + exit_code = -1 + stream = run_dir / "stream.jsonl" + failed = exit_code != 0 or any( + event.get("type") in ("error", "turn.failed") + or ( + event.get("type") == "result" + and (event.get("is_error") or event.get("subtype") not in (None, "success")) + ) + for event in parse_events(stream) + ) + if failed: + tail = "".join( + path.read_text(encoding="utf-8", errors="replace")[-16000:] + for path in (stream, run_dir / "stderr.log") + if path.is_file() + ) + if providers.USAGE_LIMIT.search(tail): + shutil.rmtree(run_dir) + events.request_stop("usage_limit") + raise events.Stopped("usage_limit") + if config["workspace"] == "project": + (run_dir / "deliverable.patch").write_text( + workspace.diff_since(ws, base, (*self.substrate.NATIVE_PATHS, ".harness2")), + encoding="utf-8", + ) + if (ws / "output").is_dir(): + shutil.copytree(ws / "output", run_dir / "output") + (run_dir / "final.md").write_text( + self.substrate.final_message(stream), encoding="utf-8" + ) + meta = { + "task_id": task_id, + "deliverable": config["deliverable"], + "ok": exit_code == 0 and not timed_out, + "exit_code": exit_code, + "wall_s": round(time.monotonic() - started, 3), + "cost_usd": self.substrate.cost_usd(stream), + "timed_out": timed_out, + } + (run_dir / "meta.json").write_text(json.dumps(meta, indent=2), encoding="utf-8") + runmark.mark_complete(run_dir, run_id=run_id, harness_dir=harness_dir) + return run_dir + + def rollout_cost(self, run_dir: Path) -> float: + return float(json.loads((run_dir / "meta.json").read_text())["cost_usd"]) + + def delivered(self, run_dir: Path) -> bool: + if not (run_dir / "meta.json").is_file(): + return False + kind = json.loads((run_dir / "meta.json").read_text())["deliverable"] + if kind == "files": + return any(path.is_file() for path in (run_dir / "output").rglob("*")) + path = run_dir / ("deliverable.patch" if kind == "diff" else "final.md") + return path.is_file() and bool(path.read_text(encoding="utf-8").strip()) + + def trajectory(self, run_dir: Path, *, obs_trunc: int) -> str: + return self.substrate.read_trajectory( + run_dir / "stream.jsonl", obs_trunc=obs_trunc, max_chars=200_000 + ) + + def deliverable_files(self, run_dir: Path) -> dict[str, str]: + kind = json.loads((run_dir / "meta.json").read_text())["deliverable"] + if kind == "files": + paths = { + p.relative_to(run_dir / "output").as_posix(): p + for p in sorted((run_dir / "output").rglob("*")) + if p.is_file() + } + else: + name = "deliverable.patch" if kind == "diff" else "final.md" + paths = {name: run_dir / name} + files = {} + for name, path in paths.items(): + if not path.is_file(): + continue + data = path.read_bytes() + try: + files[name] = data.decode("utf-8") + except UnicodeDecodeError: + files[name] = f"(binary file, {len(data)} bytes)" + return files + + def deliverable_text(self, run_dir: Path) -> str: + files = self.deliverable_files(run_dir) + text = "\n\n".join(files.values()) + if "deliverable.patch" in files: + text += "\n\n" + (run_dir / "final.md").read_text(encoding="utf-8") + return text + + def validate_tree(self, tree: dict[str, str]) -> None: + pass + + def has_judge(self, task_id: str) -> bool: + config = _read_task(self.tasks_dir, task_id) + return bool(config.get("check") or (self.tasks_dir / task_id / "rubric.md").is_file()) + + def score(self, run_dir: Path, task_id: str) -> dict | None: + config = _read_task(self.tasks_dir, task_id) + task_dir = self.tasks_dir / task_id + if config.get("check"): + check_dir = task_dir / "check" + digest = hashlib.sha256() + for path in sorted(check_dir.rglob("*")): + digest.update(path.relative_to(check_dir).as_posix().encode() + b"\0") + digest.update(b"d" if path.is_dir() else b"f" + path.read_bytes()) + digest.update(b"\0") + identity = {"check": config["check"], "check_sha256": digest.hexdigest()} + elif (task_dir / "rubric.md").is_file(): + judge_channel = _channel(self.settings, "judge") + identity = { + "channel": judge_channel.kind, + "model": judge_channel.model, + "rubric_sha256": hashlib.sha256((task_dir / "rubric.md").read_bytes()).hexdigest(), + } + else: + return None + cache = run_dir / "judge.json" + if cache.is_file(): + try: + cached = json.loads(cache.read_text(encoding="utf-8")) + except (OSError, ValueError): + cached = None + if isinstance(cached, dict) and cached.get("identity") == identity: + return cached + cache.unlink(missing_ok=True) + result = None + if config.get("check"): + with self._workspace(task_id, config) as ws: + if config["workspace"] == "project": + try: + workspace.apply_patch(ws, (run_dir / "deliverable.patch").read_text()) + except (OSError, subprocess.CalledProcessError): + return None + elif (run_dir / "output").is_dir(): + shutil.copytree(run_dir / "output", ws / "output", dirs_exist_ok=True) + if (task_dir / "check").is_dir(): + shutil.copytree(task_dir / "check", ws / "check", dirs_exist_ok=True) + try: + process = subprocess.run( + config["check"], + shell=True, + cwd=ws, + text=True, + capture_output=True, + timeout=config.get("timeout") or self.executor["timeout"], + env=os.environ + | {"HARNESS2_RUN_DIR": str(run_dir), "HARNESS2_TASK_DIR": str(task_dir)}, + ) + code, output, stderr = process.returncode, process.stdout, process.stderr + except subprocess.TimeoutExpired as exc: + code = -1 + output = (exc.stdout or b"").decode("utf-8", errors="replace") + stderr = (exc.stderr or b"").decode("utf-8", errors="replace") + lines = output.strip().splitlines() + try: + counts = _judge_counts(json.loads(lines[-1])) if lines else None + except ValueError: + counts = None + result = counts or {"n_passed": int(code == 0), "n_criteria": 1} + result.update(exit_code=code, output_tail=(output + stderr)[-2000:]) + elif (task_dir / "rubric.md").is_file(): + with tempfile.TemporaryDirectory(prefix="harness2-judge-") as tmp: + context = Path(tmp) + shutil.copyfile(task_dir / "task.md", context / "task.md") + shutil.copyfile(task_dir / "rubric.md", context / "rubric.md") + dest = context / "deliverables" + dest.mkdir() + if config["deliverable"] == "files" and (run_dir / "output").is_dir(): + shutil.copytree(run_dir / "output", dest, dirs_exist_ok=True) + else: + name = "deliverable.patch" if config["deliverable"] == "diff" else "final.md" + if (run_dir / name).is_file(): + shutil.copyfile(run_dir / name, dest / name) + shutil.copyfile(run_dir / "final.md", dest / "final.md") + try: + reply = Proposer(judge_channel).call( + mission=RUBRIC_MISSION, + context_dir=context, + label=f"judge.{task_id}.{slug(run_dir.name)}", + ) + except ProposerError: + return None + blocks = re.findall(r"```json\s*(.*?)```", reply.text, flags=re.S) + result = _judge_counts(_parse_json(blocks[-1] if blocks else reply.text)) + if result is not None: + result["all_pass"] = result["n_passed"] == result["n_criteria"] + result["identity"] = identity + cache.write_text(json.dumps(result, indent=2), encoding="utf-8") + return result + + +def _judge_counts(value: object) -> dict | None: + if not isinstance(value, dict): + return None + passed, total = value.get("n_passed"), value.get("n_criteria") + if type(passed) is not int or type(total) is not int or not 0 <= passed <= total or total <= 0: + return None + result = {"n_passed": passed, "n_criteria": total} + if isinstance(value.get("notes"), str): + result["notes"] = value["notes"] + return result + + +def make_adapter(ctx: SuiteContext) -> GenericAdapter: + return GenericAdapter(ctx) + + +def check(ctx: SuiteContext) -> list[str]: + problems = [] + settings = ctx.settings + executor = settings.get("executor.name") + if executor not in ("claude", "codex", "command"): + problems.append("the tasks suite runs on claude, codex or command") + elif executor != "command": + try: + cli_argv(settings.get("executor.bin"), executor) + except ConfigError as exc: + problems.append(str(exc)) + elif not settings.get("executor.command") or not settings.get("executor.editable"): + problems.append("command executor needs executor.command and non-empty executor.editable") + for role in ("executor", "improver", "selector", "reflector", "judge"): + if role == "executor": + if executor == "command": + continue + values = settings.section(role) + kind = executor + else: + values = settings.role(role) + kind = values["channel"] + if kind == "human": + if role != "selector": + problems.append("the human channel is only allowed for selector") + continue + try: + if role != "executor": + _channel(settings, role) + provider_name, _ = providers.split_model(values["model"]) + problems.extend(providers.check(settings.provider(provider_name))) + except ConfigError as exc: + problems.append(f"{role}: {exc}") + tasks_dir = settings.path("paths.tasks") + for path in sorted(tasks_dir.iterdir()) if tasks_dir.is_dir() else []: + if not path.is_dir(): + continue + try: + config = _read_task(tasks_dir, path.name) + if (path / "task.md").read_text(encoding="utf-8").startswith("TODO:"): + problems.append(f"task {path.name}: task.md still holds the template text") + if config["workspace"] == "project": + result = subprocess.run( + ["git", "-C", str(ctx.project), "rev-parse", "--show-toplevel"], + capture_output=True, + ) + if result.returncode: + problems.append(f"task {path.name}: project workspace needs a git repository") + except (ConfigError, OSError) as exc: + problems.append(str(exc)) + return list(dict.fromkeys(problems)) diff --git a/harness2/workspace.py b/harness2/workspace.py new file mode 100644 index 00000000..e2de6a24 --- /dev/null +++ b/harness2/workspace.py @@ -0,0 +1,117 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Temporary project worktrees and file workspaces for rollouts and judges.""" + +from __future__ import annotations + +import shutil +import subprocess +import tempfile +from collections.abc import Iterator, Sequence +from contextlib import contextmanager +from pathlib import Path + +from .settings import ConfigError + + +@contextmanager +def project_workspace(project: Path, ref: str) -> Iterator[Path]: + ws = Path(tempfile.mkdtemp(prefix="harness2-ws-")).resolve() + try: + pruned = subprocess.run( + ["git", "-C", str(project), "worktree", "prune"], + capture_output=True, + text=True, + ) + if pruned.returncode: + raise ConfigError(f"cannot prune project worktrees: {pruned.stderr.strip()}") + result = subprocess.run( + ["git", "-C", str(project), "worktree", "add", "--detach", "--", str(ws), ref], + capture_output=True, + text=True, + ) + if result.returncode: + raise ConfigError( + f"cannot create project workspace at {ref!r}: {result.stderr.strip()}" + ) + yield ws + finally: + subprocess.run( + ["git", "-C", str(project), "worktree", "remove", "--force", str(ws)], + capture_output=True, + ) + shutil.rmtree(ws, ignore_errors=True) + + +@contextmanager +def files_workspace(src: Path | None) -> Iterator[Path]: + ws = Path(tempfile.mkdtemp(prefix="harness2-ws-")).resolve() + try: + if src is not None: + shutil.copytree(src, ws, dirs_exist_ok=True) + (ws / "output").mkdir(exist_ok=True) + yield ws + finally: + shutil.rmtree(ws, ignore_errors=True) + + +def commit_all(ws: Path, message: str) -> None: + subprocess.run(["git", "add", "-A"], cwd=ws, check=True, capture_output=True) + subprocess.run( + [ + "git", + "-c", + "user.name=harness2", + "-c", + "user.email=harness2@localhost", + "-c", + "commit.gpgsign=false", + "commit", + "--no-verify", + "--allow-empty", + "-qm", + message, + ], + cwd=ws, + check=True, + capture_output=True, + ) + + +def diff_since(ws: Path, base: str, exclude: Sequence[str]) -> str: + subprocess.run(["git", "add", "-A"], cwd=ws, check=True, capture_output=True) + pathspecs = [ + p.replace(":(", ":(exclude,", 1) if p.startswith(":(") else f":(exclude){p}" + for p in exclude + ] + return subprocess.run( + ["git", "diff", "--cached", "--binary", base, "--", ".", *pathspecs], + cwd=ws, + check=True, + capture_output=True, + text=True, + ).stdout + + +def apply_patch(ws: Path, patch: str) -> None: + if patch.strip(): + subprocess.run( + ["git", "apply", "--whitespace=nowarn"], + cwd=ws, + input=patch, + text=True, + check=True, + capture_output=True, + ) diff --git a/plugins/harness2-benchmarks/LICENSE b/plugins/harness2-benchmarks/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/plugins/harness2-benchmarks/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/harness2-benchmarks/README.md b/plugins/harness2-benchmarks/README.md new file mode 100644 index 00000000..df458aec --- /dev/null +++ b/plugins/harness2-benchmarks/README.md @@ -0,0 +1,53 @@ +# harness2-benchmarks + +The three benchmarks of the paper +[Harness²: Recursive Agent Harnessing for an Open World](https://arxiv.org/abs/2610.14224), +packaged as suites for the `harness2` command of [Harness² CLI](../../README.md): + +| Suite | Benchmark | Work | +|---|---|---| +| `jb` | [JobBench](https://github.com/Job-Bench/job-bench-eval) | professional deliverables across occupations | +| `lab` | [LAB](https://github.com/harveyai/harvey-labs) | legal work products | +| `wb-office`, `wb-web`, `wb-code` | [WorkBuddy-Bench](https://github.com/Tencent/workbuddy-bench) | office documents, web apps, code | + +They run on the OpenCode or Codex executor, use the pinned upstream revisions and +compatibility patches of the paper, and grade with each benchmark's own judge. + +## Install + +From a checkout of this repository, next to `harness2-cli`: + +```bash +uv pip install -e . -e 'plugins/harness2-benchmarks[render]' +harness2 setup lab # or jb, wb-office, wb-web, wb-code +``` + +Setup downloads the upstream checkouts, task data and a pinned OpenCode build into +`~/.cache/harness2/benchmarks/` (set `HARNESS2_BENCH_DEPS` to put them elsewhere). +[Setup](docs/SETUP.md) covers system requirements, model credentials and the Codex +executor. + +## Run + +The benchmarks keep the paper's configuration: models and endpoints come from +`HARNESS2_*` environment variables (or `.harness2/.env`), as described in +[Setup](docs/SETUP.md#models-and-credentials). Choose the executor with +`-c executor.name=opencode` or `-c executor.name=codex`: + +```bash +harness2 -c executor.name=opencode -c suite.name=lab doctor +harness2 -c executor.name=opencode run --suite lab --k 1 --mode parallel \ + --only-domains antitrust-competition --max-tasks 1 +harness2 judge +harness2 report +``` + +[Benchmarks](docs/BENCHMARKS.md) describes each benchmark's data, domains and grading; +[Experiments](docs/EXPERIMENTS.md) covers task selection, concurrency, reproducibility and +network restrictions during evaluation. + +## Licenses + +The vendored runners and patches under `harness2_benchmarks/third_party/` keep their +upstream licenses; see [third-party notices](harness2_benchmarks/third_party/README.md). +Benchmark data is downloaded from its publishers under their terms. diff --git a/docs/BENCHMARKS.md b/plugins/harness2-benchmarks/docs/BENCHMARKS.md similarity index 74% rename from docs/BENCHMARKS.md rename to plugins/harness2-benchmarks/docs/BENCHMARKS.md index 18ca85f0..5f9d7aae 100644 --- a/docs/BENCHMARKS.md +++ b/plugins/harness2-benchmarks/docs/BENCHMARKS.md @@ -2,7 +2,7 @@ Complete the [installation and credentials](SETUP.md) first. Each benchmark installs separately; install the ones you intend to run. Every command -below takes `--substrate cx` to use the +below takes `-c executor.name=codex` to use the [Codex executor](SETUP.md#codex-executor) instead of OpenCode. Choose the recursion mode with `--mode parallel` or `--mode sequential` and the task-level recursion budget with `--k`. Keep both identical across the run, judge, and @@ -16,17 +16,17 @@ groups tasks by practice area and removes criteria from the material passed to the improver. Needs Pandoc and Poppler. ```bash -harness2-setup lab -harness2-check --bench lab -harness2 --bench lab --k 1 --mode parallel -harness2-judge --bench lab --k 1 --mode parallel -harness2-report --bench lab --k 1 --mode parallel +harness2 setup lab +harness2 -c executor.name=opencode -c suite.name=lab doctor +harness2 -c executor.name=opencode run --suite lab --k 1 --mode parallel +harness2 judge +harness2 report ``` Sequential recursion runs the same commands with `--mode sequential`. Tasks arrive with the pinned checkout at -`dependencies/harvey-labs/tasks///task.json`, and the adapter +`~/.cache/harness2/benchmarks/harvey-labs/tasks///task.json`, and the adapter discovers all installed tasks; no separate task list is required. Setup installs the LAB dependency environment, the runner overlays, and the included Vertex judge patch. @@ -34,9 +34,9 @@ judge patch. The judge reads outputs only after selection. Reports use Partial (fraction of criteria passed) and All-pass (all criteria passed). A failed judge remains unscored and writes its output to `judge.log` in the run directory; a Vertex -`429 RESOURCE_EXHAUSTED` there means quota, so rerun `harness2-judge` later. +`429 RESOURCE_EXHAUSTED` there means quota, so rerun `harness2 judge` later. Changing the judge model changes the measurement. Outputs are under -`runs/lab/`. +`.harness2/runs//`. ## JobBench @@ -46,18 +46,18 @@ Professional tasks, with the library grouped by the dataset's occupation directories. ```bash -harness2-setup jb -harness2-check --bench jb -harness2 --bench jb --k 1 --mode parallel -harness2-judge --bench jb --k 1 --mode parallel -harness2-report --bench jb --k 1 --mode parallel +harness2 setup jb +harness2 -c executor.name=opencode -c suite.name=jb doctor +harness2 -c executor.name=opencode run --suite jb --k 1 --mode parallel +harness2 judge +harness2 report ``` Sequential recursion runs the same commands with `--mode sequential`. Setup downloads only the main dataset from Hugging Face — public and ungated, so no account, token, or terms acceptance is required — records its revision, and -places tasks at `dependencies/job-bench-eval/dataset/main/`. It also installs the +places tasks at `~/.cache/harness2/benchmarks/job-bench-eval/dataset/main/`. It also installs the document extraction dependencies and a judge patch that handles reasoning effort and empty completions. Rubrics stay outside the improver's evidence. @@ -65,7 +65,7 @@ With a Gemini `HARNESS2_JUDGE_MODEL`, the judge uses the project's Vertex OpenAI-compatible endpoint and Application Default Credentials. For a separate judge provider, set `HARNESS2_JUDGE_BASE_URL`, `HARNESS2_JUDGE_API_KEY`, and the model ID that endpoint accepts. The reported score uses the upstream rubric -weights; unscored tasks are reported separately. Results are under `runs/jb/`. +weights; unscored tasks are reported separately. Results are under `.harness2/runs//`. Note: JobBench data, including `RUBRICS.json`, is public on Hugging Face, so an executor with web access may retrieve it. Block those hosts during runs as @@ -84,22 +84,22 @@ retain that license. ```bash docker info -harness2-setup wb --subset office -harness2-check --bench wb --subset office -harness2 --bench wb --subset office --k 1 --mode parallel -harness2-judge --bench wb --subset office --k 1 --mode parallel -harness2-report --bench wb --subset office --k 1 --mode parallel +harness2 setup wb-office +harness2 -c executor.name=opencode -c suite.name=wb-office doctor +harness2 -c executor.name=opencode run --suite wb-office --k 1 --mode parallel +harness2 judge +harness2 report ``` Sequential recursion runs the same commands with `--mode sequential`. -`harness2-setup wb` without `--subset` installs all three subsets in one run; +`python plugins/harness2-benchmarks/scripts/setup_benchmark.py wb` without `--subset` installs all three subsets in one run; pass `--subset` to install only the one you intend to run. Every other command takes one subset at a time. Setup installs the pinned Harbor-based runtime, applies the included adapter, mount, and credential/transport patches, copies the harness runners, creates -solver and judge routes from `.env`, downloads the requested subset with the +solver and judge routes from `.harness2/.env`, downloads the requested subset with the upstream fetch script, and builds the OpenCode harness image. The public Vertex setup uses a `google-vertex/gemini...` executor and a bare `gemini...` judge ID; no model YAML or account-specific proxy configuration needs to be copied from @@ -115,10 +115,13 @@ cannot be verified is not unpacked. A verified download prints `wb-bench--v1.0.tar.gz: OK`. Tasks are discovered from `task.toml` and `instruction.md` under -`dependencies/workbuddy-bench/datasets/wb-bench--v1.0/tasks/`. Libraries use +`~/.cache/harness2/benchmarks/workbuddy-bench/datasets/wb-bench--v1.0/tasks/`. Libraries use `metadata.role` on code and `metadata.category` on office and web. Each rollout uses a fresh benchmark container. The adapter reads the verifier's `score.json`; the improver never receives verifier output. Missing verifier results remain -unscored. Runs live under `runs/wb//`. +unscored. Runs live under `.harness2/runs//`. -For Codex, start the bridge with `harness2-bridge --host 0.0.0.0` so that the containers can reach it. +For Codex, start the bridge with `--host 0.0.0.0` (for example +`harness2 bridge --model vertex/gemini-3.5-flash --host 0.0.0.0`) so that the containers can +reach it, and set `HARNESS2_CODEX_CONTAINER_URL` as described in +[Setup](SETUP.md#codex-executor). diff --git a/docs/EXPERIMENTS.md b/plugins/harness2-benchmarks/docs/EXPERIMENTS.md similarity index 74% rename from docs/EXPERIMENTS.md rename to plugins/harness2-benchmarks/docs/EXPERIMENTS.md index bd52e150..c37229ff 100644 --- a/docs/EXPERIMENTS.md +++ b/plugins/harness2-benchmarks/docs/EXPERIMENTS.md @@ -24,18 +24,18 @@ separated by benchmark, executor, mode, budget, and seed. | `--k K --mode parallel` | Explore `K` candidates per scope and produce `K` compositions | | `--k K --mode sequential` | Refine the harness through `K` successive steps | | `--workers N` | Run `N` domains concurrently; tasks within each domain stay ordered | -| `--substrate cx` | Use Codex instead of the default OpenCode executor | +| `-c executor.name=codex` | Use Codex instead of the default OpenCode executor | | `--seed N` | Shuffle the task stream reproducibly; does not control model sampling | Use the same benchmark, executor, subset, `k`, mode, and seed for refinement, judging, and reporting. For example: ```bash -export HARNESS2_RUNS_DIR=runs/lab-sequential-k3 -harness2 --bench lab --k 3 --mode sequential \ +export HARNESS2_PATHS__RUNS=.harness2/runs/lab-sequential-k3 +harness2 -c executor.name=opencode run --suite lab --k 3 --mode sequential \ --only-domains antitrust-competition --max-tasks 1 -harness2-judge --bench lab --k 3 --mode sequential -harness2-report --bench lab --k 3 --mode sequential +harness2 judge +harness2 report ``` Use a separate output root when changing the budget or mode to keep experiments @@ -56,20 +56,20 @@ reporting in sequence. Set `SUBSTRATE`, `K`, `MODE`, and, for WorkBuddy, `SUBSET`; additional arguments are passed to the refinement command: ```bash -bash scripts/run_lab.sh --only-domains antitrust-competition --max-tasks 1 -SUBSTRATE=cx K=3 MODE=sequential bash scripts/run_jobbench.sh --max-tasks 1 -SUBSET=web bash scripts/run_workbuddy.sh --max-tasks 1 +bash plugins/harness2-benchmarks/scripts/run_lab.sh --only-domains antitrust-competition --max-tasks 1 +SUBSTRATE=cx K=3 MODE=sequential bash plugins/harness2-benchmarks/scripts/run_jobbench.sh --max-tasks 1 +SUBSET=web bash plugins/harness2-benchmarks/scripts/run_workbuddy.sh --max-tasks 1 ``` ## Reproducibility When comparing budgets or modes, keep task selection, executor, improver, judge, -and web access fixed. Use a distinct `HARNESS2_RUNS_DIR` when changing models or +and web access fixed. Use a distinct `HARNESS2_PATHS__RUNS` when changing models or task selection; the saved configuration prevents incompatible resumes. JobBench's dataset is resolved at download time rather than pinned to a commit. Setup records the downloaded Hugging Face revision in -`dependencies/job-bench-eval/dataset/main/.harness2-download.json`. Report that +`~/.cache/harness2/benchmarks/job-bench-eval/dataset/main/.harness2-download.json`. Report that revision alongside JobBench results. ### Task partitions @@ -77,8 +77,8 @@ revision alongside JobBench results. To create a reproducible, domain-stratified partition of the installed tasks: ```bash -python scripts/split_tasks.py --bench lab --seed 42 --output splits/lab.json -harness2 --bench lab --split-file splits/lab.json --split test --mode parallel +python plugins/harness2-benchmarks/scripts/split_tasks.py --bench lab --seed 42 --output splits/lab.json +harness2 -c executor.name=opencode run --suite lab --split-file splits/lab.json --split test --mode parallel ``` The default train fraction is 0.5. Generated partitions are local files, not @@ -86,9 +86,9 @@ release assets, and the script refuses to overwrite an existing file. ## Baseline evaluation -`harness2-judge --bases` scores a run namespace's reference executions, writing +`harness2 judge --bases` scores a run namespace's reference executions, writing `base_judged.json` beside `judged.json` for the base-harness comparison. -`harness2-report` covers the refined harness only. +`harness2 report` covers the refined harness only. ## Web access @@ -110,13 +110,13 @@ tool setting. Report the restrictions used alongside results. ## Results and resume -The default output roots are `runs/lab/`, `runs/jb/`, and -`runs/wb//`. They contain task summaries, execution artifacts, adopted +The default output roots are `.harness2/runs//`, `.harness2/runs//`, and +`.harness2/runs//`. They contain task summaries, execution artifacts, adopted harnesses, reflection notes, and graded results. Repeat the same command to resume. Failed evaluations remain unscored rather -than being counted as zero. Use a new `HARNESS2_RUNS_DIR` when changing models or +than being counted as zero. Use a new `HARNESS2_PATHS__RUNS` when changing models or task selection. -Refinement, judging, and reporting use `harness2`, `harness2-judge`, and -`harness2-report`; each supports `--help`. +Refinement, judging, and reporting use `harness2 run`, `harness2 judge`, and +`harness2 report`; each supports `--help`. diff --git a/docs/SETUP.md b/plugins/harness2-benchmarks/docs/SETUP.md similarity index 57% rename from docs/SETUP.md rename to plugins/harness2-benchmarks/docs/SETUP.md index 0d037b8a..c11783f2 100644 --- a/docs/SETUP.md +++ b/plugins/harness2-benchmarks/docs/SETUP.md @@ -1,6 +1,6 @@ # Setup guide -Start with the [quick start](../README.md#install). This guide covers benchmark +Start with the [quick start](../../../README.md#install). This guide covers benchmark dependencies, model credentials, and the optional Codex executor. For individual benchmarks, see the [benchmark guides](BENCHMARKS.md). @@ -22,19 +22,27 @@ Pandoc and Poppler are needed by LAB. Also install: - [Docker Engine](https://docs.docker.com/engine/install/) for WorkBuddy-Bench, with permission to run `docker info`. -Benchmark setup installs its own pinned Bun runtime under `dependencies/`. +Benchmark setup installs its own pinned Bun runtime under `~/.cache/harness2/benchmarks/`. Keep the main Harness² environment active when running the commands below. ## Models and credentials -Copy `.env.example` to `.env` if you have not already done so. Set your Vertex AI -project and model IDs, then authenticate: +Put your Vertex AI project and model IDs in `.harness2/.env` (or export them in the +shell), then authenticate: + +```bash +# .harness2/.env +GOOGLE_CLOUD_PROJECT=my-gcp-project +HARNESS2_SOLVER_MODEL=google-vertex/gemini-3.5-flash +HARNESS2_IMPROVER_MODEL=google-vertex/gemini-3.5-flash +HARNESS2_JUDGE_MODEL=gemini-3.5-flash +``` ```bash gcloud auth application-default login ``` -Shell environment variables take precedence over `.env`. +Shell environment variables take precedence over `.harness2/.env`. `GOOGLE_CLOUD_PROJECT` identifies your project; `GOOGLE_CLOUD_LOCATION` defaults to `global`. Your account must have access to each configured model. @@ -43,7 +51,7 @@ to `global`. Your account must have access to each configured model. | `HARNESS2_SOLVER_MODEL` | Executor model in OpenCode's `provider/model` format | | `HARNESS2_IMPROVER_MODEL` | Improver model in the same format; defaults to the executor | | `HARNESS2_JUDGE_MODEL` | Bare judge model ID; the adapter supplies the Vertex prefix | -| `HARNESS2_RUNS_DIR` | Output root; defaults to `runs/` | +| `HARNESS2_PATHS__RUNS` | Output root; defaults to `.harness2/runs/` | The provided configuration uses Vertex Gemini executors and judges, and supports either Vertex Claude or Gemini as the improver. LAB also supports the upstream @@ -70,31 +78,31 @@ route: the installation check runs Install only the benchmark you intend to run: ```bash -harness2-setup lab -harness2-check --bench lab +harness2 setup lab +harness2 -c executor.name=opencode -c suite.name=lab doctor ``` -Use `jb` for JobBench or `wb` for WorkBuddy-Bench. See the +Use `jb` for JobBench or `wb-office` for WorkBuddy-Bench. See the [benchmark guides](BENCHMARKS.md) for subset selection and licensing. The runner overlays and compatibility patches are included in -[`third_party/`](../third_party/README.md). Setup clones supported upstream -revisions under `dependencies/`, applies the patches, creates the benchmark +[`third_party/`](../harness2_benchmarks/third_party/README.md). Setup clones supported upstream +revisions under `~/.cache/harness2/benchmarks/`, applies the patches, creates the benchmark environments, and downloads task data. Each benchmark environment lives inside its upstream checkout: -- `dependencies/harvey-labs/.venv` -- `dependencies/job-bench-eval/.venv` -- `dependencies/workbuddy-bench/.venv` +- `~/.cache/harness2/benchmarks/harvey-labs/.venv` +- `~/.cache/harness2/benchmarks/job-bench-eval/.venv` +- `~/.cache/harness2/benchmarks/workbuddy-bench/.venv` Harness² selects these interpreters automatically; do not activate them for the main commands. ### Installation checks -`harness2-setup --dry-run` previews setup without making changes. -`harness2-check` checks installed dependencies, credentials, and configuration +`harness2 setup --dry-run` previews setup without making changes. +`harness2 doctor` checks installed dependencies, credentials, and configuration without calling a model. Refinement runs perform the same checks, including judge configuration, before starting. @@ -105,37 +113,46 @@ return a nonzero exit status when incomplete. ## Codex executor The Codex executor connects to a Responses API endpoint. For Gemini on Vertex -AI, the included `harness2-bridge` provides this interface through a local -LiteLLM server, using the same Application Default Credentials and -`GOOGLE_CLOUD_PROJECT`. +AI, `harness2 bridge` provides this interface through a local LiteLLM server, using +the same Application Default Credentials. -Install the optional dependencies and start the bridge in a separate session: +Add a Vertex provider to `.harness2/config.toml`, install the optional dependencies and +start the bridge in a separate session: + +```toml +[providers.vertex] +type = "vertex" +project = "my-gcp-project" +region = "global" +``` ```bash -uv pip install -e '.[render,bridge]' -harness2-bridge # for WorkBuddy: harness2-bridge --host 0.0.0.0 +uv pip install -e '.[bridge]' -e 'plugins/harness2-benchmarks[render]' +harness2 bridge --model vertex/gemini-3.5-flash # for WorkBuddy: add --host 0.0.0.0 ``` -The bridge prints the `HARNESS2_CODEX_*` settings to add to `.env`. Its port and -generated access key are stored under `dependencies/codex/` and reused on -restart. Pass `--substrate cx` to setup, checks, refinement, judging, and +The bridge prints its endpoint and access key. Set `HARNESS2_CODEX_BASE_URL` to that +endpoint (`http://127.0.0.1:/v1`), `HARNESS2_CODEX_API_KEY` to the key and +`HARNESS2_CODEX_MODEL` to the model name in `.harness2/.env`. The port and key are stored +under `~/.cache/harness2/bridge/` and reused on restart. Pass `-c executor.name=codex` to setup, checks, refinement, judging, and reporting: ```bash -harness2-setup lab --substrate cx -harness2-check --bench lab --substrate cx -harness2 --bench lab --substrate cx --k 1 --mode parallel \ +harness2 setup lab --substrate cx +harness2 -c executor.name=codex -c suite.name=lab doctor +harness2 -c executor.name=codex run --suite lab --k 1 --mode parallel \ --only-domains antitrust-competition --max-tasks 1 ``` WorkBuddy executes Codex inside Docker containers, so the bridge must listen on -`--host 0.0.0.0`; it also prints `HARNESS2_CODEX_CONTAINER_URL`. This binds the -bridge to all network interfaces; restrict access to the intended clients. +`--host 0.0.0.0`, and `HARNESS2_CODEX_CONTAINER_URL` must give the address the containers +reach it at, usually `http://host.docker.internal:/v1`. This binds the bridge to all +network interfaces; restrict access to the intended clients. To use another Responses API endpoint, set `HARNESS2_CODEX_BASE_URL`, `HARNESS2_CODEX_API_KEY`, and `HARNESS2_CODEX_MODEL` directly. Setup records the -endpoint and model in `dependencies/codex/home/config.toml`; rerun setup after -changing either. `harness2-check` reports stale configuration until then. +endpoint and model in `~/.cache/harness2/benchmarks/codex/home/config.toml`; rerun setup after +changing either. `harness2 doctor` reports stale configuration until then. Codex runs with its sandbox disabled. Review the [executor network restrictions](EXPERIMENTS.md#web-access) before evaluation. @@ -152,14 +169,14 @@ Setup downloads dependencies from the following hosts: | A Docker registry | WorkBuddy container image | | `astral.sh` | CPython 3.12 when the host has no system Python 3.12 | -Use `harness2-setup --dry-run` to review setup commands when configuring +Use `harness2 setup --dry-run` to review setup commands when configuring network access. Experiments also access the model endpoints configured in -`.env`. Setup downloads and executor web access have different requirements; +`.harness2/.env`. Setup downloads and executor web access have different requirements; see [web access during evaluation](EXPERIMENTS.md#web-access). ## Wheel installations -A wheel ships the `harness2` package only. The task partitioner, -`python scripts/split_tasks.py`, requires this source tree. For a wheel -installation, work from a dedicated directory: `.env`, `dependencies/`, and -`runs/` are resolved there. +A wheel ships the `harness2_benchmarks` package only. The task partitioner, +`python plugins/harness2-benchmarks/scripts/split_tasks.py`, requires this source tree. For a wheel +installation, work from a dedicated directory: `.harness2/.env`, `~/.cache/harness2/benchmarks/`, and +`.harness2/runs/` are resolved there. diff --git a/harness2/benchmarks/__init__.py b/plugins/harness2-benchmarks/harness2_benchmarks/__init__.py similarity index 100% rename from harness2/benchmarks/__init__.py rename to plugins/harness2-benchmarks/harness2_benchmarks/__init__.py diff --git a/harness2/config.py b/plugins/harness2-benchmarks/harness2_benchmarks/config.py similarity index 68% rename from harness2/config.py rename to plugins/harness2-benchmarks/harness2_benchmarks/config.py index 59e0703d..e16daba0 100644 --- a/harness2/config.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/config.py @@ -20,47 +20,17 @@ import sys from pathlib import Path +from harness2.core.spec import ChannelSpec +from harness2.providers import Provider +from harness2.settings import ConfigError, cache_dir +from harness2.settings import load_dotenv as load_dotenv + PKG = Path(__file__).resolve().parent -ROOT = PKG.parent if (PKG.parent / "pyproject.toml").is_file() else Path.cwd() -THIRD_PARTY = ROOT / "third_party" +THIRD_PARTY = PKG / "third_party" METHOD_VERSION = 2 - -DEPENDENCIES = ROOT / "dependencies" - - -class ConfigError(RuntimeError): - """A required deployment value is missing; the message carries the remediation.""" - - -def load_dotenv(path: Path = ROOT / ".env") -> None: - """Load KEY=VALUE lines from `path` into os.environ with setdefault (the shell wins). - - Accepts blank lines, `#` comments, an optional `export ` prefix, and single- or - double-quoted values. A missing or unreadable file is not an error. - """ - try: - text = Path(path).read_text(encoding="utf-8") - except OSError: - return - for raw in text.splitlines(): - line = raw.strip() - if not line or line.startswith("#") or "=" not in line: - continue - if line.startswith("export "): - line = line[len("export ") :].lstrip() - key, _, value = line.partition("=") - key = key.strip() - if not key or not all(c.isalnum() or c == "_" for c in key): - continue - value = value.strip() - if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'): - value = value[1:-1] - elif " #" in value: - value = value.split(" #", 1)[0].rstrip() - os.environ.setdefault(key, value) - - -load_dotenv() +DEPENDENCIES = ( + Path(os.environ.get("HARNESS2_BENCH_DEPS") or cache_dir() / "benchmarks").expanduser().resolve() +) def env(name: str, default=None): @@ -83,12 +53,11 @@ def require(name: str, *, needed_by: str) -> str: v = env(name) if not v: raise ConfigError( - f"{name} is required for {needed_by}; set it in .env or the shell (see .env.example)" + f"{name} is required for {needed_by}; set it in .harness2/.env or the shell (see .env.example)" ) return str(v) -RUNS = Path(env("HARNESS2_RUNS_DIR", ROOT / "runs")).expanduser().resolve() VERTEX_PROJECT = env("GOOGLE_CLOUD_PROJECT", env("HARNESS2_VERTEX_PROJECT", "")) VERTEX_REGION = env("GOOGLE_CLOUD_LOCATION", "global") if VERTEX_PROJECT: @@ -187,3 +156,53 @@ def validate_models(*, judge: bool = False) -> None: raise ConfigError("Set GOOGLE_CLOUD_PROJECT for Vertex AI") if judge: require("HARNESS2_JUDGE_MODEL", needed_by="benchmark judging") + + +CHANNELS = ("claude_cli", "opencode") + + +def resolve_channel_model( + kind: str | None, + model: str | None, + *, + effort: str = "high", + steps: int = 120, + timeout: int = 3600, +) -> ChannelSpec: + """The one channel->model mapping; a lone override of either raises.""" + kwargs = {"effort": effort, "steps": steps, "timeout": timeout} + if (kind or DEFAULT_PROPOSER_CHANNEL) == "opencode": + kwargs["bin"] = ( + f"{BUN} run --cwd {OPENCODE_DIR}/packages/opencode --conditions=browser src/index.ts" + ) + if VERTEX_PROJECT: + kwargs["provider"] = Provider( + "vertex", "vertex", project=VERTEX_PROJECT, region=VERTEX_REGION + ) + use_default = kind is None and model is None + if (kind is None) != (model is None): + raise ValueError( + "channel and model must be overridden TOGETHER (a bare-id/provider-prefixed " + f"mismatch fails silently downstream); got kind={kind!r}, model={model!r}" + ) + if kind is None: + kind = DEFAULT_PROPOSER_CHANNEL + if kind not in CHANNELS: + raise ValueError(f"unknown channel {kind!r}; expected one of {CHANNELS}") + if model is None: + model = PROPOSER_MODEL_BY_CHANNEL[kind] + if not model.strip(): + if use_default: + return ChannelSpec(kind, "", **kwargs) + raise ValueError("model override must not be empty") + if kind == "opencode" and "/" not in model: + raise ValueError( + f"opencode channel needs a provider-prefixed model id " + f"in HARNESS2_IMPROVER_MODEL or HARNESS2_SOLVER_MODEL, got {model!r}" + ) + if kind == "claude_cli" and "/" in model: + raise ValueError( + f"claude_cli channel takes a bare Anthropic id " + f"(e.g. {PROPOSER_MODEL_BY_CHANNEL['claude_cli']!r}), got {model!r}" + ) + return ChannelSpec(kind, model, **kwargs) diff --git a/dependencies/ASSETS.json b/plugins/harness2-benchmarks/harness2_benchmarks/dependencies_meta/ASSETS.json similarity index 82% rename from dependencies/ASSETS.json rename to plugins/harness2-benchmarks/harness2_benchmarks/dependencies_meta/ASSETS.json index 64c838fd..81d6c07a 100644 --- a/dependencies/ASSETS.json +++ b/plugins/harness2-benchmarks/harness2_benchmarks/dependencies_meta/ASSETS.json @@ -1,6 +1,6 @@ { "third_party/jb_runner/LICENSE": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4", - "third_party/jb_runner/PROVENANCE.md": "7e16c8f5e43aa272baa5cf39bf245b42cf1f40f885457ac4123c91b7de0b06ef", + "third_party/jb_runner/PROVENANCE.md": "db5d8237e3670dcdc098fb93f7b6b57b255b6c6b38dc6e37824561491d1bb1d9", "third_party/jb_runner/codex_base_harness/AGENTS.md": "b5df5264478dfce7d6af52f8443a1797b17967d1bc4a8c4f8c5fe43933516a89", "third_party/jb_runner/harness_baseline_oc/LongTermMEMORY.md": "c78b55245ef1433c017bc3f6f796094e16c640b98543836f6d930972243d28d2", "third_party/jb_runner/harness_baseline_oc/guardrails.md": "ebd0b0f287bcc9bea5efa063d632677b3c0fdf014c7b67bec088c11253605a53", @@ -10,19 +10,19 @@ "third_party/jb_runner/harness_baseline_oc/sub_agents/README.md": "56c5507e02c30c0d611efe2d7426141e9fc626a91599738afc7e003a07d3adb0", "third_party/jb_runner/harness_baseline_oc/systemprompt.md": "f5fa6862d87e4c51b78a4ce4446dace9438b3d201e0350c91ca02d0c83f552e3", "third_party/jb_runner/harness_baseline_oc/tool_descriptions/tool_guidance.md": "5ae4f052905a31e9573abe4971172bf40b4e1f5d265d1be072ed444232371f41", - "third_party/jb_runner/run_benchmark_codex_cli_adapt.sh": "a94c645d3061da354591dde446d11f8b3e6d08b4005f8f587668aae07c0fc100", + "third_party/jb_runner/run_benchmark_codex_cli_adapt.sh": "91ccd802f423dfd1abd0d8f28ac1d962fb3aae896a778532d51e934f8242876f", "third_party/lab_runner/LICENSE": "f92627d2ebe80fc0add3b171b2d7eee5e28a98dd0d0a4a5ee5829314243bb3b9", - "third_party/lab_runner/PROVENANCE.md": "b5b42319000e1522765d2520dffcd9d1058c5472cc49046ec1906ffffa66c53c", - "third_party/lab_runner/adapt_runs_cx/run_codex_lab.py": "567d6a06d8104f853b8d83953af333ea8ef57a9053e0c9788a42692c48196556", + "third_party/lab_runner/PROVENANCE.md": "942fd05e4a4ca804c3650e7681ac788bf9febb2aa57210cc39de0cb9c74344b6", + "third_party/lab_runner/adapt_runs_cx/run_codex_lab.py": "2c9f911adc9d55c968e81ae7cc2ad4585975316ac61ed7634d0453868b3fee7e", "third_party/lab_runner/opencode/README.md": "20f2739eada1a76560c412d39603e040c420d3d36a2bf67a99a814500306f0ae", "third_party/lab_runner/opencode/__init__.py": "df2b298f529bbcba94713aa748b5a04569f38bbb3f96c9219b7fb0d7186066a8", - "third_party/lab_runner/opencode/render.py": "b636a60d039fdfa66d64eb5d5d3c93a99464be2bcbd5b80dc8f91d1f104ea5ee", + "third_party/lab_runner/opencode/render.py": "dbdace5189da9162b916103523fb3e0295239c02561c01360acd4aa72110e749", "third_party/lab_runner/opencode/render_tree.py": "54d64b461e430905b212b46902512aad38d3585fb5921b7a9982eea0413a4b73", "third_party/lab_runner/opencode/run_opencode.py": "7e7178583a6acb2e07bffd1fbef78f1b0095975bc2c61f8b0038fcc528090e32", "third_party/lab_runner/opencode/scripts/read_doc.py": "6b3c8feb3b90b0b2bbbf4cb491c81f1114b67ada04d2c988fac8cb1af68533b5", "third_party/lab_runner/opencode/session.py": "e804e29a20ac4fd71454c85b9e15432cf303739c7f8d7aacc0df19f732bfcb62", "third_party/lab_runner/tts_lab/__init__.py": "4ae8a9112bc00a182c983a35c8c5db40717007cefbc1b0bed89f04bc384a0106", - "third_party/lab_runner/tts_lab/config.py": "f6eb8551c3cb3538de42a2e43cd5cd1aa466370d64dd681e46ec70f1c3dcf965", + "third_party/lab_runner/tts_lab/config.py": "e2071fdad3664ca0f25e36d9c2c9cf1ea7e7398bc913b3921a183c9c5c8aaba7", "third_party/licenses/harvey-labs.LICENSE": "f92627d2ebe80fc0add3b171b2d7eee5e28a98dd0d0a4a5ee5829314243bb3b9", "third_party/licenses/job-bench-eval.LICENSE": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4", "third_party/licenses/opencode.LICENSE": "625f0f619133f89bbbb2abe37369613dfa1885eba1e50d02170deb62bb42cb6b", @@ -32,12 +32,12 @@ "third_party/patches/opencode-compat.diff": "c9a8a0e826d48f3fcfc7d818321d128c28e1f7f7c594954c19453dfca4946dc8", "third_party/patches/workbuddy-bench-runtime.diff": "1d581ee0c23a9a5574ce0f9317d0bda17f6c3a2556898c89b34ccbdbd5b87b3f", "third_party/wb_runner/LICENSE": "b466fc7a63acc5cac4275fc4b0f1e22f332ad188ea600b217076ea3a8e040956", - "third_party/wb_runner/PROVENANCE.md": "9011685c8808fb6ab56d23c823af884e5905caecd00f64323f3d02f22e45524c", - "third_party/wb_runner/agents/codex_agent.py": "dbdf48c19c51d7a0389caf3f5601c1ff967ef534137d829afe8ca8792db40c9d", + "third_party/wb_runner/PROVENANCE.md": "71f11c2fa0858d7f83b9efee09443438c61ba3cc58519a4dab4bf540c200990b", + "third_party/wb_runner/agents/codex_agent.py": "b031dcd1846b028ce97c98b1f5d4d738ad28197f40e54cafdd0a3800fa5ab5a7", "third_party/wb_runner/agents/codex_harness_agent.py": "fc1d1ecfc2de1debabb0ff84ec7a113b05536873ac76f73806835ff873b2f580", "third_party/wb_runner/agents/oc_agent.py": "5375a49b9a9e6d6d6212b47156b36dca17934e54728ddeed6218ac3662c67fff", "third_party/wb_runner/agents/oc_harness_agent.py": "647e31187af1305a5ddc42f7e41740795f825e0c708f63f5d1f29b954625c09c", - "third_party/wb_runner/configs/harnesses/codex-harness/_defaults.yaml": "5dddd5c358eff75ca69f7277ae6ccd58a2e208e19d722b862f04062a3718479e", + "third_party/wb_runner/configs/harnesses/codex-harness/_defaults.yaml": "d0b85da88e06498d37fbc7a8cfe6af5bb42be8c573458e158d1755ca88a28853", "third_party/wb_runner/configs/harnesses/codex-harness/versions/0.144.6.yaml": "f6ebdc457786c3869ceb8a11ee0c4a63db53a080961075c15edff2a7640293bf", "third_party/wb_runner/configs/harnesses/oc-harness/_defaults.yaml": "418843868dc1b01431b339791ae63bb23c88cad3238bf9ab844fbefae8acd71e", "third_party/wb_runner/configs/harnesses/oc-harness/versions/1.14.18.yaml": "fd0515a9d539c011501984b1017f90410cef79045e172c20a9598596f39106a6", diff --git a/dependencies/PINNED.md b/plugins/harness2-benchmarks/harness2_benchmarks/dependencies_meta/PINNED.md similarity index 84% rename from dependencies/PINNED.md rename to plugins/harness2-benchmarks/harness2_benchmarks/dependencies_meta/PINNED.md index bd62a57c..9155f451 100644 --- a/dependencies/PINNED.md +++ b/plugins/harness2-benchmarks/harness2_benchmarks/dependencies_meta/PINNED.md @@ -1,6 +1,6 @@ # Upstream pins and patches -`harness2/config.py` is the authoritative pin table. The setup script fetches +`harness2_benchmarks/config.py` is the authoritative pin table. The setup script fetches these commits directly from the public remotes and refuses to overwrite a checkout at another revision. @@ -26,7 +26,7 @@ checkout at another revision. undefined constant in the upstream Anthropic route support. Patches are checked before applying and skipped if already applied. Derived runner -files belong in `harness2/third_party/{jb,lab,wb}_runner/`, which this copy does not +files belong in `harness2_benchmarks/third_party/{jb,lab,wb}_runner/`, which this copy does not include; setup copies the LAB and WorkBuddy overlays into the upstream checkouts. The upstream dependency locks are used for Python installations. OpenCode uses @@ -35,8 +35,8 @@ JobBench's Hugging Face revision is resolved once for a download and recorded in `dataset/main/.harness2-download.json`. WorkBuddy's dataset fetcher, as patched by setup, verifies each archive against the dataset repository's `SHA256SUMS` and refuses one it cannot verify; see -[the WorkBuddy guide](../docs/BENCHMARKS.md#workbuddy-bench). LAB task files are part of its pinned Git checkout. +[the WorkBuddy guide](../../docs/BENCHMARKS.md#workbuddy-bench). LAB task files are part of its pinned Git checkout. These inputs support the public execution workflow. Historical paper runs may have used different tasks, model deployments, and additional local tooling; -see [experiment notes](../README.md#experiments). +see [experiment notes](../../../../README.md#experiments). diff --git a/dependencies/README.md b/plugins/harness2-benchmarks/harness2_benchmarks/dependencies_meta/README.md similarity index 90% rename from dependencies/README.md rename to plugins/harness2-benchmarks/harness2_benchmarks/dependencies_meta/README.md index 85346f22..d4508617 100644 --- a/dependencies/README.md +++ b/plugins/harness2-benchmarks/harness2_benchmarks/dependencies_meta/README.md @@ -1,8 +1,8 @@ # Third-party code and data Benchmark tools are downloaded into this directory and excluded from version -control. Pins are defined in `harness2/config.py`; setup commands are in the -[benchmark guides](../docs/BENCHMARKS.md). +control. Pins are defined in `harness2_benchmarks/config.py`; setup commands are in the +[benchmark guides](../../docs/BENCHMARKS.md). | Upstream | License | Release integration | |---|---|---| diff --git a/harness2/install.py b/plugins/harness2-benchmarks/harness2_benchmarks/install.py similarity index 98% rename from harness2/install.py rename to plugins/harness2-benchmarks/harness2_benchmarks/install.py index e126bc42..dee43c6e 100644 --- a/harness2/install.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/install.py @@ -54,8 +54,8 @@ def missing_runners() -> str: return "" return ( "benchmark runners and patches are not included in this copy " - f"({len(absent)} of {len(need)} missing under third_party/); " - "see third_party/README.md" + f"({len(absent)} of {len(need)} missing under {config.THIRD_PARTY}/); " + f"see {config.THIRD_PARTY}/README.md" ) @@ -229,7 +229,7 @@ def _share_with_containers(root: Path) -> None: path.chmod(mode | extra) -CODEX_CONFIG_REMEDY = "rerun harness2-setup --substrate cx" +CODEX_CONFIG_REMEDY = "rerun harness2 setup --substrate cx" CODEX_PROVIDER = "harness2" @@ -286,7 +286,7 @@ def download_jobbench(*, dry_run=False): """Install the JobBench task tree from its public Hugging Face dataset repo. The dataset is public, so an executor with web access may retrieve it; - see `docs/BENCHMARKS.md`. + see `plugins/harness2-benchmarks/docs/BENCHMARKS.md`. """ repo = config.jb_repo() dest = repo / "dataset" / "main" diff --git a/harness2/benchmarks/jb_opencode_runner.py b/plugins/harness2-benchmarks/harness2_benchmarks/jb_opencode_runner.py similarity index 98% rename from harness2/benchmarks/jb_opencode_runner.py rename to plugins/harness2-benchmarks/harness2_benchmarks/jb_opencode_runner.py index 1b35190d..37091580 100644 --- a/harness2/benchmarks/jb_opencode_runner.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/jb_opencode_runner.py @@ -26,7 +26,9 @@ from datetime import datetime from pathlib import Path -JOBBENCH = Path(__file__).resolve().parents[1] +from harness2_benchmarks import config + +JOBBENCH = config.jb_repo() TEMP_BASE = Path("/tmp/hevo_ws") TRAJ_DIR_NAME = "model_traj" EVENT_PREFIX = '{"type":' @@ -76,7 +78,7 @@ def _settings() -> Settings: render_dir = Path(env["HARNESS_RENDER_DIR"]) if not (render_dir / ".opencode").is_dir(): raise SystemExit(f"[ERROR] HARNESS_RENDER_DIR has no .opencode/: {render_dir}") - opencode_dir = Path(env.get("OPENCODE_DIR") or JOBBENCH / "opencode") + opencode_dir = Path(env.get("OPENCODE_DIR") or config.OPENCODE_DIR) if not (opencode_dir / "packages" / "opencode").is_dir(): raise SystemExit(f"[ERROR] OpenCode not installed at {opencode_dir}") tasks_base = Path(env.get("TASKS_BASE_DIR") or JOBBENCH / "dataset" / "main") diff --git a/harness2/benchmarks/jobbench.py b/plugins/harness2-benchmarks/harness2_benchmarks/jobbench.py similarity index 97% rename from harness2/benchmarks/jobbench.py rename to plugins/harness2-benchmarks/harness2_benchmarks/jobbench.py index 3d5e1720..a670b84d 100644 --- a/harness2/benchmarks/jobbench.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/jobbench.py @@ -33,14 +33,17 @@ from datetime import datetime, timezone from pathlib import Path -from .. import config -from ..core import runmark as RM -from ..core import tree as HT -from ..core.spec import BenchPrompts, BenchSpec, resolve_channel_model -from ..install import codex_config_error -from ..substrate.base import Install, RunRequest -from ..substrate.codex import CodexSubstrate -from ..substrate.opencode import OUTPUT_TOKEN_MAX, OpencodeSubstrate +from harness2.core import runmark as RM +from harness2.core import tree as HT +from harness2.core.spec import BenchPrompts, BenchSpec +from harness2.substrate.base import Install, RunRequest +from harness2.substrate.codex import CodexSubstrate +from harness2.substrate.opencode import OUTPUT_TOKEN_MAX, OpencodeSubstrate + +from . import config +from .config import resolve_channel_model +from .install import codex_config_error +from .preflight import codex as codex_preflight from .tasks import discover, select _RUNNER_DIR = config.THIRD_PARTY / "jb_runner" @@ -500,17 +503,17 @@ def render_opencode_tree(staged: Path, render_dir: Path, ctx: dict) -> None: def _channels() -> dict: - """Improver channel, resolved through core.spec.resolve_channel_model.""" + """Improver channel, resolved through config.resolve_channel_model.""" ch = resolve_channel_model(None, None, effort="high") return {"proposer": ch, "selector": ch} -def make_spec(substrate: str) -> BenchSpec: +def make_spec(substrate: str, runs_root: Path) -> BenchSpec: if substrate not in _INSTALL_MODE: raise ValueError(f"JobBench substrate must be 'oc' or 'cx', got {substrate!r}") return BenchSpec( name="jb", - runs_root=config.RUNS / "jb", + runs_root=Path(runs_root), install_mode=_INSTALL_MODE[substrate], prompts=jobbench_prompts(), channels=_channels(), @@ -626,12 +629,13 @@ def __init__( self, substrate: str = "oc", *, + runs_root: Path, timeout: int | None = None, store_root: Path | None = None, split_file: Path | None = None, ): self.split_file = split_file - self.spec = make_spec(substrate) + self.spec = make_spec(substrate, runs_root) launcher = self._launch_opencode if substrate == "oc" else self._launch_codex self.substrate = ( OpencodeSubstrate(launcher=launcher) @@ -639,7 +643,7 @@ def __init__( else CodexSubstrate(launcher=launcher) ) self.timeout = int(timeout or _DEFAULT_TIMEOUT[substrate]) - self.store = Path(store_root) if store_root else config.RUNS / "jb" / "dataset" + self.store = Path(store_root) if store_root else self.spec.runs_root / "dataset" def tasks(self, split: str = "all") -> list[str]: return select(discover("jb"), split, self.split_file, benchmark="jb") @@ -658,8 +662,8 @@ def task_prompt(self, task_id: str) -> str: p = self._dataset_task(task_id) / "task_folder" / "TASK_INSTRUCTIONS.txt" if not p.is_file(): raise FileNotFoundError( - f"task instructions missing: {p} — re-materialise dependencies/job-bench-eval's " - f"dataset/main (recipe in dependencies/PINNED.md)" + f"task instructions missing: {p} — re-materialise {config.jb_repo()}'s " + f"dataset/main (recipe in harness2_benchmarks/dependencies_meta/PINNED.md)" ) return p.read_text(encoding="utf-8", errors="replace") @@ -697,8 +701,8 @@ def _ensure_store_task(self, task_id: str) -> Path: src = self._dataset_task(task_id) / "task_folder" if not src.is_dir(): raise FileNotFoundError( - f"dataset task inputs missing: {src} — re-materialise dependencies/job-bench-eval " - f"(recipe in dependencies/PINNED.md)" + f"dataset task inputs missing: {src} — re-materialise {config.jb_repo()} " + f"(recipe in harness2_benchmarks/dependencies_meta/PINNED.md)" ) dst.parent.mkdir(parents=True, exist_ok=True) with _STORE_SEED_LOCK: @@ -785,7 +789,6 @@ def _launch_common(self, req: RunRequest) -> tuple[str, str, int]: def _launch_opencode(self, req: RunRequest, install: Install) -> list[Path]: task_id, label, t = self._launch_common(req) - self.substrate.preflight() render_dir = Path(tempfile.mkdtemp(prefix=f"hrender_{label}_")) render_opencode_tree(install.staging_dir, render_dir, _render_ctx(self.domain_of(task_id))) env = { @@ -807,7 +810,7 @@ def _launch_opencode(self, req: RunRequest, install: Install) -> list[Path]: def _launch_codex(self, req: RunRequest, install: Install) -> list[Path]: task_id, label, t = self._launch_common(req) - self.substrate.preflight() + codex_preflight() codex_home = config.CODEX_HOME_DIR stale = codex_config_error() if stale is not None: @@ -815,6 +818,8 @@ def _launch_codex(self, req: RunRequest, install: Install) -> list[Path]: env = self.substrate.exec_env(os.environ, codex_home=codex_home) env.update( { + # the pinned Codex dist ships its helper binaries beside the CLI + "PATH": str(Path(config.CODEX_DIST) / "codex-path") + os.pathsep + env["PATH"], "HARNESS2_CODEX_BIN": str(Path(config.CODEX_DIST) / "codex"), "TASK_IDS": task_id, "LABEL": label, @@ -952,7 +957,7 @@ def base_harness(self) -> Path: if not HT.is_tree(self.substrate.surface(), base): raise FileNotFoundError( f"JobBench base harness missing or not a {self.substrate.surface().name} " - f"tree: {base} — restore third_party/jb_runner/; an absent base would make " + f"tree: {base} — restore {config.THIRD_PARTY / 'jb_runner'}/; an absent base would make " f"every base rollout silently harness-free" ) return base @@ -963,11 +968,12 @@ def validate_tree(self, tree: dict) -> None: def preflight(self) -> None: """Check everything money-adjacent before a rollout is paid for.""" - self.substrate.preflight() + if self.substrate.name == "cx": + codex_preflight() judge_py = config.jb_repo() / "eval" / "judge.py" if not judge_py.is_file(): raise RuntimeError( - f"upstream JobBench judge missing at {judge_py}; run scripts/setup_benchmark.py jb" + f"upstream JobBench judge missing at {judge_py}; run harness2 setup jb" ) if self.substrate.name == "cx": config.require("HARNESS2_CODEX_API_KEY", needed_by="JB codex cells") @@ -978,7 +984,7 @@ def preflight(self) -> None: if stale is not None: raise RuntimeError(stale) py = config.JB_JUDGE_PYTHON - remedy = "python scripts/setup_benchmark.py jb" + remedy = "harness2 setup jb" if not py.is_file(): raise RuntimeError(f"JB judge interpreter missing at {py}; {remedy}") try: @@ -1078,7 +1084,7 @@ def _run_judge(self, task_id: str, label: str, run_dir: Path) -> bool: py = config.JB_JUDGE_PYTHON if not py.is_file(): print( - f" [jb] judge interpreter missing ({py}); run scripts/setup_benchmark.py jb " + f" [jb] judge interpreter missing ({py}); run harness2 setup jb " f"to a python with pandas/openpyxl/python-docx/mammoth/pdfplumber -- judge " f"not run (None, never 0)", flush=True, diff --git a/harness2/benchmarks/lab.py b/plugins/harness2-benchmarks/harness2_benchmarks/lab.py similarity index 97% rename from harness2/benchmarks/lab.py rename to plugins/harness2-benchmarks/harness2_benchmarks/lab.py index 77320b21..600e7df2 100644 --- a/harness2/benchmarks/lab.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/lab.py @@ -31,13 +31,16 @@ from functools import cache from pathlib import Path -from .. import config -from ..core import runmark as RM -from ..core import tree as HT -from ..core.spec import BenchPrompts, BenchSpec, resolve_channel_model -from ..substrate.base import Install, RunRequest -from ..substrate.codex import CODEX_SURFACE, CodexSubstrate -from ..substrate.opencode import OPENCODE_SURFACE, OpencodeSubstrate +from harness2.core import runmark as RM +from harness2.core import tree as HT +from harness2.core.spec import BenchPrompts, BenchSpec +from harness2.substrate.base import Install, RunRequest +from harness2.substrate.codex import CODEX_SURFACE, CodexSubstrate +from harness2.substrate.opencode import OPENCODE_SURFACE, OpencodeSubstrate + +from . import config +from .config import resolve_channel_model +from .preflight import codex as codex_preflight from .tasks import discover, select JUDGE_MODEL = config.LAB_JUDGE_MODEL @@ -424,8 +427,8 @@ def _require_runner(path: Path) -> None: if not Path(path).is_file(): raise RuntimeError( f"upstream LAB runner missing: {path}. The upstream clone does not track it at " - f"the pinned commit; it ships as the tracked overlay third_party/lab_runner/ — " - f"run scripts/setup_benchmark.py to copy it into place." + f"the pinned commit; it ships as the tracked overlay {config.THIRD_PARTY / 'lab_runner'}/ — " + f"run harness2 setup to copy it into place." ) @@ -470,19 +473,19 @@ def _require_runtime_scripts() -> frozenset[str]: f"no skill scripts under {skills} — an empty exemption set is precisely the " f"condition this derivation exists to prevent, and it fails silently as mass " f"fell_back rather than as an error; materialise the LAB checkout with " - f"scripts/setup_benchmark.py lab" + f"harness2 setup lab" ) return names -def lab_spec(substrate_name: str) -> BenchSpec: +def lab_spec(substrate_name: str, runs_root: Path) -> BenchSpec: """The LAB BenchSpec for one substrate cell. A value, not a code path.""" if substrate_name not in ("oc", "cx"): raise ValueError(f"substrate must be 'oc' or 'cx', got {substrate_name!r}") channel = resolve_channel_model(None, None, effort="high") return BenchSpec( name="lab", - runs_root=config.RUNS / "lab", + runs_root=Path(runs_root), install_mode="workspace_config" if substrate_name == "oc" else "codex_root", prompts=LAB_BASE, channels={"proposer": channel, "selector": channel}, @@ -506,6 +509,7 @@ class LabAdapter: def __init__( self, *, + runs_root: Path, substrate: str = "oc", model: str | None = None, timeout: int | None = None, @@ -517,16 +521,16 @@ def __init__( self.split_file = split_file if skill_mode not in ("native", "inline"): raise ValueError(f"skill_mode must be 'native' or 'inline', got {skill_mode!r}") - self.spec = lab_spec(substrate) + self.spec = lab_spec(substrate, runs_root) if substrate == "oc": self.substrate = OpencodeSubstrate(launcher=self._launch_oc) self.model = model or config.SOLVER_MODEL_OC - self._results_root = config.RUNS / "lab" / "results" / "oc" + self._results_root = self.spec.runs_root / "results" / "oc" else: self.substrate = CodexSubstrate(launcher=self._launch_cx) self.model = model or config.SOLVER_MODEL_CX - self._results_root = config.RUNS / "lab" / "results" / "cx" + self._results_root = self.spec.runs_root / "results" / "cx" self.timeout = int(timeout or 3600) self.skill_mode = skill_mode self.solver_effort = solver_effort @@ -712,7 +716,7 @@ def _launch_cx(self, req: RunRequest, install: Install) -> list[Path]: "ADAPT_LABCX_CODEX_BIN": str(Path(config.CODEX_DIST) / "codex"), "ADAPT_LABCX_RESULTS": str(self._results_root), "ADAPT_LABCX_SCRATCH": os.environ.get("ADAPT_LABCX_SCRATCH") - or str(config.RUNS / "lab" / "scratch"), + or str(self.spec.runs_root / "scratch"), "ADAPT_LABCX_BASE_URL": os.environ.get("ADAPT_LABCX_BASE_URL") or config.CODEX_BASE_URL, } @@ -930,10 +934,11 @@ def _check_tool_gate(self, run_dir: Path) -> None: def preflight(self) -> None: """Everything money-adjacent, checked before a rollout is paid for.""" - self.substrate.preflight() + if self.substrate.name == "cx": + codex_preflight() if not config.LAB_PYTHON.is_file(): raise RuntimeError( - f"LAB interpreter missing at {config.LAB_PYTHON}; run python scripts/setup_benchmark.py lab" + f"LAB interpreter missing at {config.LAB_PYTHON}; run harness2 setup lab" ) if self.substrate.name == "oc": _require_runner(config.lab_repo() / "opencode" / "run_opencode.py") @@ -955,7 +960,7 @@ def preflight(self) -> None: raise RuntimeError( f"`import evaluation.scoring` fails under {config.LAB_PYTHON} " f"({r.stderr.strip().splitlines()[-1] if r.stderr.strip() else 'no output'}) " - f"— run python scripts/setup_benchmark.py lab to install its dependency environment" + f"— run harness2 setup lab to install its dependency environment" ) def score(self, run_dir: Path, task_id: str) -> dict | None: diff --git a/harness2/preflight.py b/plugins/harness2-benchmarks/harness2_benchmarks/preflight.py similarity index 79% rename from harness2/preflight.py rename to plugins/harness2-benchmarks/harness2_benchmarks/preflight.py index 93651cde..a8ad6ab3 100644 --- a/harness2/preflight.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/preflight.py @@ -18,11 +18,13 @@ import json import shutil +import socket import subprocess +from pathlib import Path +from urllib.parse import urlparse from . import config from .install import PATCHES, missing_runners -from .substrate.opencode import OpencodeSubstrate def check(adapter, *, judge=False) -> list[str]: @@ -31,7 +33,7 @@ def check(adapter, *, judge=False) -> list[str]: return [missing] errors = [] - def probe(label, cmd, *, cwd=None, remedy="run harness2-setup for this benchmark"): + def probe(label, cmd, *, cwd=None, remedy="run harness2 setup "): try: r = subprocess.run( [str(c) for c in cmd], cwd=cwd, capture_output=True, text=True, timeout=45 @@ -46,7 +48,6 @@ def probe(label, cmd, *, cwd=None, remedy="run harness2-setup for this benchmark for validate in ( lambda: config.validate_models(judge=judge), adapter.tasks, - OpencodeSubstrate().preflight, adapter.preflight, ): try: @@ -73,8 +74,15 @@ def probe(label, cmd, *, cwd=None, remedy="run harness2-setup for this benchmark f"{dependency} patch", ["git", "-C", target, "apply", "--reverse", "--check", patch], ) + if not (config.OPENCODE_DIR / "packages/opencode/src/index.ts").is_file(): + errors.append( + f"OpenCode checkout missing at {config.OPENCODE_DIR}; " + f"run harness2 setup {adapter.spec.name}" + ) + if not config.BUN.is_file(): + errors.append(f"bun not found at {config.BUN}; run harness2 setup {adapter.spec.name}") if not (config.OPENCODE_DIR / "node_modules").is_dir(): - errors.append(f"OpenCode dependencies missing; run harness2-setup {bench}") + errors.append(f"OpenCode dependencies missing; run harness2 setup {adapter.spec.name}") if any( m.startswith("google-vertex") for m in (config.SOLVER_MODEL_OC, config.IMPROVER_MODEL) @@ -119,7 +127,7 @@ def probe(label, cmd, *, cwd=None, remedy="run harness2-setup for this benchmark probe( "WorkBuddy document rendering", [config.WB_RENDER_PY, "-c", "import openpyxl, pptx"], - remedy="install 'harness2[render]' in the main environment", + remedy="install 'harness2-benchmarks[render]' in the main environment", ) probe( "WorkBuddy runtime", @@ -158,3 +166,23 @@ def require_ready(adapter, *, judge=False): raise config.ConfigError( "Setup incomplete; nothing has run yet:\n - " + "\n - ".join(errors) ) + + +def codex(*, require_bridge: bool = False) -> None: + """Check the CLI and Responses endpoint, including container configuration.""" + config.require("HARNESS2_CODEX_BASE_URL", needed_by="Codex Responses endpoint") + config.require("HARNESS2_CODEX_API_KEY", needed_by="Codex endpoint authentication") + url = urlparse(config.CODEX_BASE_URL) + if url.scheme not in ("http", "https") or not url.hostname: + raise config.ConfigError("HARNESS2_CODEX_BASE_URL must be an HTTP(S) URL") + binary = Path(config.CODEX_DIST) / "codex" + if not binary.is_file(): + raise RuntimeError("Codex is not installed; run harness2 setup --substrate cx") + if require_bridge and not config.CODEX_CONTAINER_URL: + raise RuntimeError( + "Set HARNESS2_CODEX_CONTAINER_URL to a container-accessible Responses endpoint" + ) + with socket.create_connection( + (url.hostname, url.port or (443 if url.scheme == "https" else 80)), timeout=5 + ): + pass diff --git a/plugins/harness2-benchmarks/harness2_benchmarks/setup.py b/plugins/harness2-benchmarks/harness2_benchmarks/setup.py new file mode 100644 index 00000000..716ae3aa --- /dev/null +++ b/plugins/harness2-benchmarks/harness2_benchmarks/setup.py @@ -0,0 +1,120 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Install a benchmark's pinned tools, runner overlays, dependencies, and data.""" + +from __future__ import annotations + +import argparse +import shutil +import subprocess + +from . import config, install + + +def main(argv=None): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("benchmark", choices=("jb", "lab", "wb")) + p.add_argument( + "--subset", + choices=("office", "web", "code"), + default=None, + help="wb only: one subset to install; omitted installs office, web, and code", + ) + p.add_argument("--substrate", choices=("oc", "cx"), default="oc") + p.add_argument("--dry-run", action="store_true") + args = p.parse_args(argv) + if args.subset and args.benchmark != "wb": + p.error("--subset only applies to wb") + dry = args.dry_run + missing = install.missing_runners() + if missing and not dry: + p.exit(1, f"setup failed: {missing}\n") + if missing: + print(f"Note: {missing}") + name = {"jb": "job-bench-eval", "lab": "harvey-labs", "wb": "workbuddy-bench"}[args.benchmark] + try: + if not dry: + for binary in ("git", "uv", "npm") + ( + ("docker", "curl") if args.benchmark == "wb" else () + ): + if not shutil.which(binary): + raise RuntimeError(f"Missing {binary}; see README.md") + install.clone("opencode", dry_run=dry) + install.patch("opencode", dry_run=dry) + install.run( + ["npm", "install", "--prefix", config.DEPENDENCIES / "tools", "bun@1.3.11"], dry_run=dry + ) + install.run( + [config.BUN, "install", "--frozen-lockfile", "--ignore-scripts"], + cwd=config.OPENCODE_DIR, + dry_run=dry, + ) + install.clone(name, dry_run=dry) + install.patch(name, dry_run=dry) + install.overlays(name, dry_run=dry) + repo = config.DEPENDENCIES / name + install.run(["uv", "sync", "--locked", "--python", "3.12"], cwd=repo, dry_run=dry) + if args.benchmark == "jb": + install.run( + ["uv", "pip", "install", "--python", repo / ".venv/bin/python", "python-docx"], + dry_run=dry, + ) + install.download_jobbench(dry_run=dry) + elif args.benchmark == "lab": + install.run( + [ + "uv", + "pip", + "install", + "--python", + repo / ".venv/bin/python", + "google-auth[requests]", + ], + dry_run=dry, + ) + else: + install.workbuddy_models(dry_run=dry) + subsets = [args.subset] if args.subset else ["office", "web", "code"] + install.run( + ["bash", "scripts/dataset/fetch-dataset.sh", *subsets], cwd=repo, dry_run=dry + ) + if args.substrate == "oc": + install.run( + [ + "uv", + "run", + "--no-sync", + "bash", + "scripts/harness/build-harness-mounts.sh", + "--harness", + "opencode/1.14.18", + ], + cwd=repo, + dry_run=dry, + ) + if args.substrate == "cx": + install.codex(dry_run=dry) + except (OSError, RuntimeError, ValueError, subprocess.CalledProcessError) as exc: + p.exit(1, f"setup failed: {exc}\n") + print( + "Setup plan complete." + if dry + else "Installation complete. Run harness2 doctor before an experiment." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/plugins/harness2-benchmarks/harness2_benchmarks/suites/__init__.py b/plugins/harness2-benchmarks/harness2_benchmarks/suites/__init__.py new file mode 100644 index 00000000..2d3cc526 --- /dev/null +++ b/plugins/harness2-benchmarks/harness2_benchmarks/suites/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Benchmark suite entry points.""" + +from __future__ import annotations diff --git a/plugins/harness2-benchmarks/harness2_benchmarks/suites/jb.py b/plugins/harness2-benchmarks/harness2_benchmarks/suites/jb.py new file mode 100644 index 00000000..fd4c6e43 --- /dev/null +++ b/plugins/harness2-benchmarks/harness2_benchmarks/suites/jb.py @@ -0,0 +1,38 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""jb suite entry point.""" + +from __future__ import annotations + +from .. import preflight +from ..config import ConfigError +from ..jobbench import JobBenchAdapter +from ..setup import main + + +def make_adapter(ctx): + try: + substrate = {"opencode": "oc", "codex": "cx"}[ctx.settings.get("executor.name")] + except KeyError as exc: + raise ConfigError("the jb suite runs on the opencode or codex executor") from exc + return JobBenchAdapter(substrate=substrate, runs_root=ctx.run_root, split_file=ctx.split_file) + + +def check(ctx) -> list[str]: + return preflight.check(make_adapter(ctx), judge=True) + + +def setup(argv: list[str]) -> int: + return main(["jb", *argv]) diff --git a/plugins/harness2-benchmarks/harness2_benchmarks/suites/lab.py b/plugins/harness2-benchmarks/harness2_benchmarks/suites/lab.py new file mode 100644 index 00000000..e5ff4fce --- /dev/null +++ b/plugins/harness2-benchmarks/harness2_benchmarks/suites/lab.py @@ -0,0 +1,38 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""lab suite entry point.""" + +from __future__ import annotations + +from .. import preflight +from ..config import ConfigError +from ..lab import LabAdapter +from ..setup import main + + +def make_adapter(ctx): + try: + substrate = {"opencode": "oc", "codex": "cx"}[ctx.settings.get("executor.name")] + except KeyError as exc: + raise ConfigError("the lab suite runs on the opencode or codex executor") from exc + return LabAdapter(substrate=substrate, runs_root=ctx.run_root, split_file=ctx.split_file) + + +def check(ctx) -> list[str]: + return preflight.check(make_adapter(ctx), judge=True) + + +def setup(argv: list[str]) -> int: + return main(["lab", *argv]) diff --git a/plugins/harness2-benchmarks/harness2_benchmarks/suites/wb_code.py b/plugins/harness2-benchmarks/harness2_benchmarks/suites/wb_code.py new file mode 100644 index 00000000..a8b2b70f --- /dev/null +++ b/plugins/harness2-benchmarks/harness2_benchmarks/suites/wb_code.py @@ -0,0 +1,38 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""wb-code suite entry point.""" + +from __future__ import annotations + +from .. import preflight +from ..config import ConfigError +from ..setup import main +from ..workbuddy import WbAdapter + + +def make_adapter(ctx): + try: + substrate = {"opencode": "oc", "codex": "cx"}[ctx.settings.get("executor.name")] + except KeyError as exc: + raise ConfigError("the wb-code suite runs on the opencode or codex executor") from exc + return WbAdapter("code", substrate=substrate, runs_root=ctx.run_root, split_file=ctx.split_file) + + +def check(ctx) -> list[str]: + return preflight.check(make_adapter(ctx), judge=True) + + +def setup(argv: list[str]) -> int: + return main(["wb", "--subset", "code", *argv]) diff --git a/plugins/harness2-benchmarks/harness2_benchmarks/suites/wb_office.py b/plugins/harness2-benchmarks/harness2_benchmarks/suites/wb_office.py new file mode 100644 index 00000000..b656da92 --- /dev/null +++ b/plugins/harness2-benchmarks/harness2_benchmarks/suites/wb_office.py @@ -0,0 +1,40 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""wb-office suite entry point.""" + +from __future__ import annotations + +from .. import preflight +from ..config import ConfigError +from ..setup import main +from ..workbuddy import WbAdapter + + +def make_adapter(ctx): + try: + substrate = {"opencode": "oc", "codex": "cx"}[ctx.settings.get("executor.name")] + except KeyError as exc: + raise ConfigError("the wb-office suite runs on the opencode or codex executor") from exc + return WbAdapter( + "office", substrate=substrate, runs_root=ctx.run_root, split_file=ctx.split_file + ) + + +def check(ctx) -> list[str]: + return preflight.check(make_adapter(ctx), judge=True) + + +def setup(argv: list[str]) -> int: + return main(["wb", "--subset", "office", *argv]) diff --git a/plugins/harness2-benchmarks/harness2_benchmarks/suites/wb_web.py b/plugins/harness2-benchmarks/harness2_benchmarks/suites/wb_web.py new file mode 100644 index 00000000..cb1baca0 --- /dev/null +++ b/plugins/harness2-benchmarks/harness2_benchmarks/suites/wb_web.py @@ -0,0 +1,38 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""wb-web suite entry point.""" + +from __future__ import annotations + +from .. import preflight +from ..config import ConfigError +from ..setup import main +from ..workbuddy import WbAdapter + + +def make_adapter(ctx): + try: + substrate = {"opencode": "oc", "codex": "cx"}[ctx.settings.get("executor.name")] + except KeyError as exc: + raise ConfigError("the wb-web suite runs on the opencode or codex executor") from exc + return WbAdapter("web", substrate=substrate, runs_root=ctx.run_root, split_file=ctx.split_file) + + +def check(ctx) -> list[str]: + return preflight.check(make_adapter(ctx), judge=True) + + +def setup(argv: list[str]) -> int: + return main(["wb", "--subset", "web", *argv]) diff --git a/harness2/benchmarks/tasks.py b/plugins/harness2-benchmarks/harness2_benchmarks/tasks.py similarity index 93% rename from harness2/benchmarks/tasks.py rename to plugins/harness2-benchmarks/harness2_benchmarks/tasks.py index 6fff81cd..8281ae71 100644 --- a/harness2/benchmarks/tasks.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/tasks.py @@ -20,7 +20,7 @@ import tomllib from pathlib import Path -from .. import config +from . import config def discover(bench: str, subset: str | None = None) -> dict[str, str]: @@ -53,8 +53,8 @@ def discover(bench: str, subset: str | None = None) -> dict[str, str]: raise ValueError(f"Unknown benchmark/subset: {bench}/{subset}") if not tasks: raise FileNotFoundError( - f"No {bench} tasks found at {root}. Run harness2-setup " - f"{bench}" + (f" --subset {subset}" if subset else "") + "." + f"No {bench} tasks found at {root}. Run harness2 setup " + f"{bench}" + (f"-{subset}" if subset else "") + "." ) return tasks @@ -66,7 +66,7 @@ def select(tasks: dict[str, str], split: str, path: Path | None, *, benchmark: s if split != "all": raise ValueError( "--split train/test requires --split-file; create one with " - "python scripts/split_tasks.py, or use --split all" + "python plugins/harness2-benchmarks/scripts/split_tasks.py, or use --split all" ) return sorted(tasks) data = json.loads(Path(path).read_text(encoding="utf-8")) diff --git a/third_party/README.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/README.md similarity index 71% rename from third_party/README.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/README.md index ba0d8279..618821d7 100644 --- a/third_party/README.md +++ b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/README.md @@ -13,9 +13,9 @@ so setup works outside a source checkout. | `licenses/` | License copies covering those patches | Each runner retains its `LICENSE` and `PROVENANCE.md`. Research logic belongs in -`harness2/core/`; benchmark interfaces belong in `harness2/benchmarks/`. -Setup downloads upstream checkouts and data into the invocation's `dependencies/` +`harness2/core/`; benchmark interfaces belong in `harness2_benchmarks/`. +Setup downloads upstream checkouts and data into the invocation's `~/.cache/harness2/benchmarks/` directory. Those downloads are excluded from version control. -The repository's `dependencies/ASSETS.json` records bundled asset hashes. -Run `python scripts/check_provenance.py` after changing these files. +The repository's `harness2_benchmarks/dependencies_meta/ASSETS.json` records bundled asset hashes. +Run `python plugins/harness2-benchmarks/scripts/check_provenance.py` after changing these files. diff --git a/third_party/jb_runner/LICENSE b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/LICENSE similarity index 100% rename from third_party/jb_runner/LICENSE rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/LICENSE diff --git a/third_party/jb_runner/PROVENANCE.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/PROVENANCE.md similarity index 79% rename from third_party/jb_runner/PROVENANCE.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/PROVENANCE.md index 65e16ced..5f831350 100644 --- a/third_party/jb_runner/PROVENANCE.md +++ b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/PROVENANCE.md @@ -1,7 +1,7 @@ # job-bench-eval runner assets The files in this directory derive from the job-bench-eval integration and retain its -[upstream license](LICENSE). See the [pinned revision and patches](../../../dependencies/PINNED.md). +[upstream license](LICENSE). See the [pinned revision and patches](../../dependencies_meta/PINNED.md). What is still vendored here: @@ -19,9 +19,9 @@ directly from the package. The OpenCode launcher that used to sit beside it, `run_benchmark_opencode_harness_adapt.sh`, was a fork that shared almost nothing with upstream. It has been replaced by the first-party Python module -[`harness2/benchmarks/jb_opencode_runner.py`](../../benchmarks/jb_opencode_runner.py), +[`harness2_benchmarks/jb_opencode_runner.py`](../../jb_opencode_runner.py), which the adapter launches as a subprocess in its place. That module is release code, not a vendored asset, so it is not listed in `ASSETS.json`. -[ASSETS.json](../../../dependencies/ASSETS.json) records the shipped file checksums. -Verify them with `python scripts/check_provenance.py`. +[ASSETS.json](../../dependencies_meta/ASSETS.json) records the shipped file checksums. +Verify them with `python plugins/harness2-benchmarks/scripts/check_provenance.py`. diff --git a/third_party/jb_runner/codex_base_harness/AGENTS.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/codex_base_harness/AGENTS.md similarity index 100% rename from third_party/jb_runner/codex_base_harness/AGENTS.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/codex_base_harness/AGENTS.md diff --git a/third_party/jb_runner/harness_baseline_oc/LongTermMEMORY.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/LongTermMEMORY.md similarity index 100% rename from third_party/jb_runner/harness_baseline_oc/LongTermMEMORY.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/LongTermMEMORY.md diff --git a/third_party/jb_runner/harness_baseline_oc/guardrails.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/guardrails.md similarity index 100% rename from third_party/jb_runner/harness_baseline_oc/guardrails.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/guardrails.md diff --git a/third_party/jb_runner/harness_baseline_oc/harness.yaml b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/harness.yaml similarity index 100% rename from third_party/jb_runner/harness_baseline_oc/harness.yaml rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/harness.yaml diff --git a/third_party/jb_runner/harness_baseline_oc/plugin/README.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/plugin/README.md similarity index 100% rename from third_party/jb_runner/harness_baseline_oc/plugin/README.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/plugin/README.md diff --git a/third_party/jb_runner/harness_baseline_oc/scripts/README.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/scripts/README.md similarity index 100% rename from third_party/jb_runner/harness_baseline_oc/scripts/README.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/scripts/README.md diff --git a/third_party/jb_runner/harness_baseline_oc/sub_agents/README.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/sub_agents/README.md similarity index 100% rename from third_party/jb_runner/harness_baseline_oc/sub_agents/README.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/sub_agents/README.md diff --git a/third_party/jb_runner/harness_baseline_oc/systemprompt.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/systemprompt.md similarity index 100% rename from third_party/jb_runner/harness_baseline_oc/systemprompt.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/systemprompt.md diff --git a/third_party/jb_runner/harness_baseline_oc/tool_descriptions/tool_guidance.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/tool_descriptions/tool_guidance.md similarity index 100% rename from third_party/jb_runner/harness_baseline_oc/tool_descriptions/tool_guidance.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/tool_descriptions/tool_guidance.md diff --git a/third_party/jb_runner/run_benchmark_codex_cli_adapt.sh b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/run_benchmark_codex_cli_adapt.sh similarity index 98% rename from third_party/jb_runner/run_benchmark_codex_cli_adapt.sh rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/run_benchmark_codex_cli_adapt.sh index 0105da58..0dc6cf8e 100644 --- a/third_party/jb_runner/run_benchmark_codex_cli_adapt.sh +++ b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/run_benchmark_codex_cli_adapt.sh @@ -15,7 +15,7 @@ set -e # Which split of the dataset to run against (main | easy). Defaults to main. SPLIT="${SPLIT:-main}" # Target task root (the directory containing all profession subdirectories) -TASKS_BASE_DIR="${TASKS_BASE_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)/dataset/${SPLIT}}" +TASKS_BASE_DIR="${TASKS_BASE_DIR:-${HARNESS2_BENCH_DEPS:-${XDG_CACHE_HOME:-$HOME/.cache}/harness2/benchmarks}/job-bench-eval/dataset/${SPLIT}}" # Optional run label appended to output directory names to avoid clobbering past runs RUN_LABEL="${RUN_LABEL:-}" # TASK_IDS restricts discovery to space-separated profession/taskN identifiers. @@ -490,8 +490,8 @@ main() { # Check the executable installed by the release setup script. echo "Checking Codex CLI availability..." - "${HARNESS2_CODEX_BIN:?Run scripts/setup_benchmark.py with --substrate cx}" --version 2>&1 || { - echo "[ERROR] Codex CLI not available. Run scripts/setup_benchmark.py with --substrate cx." + "${HARNESS2_CODEX_BIN:?Run harness2 setup jb --substrate cx}" --version 2>&1 || { + echo "[ERROR] Codex CLI not available. Run harness2 setup jb --substrate cx." exit 1 } diff --git a/third_party/lab_runner/LICENSE b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/LICENSE similarity index 100% rename from third_party/lab_runner/LICENSE rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/LICENSE diff --git a/third_party/lab_runner/PROVENANCE.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/PROVENANCE.md similarity index 66% rename from third_party/lab_runner/PROVENANCE.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/PROVENANCE.md index 25c29d8f..c9327194 100644 --- a/third_party/lab_runner/PROVENANCE.md +++ b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/PROVENANCE.md @@ -1,11 +1,11 @@ # harvey-labs runner assets These runners derive from the harvey-labs integration and retain its -[upstream license](LICENSE). See the [pinned revision and patches](../../../dependencies/PINNED.md). +[upstream license](LICENSE). See the [pinned revision and patches](../../dependencies_meta/PINNED.md). The integration adds editable harness installation and configurable models, paths, and endpoints. Local model-provider names, environment variables, and log paths use the `harness2` namespace. Setup installs these runners into the LAB checkout as overlays. -[ASSETS.json](../../../dependencies/ASSETS.json) records the shipped file checksums. -Verify them with `python scripts/check_provenance.py`. +[ASSETS.json](../../dependencies_meta/ASSETS.json) records the shipped file checksums. +Verify them with `python plugins/harness2-benchmarks/scripts/check_provenance.py`. diff --git a/third_party/lab_runner/adapt_runs_cx/run_codex_lab.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/adapt_runs_cx/run_codex_lab.py similarity index 99% rename from third_party/lab_runner/adapt_runs_cx/run_codex_lab.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/adapt_runs_cx/run_codex_lab.py index cb4ed81c..ea08a58d 100644 --- a/third_party/lab_runner/adapt_runs_cx/run_codex_lab.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/adapt_runs_cx/run_codex_lab.py @@ -19,6 +19,7 @@ from pathlib import Path RUNNER_DIR = Path(__file__).resolve().parent +# install.overlays copies this runner into the upstream checkout before execution. REPO = RUNNER_DIR.parents[1] if str(REPO) not in sys.path: sys.path.insert(0, str(REPO)) diff --git a/third_party/lab_runner/opencode/README.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/README.md similarity index 100% rename from third_party/lab_runner/opencode/README.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/README.md diff --git a/third_party/lab_runner/opencode/__init__.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/__init__.py similarity index 100% rename from third_party/lab_runner/opencode/__init__.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/__init__.py diff --git a/third_party/lab_runner/opencode/render.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/render.py similarity index 99% rename from third_party/lab_runner/opencode/render.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/render.py index 747c7469..189f9c81 100644 --- a/third_party/lab_runner/opencode/render.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/render.py @@ -20,7 +20,6 @@ load_skills, ) -BENCH_ROOT = Path(__file__).resolve().parent.parent ADAPTER_SCRIPTS = Path(__file__).resolve().parent / "scripts" AGENT_NAME = "lab" diff --git a/third_party/lab_runner/opencode/render_tree.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/render_tree.py similarity index 100% rename from third_party/lab_runner/opencode/render_tree.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/render_tree.py diff --git a/third_party/lab_runner/opencode/run_opencode.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/run_opencode.py similarity index 100% rename from third_party/lab_runner/opencode/run_opencode.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/run_opencode.py diff --git a/third_party/lab_runner/opencode/scripts/read_doc.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/scripts/read_doc.py similarity index 100% rename from third_party/lab_runner/opencode/scripts/read_doc.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/scripts/read_doc.py diff --git a/third_party/lab_runner/opencode/session.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/session.py similarity index 100% rename from third_party/lab_runner/opencode/session.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/session.py diff --git a/third_party/lab_runner/tts_lab/__init__.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/tts_lab/__init__.py similarity index 100% rename from third_party/lab_runner/tts_lab/__init__.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/tts_lab/__init__.py diff --git a/third_party/lab_runner/tts_lab/config.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/tts_lab/config.py similarity index 85% rename from third_party/lab_runner/tts_lab/config.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/tts_lab/config.py index 83eeb647..c5ed8313 100644 --- a/third_party/lab_runner/tts_lab/config.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/tts_lab/config.py @@ -3,6 +3,7 @@ import json from pathlib import Path +# install.overlays copies this runner into the upstream checkout before execution. BENCH_ROOT = Path(__file__).resolve().parents[2] TASKS_DIR = BENCH_ROOT / "tasks" diff --git a/third_party/licenses/harvey-labs.LICENSE b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/licenses/harvey-labs.LICENSE similarity index 100% rename from third_party/licenses/harvey-labs.LICENSE rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/licenses/harvey-labs.LICENSE diff --git a/third_party/licenses/job-bench-eval.LICENSE b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/licenses/job-bench-eval.LICENSE similarity index 100% rename from third_party/licenses/job-bench-eval.LICENSE rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/licenses/job-bench-eval.LICENSE diff --git a/third_party/licenses/opencode.LICENSE b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/licenses/opencode.LICENSE similarity index 100% rename from third_party/licenses/opencode.LICENSE rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/licenses/opencode.LICENSE diff --git a/third_party/licenses/workbuddy-bench.LICENSE b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/licenses/workbuddy-bench.LICENSE similarity index 100% rename from third_party/licenses/workbuddy-bench.LICENSE rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/licenses/workbuddy-bench.LICENSE diff --git a/third_party/patches/harvey-labs-judge-vertex-anthropic.diff b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/patches/harvey-labs-judge-vertex-anthropic.diff similarity index 100% rename from third_party/patches/harvey-labs-judge-vertex-anthropic.diff rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/patches/harvey-labs-judge-vertex-anthropic.diff diff --git a/third_party/patches/job-bench-eval-judge.diff b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/patches/job-bench-eval-judge.diff similarity index 100% rename from third_party/patches/job-bench-eval-judge.diff rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/patches/job-bench-eval-judge.diff diff --git a/third_party/patches/opencode-compat.diff b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/patches/opencode-compat.diff similarity index 100% rename from third_party/patches/opencode-compat.diff rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/patches/opencode-compat.diff diff --git a/third_party/patches/workbuddy-bench-runtime.diff b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/patches/workbuddy-bench-runtime.diff similarity index 100% rename from third_party/patches/workbuddy-bench-runtime.diff rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/patches/workbuddy-bench-runtime.diff diff --git a/third_party/wb_runner/LICENSE b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/LICENSE similarity index 100% rename from third_party/wb_runner/LICENSE rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/LICENSE diff --git a/third_party/wb_runner/PROVENANCE.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/PROVENANCE.md similarity index 68% rename from third_party/wb_runner/PROVENANCE.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/PROVENANCE.md index 4d099148..b33f37b4 100644 --- a/third_party/wb_runner/PROVENANCE.md +++ b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/PROVENANCE.md @@ -1,11 +1,11 @@ # workbuddy-bench runner assets These runners derive from the workbuddy-bench integration and retain its -[upstream license](LICENSE). See the [pinned revision and patches](../../../dependencies/PINNED.md). +[upstream license](LICENSE). See the [pinned revision and patches](../../dependencies_meta/PINNED.md). The integration adds editable harness installation and configurable models, paths, and endpoints. Local model-provider names, environment variables, and log paths use the `harness2` namespace. Setup installs these agents and harness configurations into the WorkBuddy checkout. -[ASSETS.json](../../../dependencies/ASSETS.json) records the shipped file checksums. -Verify them with `python scripts/check_provenance.py`. +[ASSETS.json](../../dependencies_meta/ASSETS.json) records the shipped file checksums. +Verify them with `python plugins/harness2-benchmarks/scripts/check_provenance.py`. diff --git a/third_party/wb_runner/agents/codex_agent.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/agents/codex_agent.py similarity index 99% rename from third_party/wb_runner/agents/codex_agent.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/agents/codex_agent.py index f42dbc39..715b2337 100644 --- a/third_party/wb_runner/agents/codex_agent.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/agents/codex_agent.py @@ -84,7 +84,7 @@ async def install(self, environment: BaseEnvironment) -> None: f'echo "ERROR: codex binary not found at {bin_path}." >&2; ' f'echo " The job must bind-mount the staged dist (read-only):" >&2; ' f'echo " environment_override.mounts: [{{type: bind, ' - f'source: dependencies/codex/dist, target: {self._bin_dir}}}]" >&2; ' + f'source: ~/.cache/harness2/benchmarks/codex/dist, target: {self._bin_dir}}}]" >&2; ' f"exit 1; fi && " f'ln -sf "{bin_path}" /usr/local/bin/codex && ' f'v="$(/usr/local/bin/codex --version 2>&1)" && ' diff --git a/third_party/wb_runner/agents/codex_harness_agent.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/agents/codex_harness_agent.py similarity index 100% rename from third_party/wb_runner/agents/codex_harness_agent.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/agents/codex_harness_agent.py diff --git a/third_party/wb_runner/agents/oc_agent.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/agents/oc_agent.py similarity index 100% rename from third_party/wb_runner/agents/oc_agent.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/agents/oc_agent.py diff --git a/third_party/wb_runner/agents/oc_harness_agent.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/agents/oc_harness_agent.py similarity index 100% rename from third_party/wb_runner/agents/oc_harness_agent.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/agents/oc_harness_agent.py diff --git a/third_party/wb_runner/configs/harnesses/codex-harness/_defaults.yaml b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/codex-harness/_defaults.yaml similarity index 77% rename from third_party/wb_runner/configs/harnesses/codex-harness/_defaults.yaml rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/codex-harness/_defaults.yaml index e29b33f6..622ac3e8 100644 --- a/third_party/wb_runner/configs/harnesses/codex-harness/_defaults.yaml +++ b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/codex-harness/_defaults.yaml @@ -1,4 +1,4 @@ -# The adapter mounts dependencies/codex/dist read-only at /opt/codex and supplies +# The adapter mounts ~/.cache/harness2/benchmarks/codex/dist read-only at /opt/codex and supplies # the model and container-accessible Responses endpoint. The job-local # Chat Completions proxy serves the benchmark judge separately. harness: diff --git a/third_party/wb_runner/configs/harnesses/codex-harness/versions/0.144.6.yaml b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/codex-harness/versions/0.144.6.yaml similarity index 100% rename from third_party/wb_runner/configs/harnesses/codex-harness/versions/0.144.6.yaml rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/codex-harness/versions/0.144.6.yaml diff --git a/third_party/wb_runner/configs/harnesses/oc-harness/_defaults.yaml b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/oc-harness/_defaults.yaml similarity index 100% rename from third_party/wb_runner/configs/harnesses/oc-harness/_defaults.yaml rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/oc-harness/_defaults.yaml diff --git a/third_party/wb_runner/configs/harnesses/oc-harness/versions/1.14.18.yaml b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/oc-harness/versions/1.14.18.yaml similarity index 100% rename from third_party/wb_runner/configs/harnesses/oc-harness/versions/1.14.18.yaml rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/oc-harness/versions/1.14.18.yaml diff --git a/third_party/wb_runner/configs/harnesses/opencode/CONFIG.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/CONFIG.md similarity index 100% rename from third_party/wb_runner/configs/harnesses/opencode/CONFIG.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/CONFIG.md diff --git a/third_party/wb_runner/configs/harnesses/opencode/_defaults.yaml b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/_defaults.yaml similarity index 100% rename from third_party/wb_runner/configs/harnesses/opencode/_defaults.yaml rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/_defaults.yaml diff --git a/third_party/wb_runner/configs/harnesses/opencode/docker/Dockerfile b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/docker/Dockerfile similarity index 100% rename from third_party/wb_runner/configs/harnesses/opencode/docker/Dockerfile rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/docker/Dockerfile diff --git a/third_party/wb_runner/configs/harnesses/opencode/presets/settings/v1_14.json b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/presets/settings/v1_14.json similarity index 100% rename from third_party/wb_runner/configs/harnesses/opencode/presets/settings/v1_14.json rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/presets/settings/v1_14.json diff --git a/third_party/wb_runner/configs/harnesses/opencode/versions/1.14.18.yaml b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/versions/1.14.18.yaml similarity index 100% rename from third_party/wb_runner/configs/harnesses/opencode/versions/1.14.18.yaml rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/versions/1.14.18.yaml diff --git a/harness2/benchmarks/wb_render.py b/plugins/harness2-benchmarks/harness2_benchmarks/wb_render.py similarity index 99% rename from harness2/benchmarks/wb_render.py rename to plugins/harness2-benchmarks/harness2_benchmarks/wb_render.py index 3c5bee61..48568306 100644 --- a/harness2/benchmarks/wb_render.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/wb_render.py @@ -276,7 +276,7 @@ def _have(mod: str) -> bool: def _bridge_interpreter() -> str: """The renderer interpreter; empty when invoked as the bridge itself.""" try: - from .. import config + from . import config except ImportError: return "" return str(config.WB_RENDER_PY) diff --git a/harness2/benchmarks/workbuddy.py b/plugins/harness2-benchmarks/harness2_benchmarks/workbuddy.py similarity index 96% rename from harness2/benchmarks/workbuddy.py rename to plugins/harness2-benchmarks/harness2_benchmarks/workbuddy.py index 27a95b5c..45e78b34 100644 --- a/harness2/benchmarks/workbuddy.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/workbuddy.py @@ -35,16 +35,19 @@ from functools import lru_cache from pathlib import Path -from .. import config -from ..core import runmark as RM -from ..core import tree as HT -from ..core.selection import order_graded_first -from ..core.spec import BenchPrompts, BenchSpec, resolve_channel_model -from ..substrate import opencode as OC -from ..substrate.base import Install, RunRequest -from ..substrate.codex import CODEX_SURFACE, CodexSubstrate -from ..substrate.opencode import OPENCODE_SURFACE, OpencodeSubstrate +from harness2.core import runmark as RM +from harness2.core import tree as HT +from harness2.core.selection import order_graded_first +from harness2.core.spec import BenchPrompts, BenchSpec +from harness2.substrate import opencode as OC +from harness2.substrate.base import Install, RunRequest +from harness2.substrate.codex import CODEX_SURFACE, CodexSubstrate +from harness2.substrate.opencode import OPENCODE_SURFACE, OpencodeSubstrate + +from . import config from . import wb_render as RENDER +from .config import resolve_channel_model +from .preflight import codex as codex_preflight from .tasks import discover, select SCOREABLE = frozenset({"code", "web", "office"}) @@ -540,7 +543,7 @@ def _job_yaml( else "" ) head = ( - f"# GENERATED by harness2/benchmarks/workbuddy.py -- ephemeral, safe to " + f"# GENERATED by harness2_benchmarks/workbuddy.py -- ephemeral, safe to " f"delete. label={label}\n" ) return ( @@ -650,12 +653,12 @@ def run_batch( return sorted(set(_run_dirs(label_dir)) - before) -def spec_for(subset: str, substrate: str) -> BenchSpec: +def spec_for(subset: str, substrate: str, runs_root: Path) -> BenchSpec: s = SUBSETS[subset] channel = resolve_channel_model(None, None, effort="high") return BenchSpec( name=f"wb-{subset}", - runs_root=config.RUNS / "wb" / subset, + runs_root=Path(runs_root), install_mode="home_dir_jinja" if substrate == "oc" else "codex_root", prompts=workbuddy_prompts(subset), channels={"proposer": channel, "selector": channel}, @@ -672,7 +675,12 @@ class WbAdapter: """One (subset, substrate) cell of WB-Bench. A different cell is a different instance.""" def __init__( - self, subset: str = "office", *, substrate: str = "oc", split_file: Path | None = None + self, + subset: str = "office", + *, + runs_root: Path, + substrate: str = "oc", + split_file: Path | None = None, ): self.split_file = split_file if subset not in SUBSETS: @@ -681,7 +689,7 @@ def __init__( raise ValueError(f"unknown substrate {substrate!r}; one of {SUBSTRATES}") self.subset = subset self.subset_info = SUBSETS[subset] - self.spec = spec_for(subset, substrate) + self.spec = spec_for(subset, substrate, runs_root) self.substrate = ( OpencodeSubstrate(launcher=self._launch) if substrate == "oc" @@ -776,7 +784,7 @@ def _launch(self, req: RunRequest, install: Install) -> list[Path]: k = len(req.run_ids) label = self._label(req.run_ids[0], k) if self.substrate.name == "cx": - self.substrate.preflight(require_dist=True, require_bridge=True) + codex_preflight(require_bridge=True) def _have(task_id: str) -> int: return sum( @@ -936,13 +944,13 @@ def preflight(self) -> None: if shutil.which("uv") is None: raise RuntimeError( "`uv` is not on PATH; WB batches run as `uv run ./scripts/run.sh` inside " - "the benchmark repo (see docs/BENCHMARKS.md)" + "the benchmark repo (see plugins/harness2-benchmarks/docs/BENCHMARKS.md)" ) repo = config.wb_repo() if not (repo / "scripts" / "run.sh").is_file(): raise RuntimeError( f"the WorkBuddy-Bench checkout at {repo} has no scripts/run.sh — run " - f"scripts/setup_benchmark.py wb" + f"harness2 setup wb-{self.subset}" ) slugs = ( [_harness_oc(fork=False), _harness_oc(fork=True)] @@ -955,13 +963,11 @@ def preflight(self) -> None: if not entry.is_file(): raise RuntimeError( f"harness entry {slug!r} is missing at {entry} — the harness configs " - f"ship as third_party/wb_runner/configs/harnesses/; run " - f"python scripts/setup_benchmark.py wb" + f"ship as {config.THIRD_PARTY / 'wb_runner/configs/harnesses'}/; run " + f"harness2 setup wb-{self.subset}" ) if self.substrate.name == "cx": - self.substrate.preflight(require_dist=True, require_bridge=True) - else: - self.substrate.preflight() + codex_preflight(require_bridge=True) def _killed(self, run_dir: Path) -> bool: """True when the capture stream ends on a provider error object, i.e. the rollout was diff --git a/plugins/harness2-benchmarks/pyproject.toml b/plugins/harness2-benchmarks/pyproject.toml new file mode 100644 index 00000000..f2ffa175 --- /dev/null +++ b/plugins/harness2-benchmarks/pyproject.toml @@ -0,0 +1,46 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[build-system] +requires = ["setuptools>=77", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "harness2-benchmarks" +version = "0.1.0" +description = "The Harness² paper benchmarks (JobBench, LAB, WorkBuddy-Bench) as harness2 suites" +readme = "README.md" +license = "Apache-2.0" +requires-python = ">=3.12,<3.14" +dependencies = ["harness2-cli"] + +[project.optional-dependencies] +render = ["openpyxl>=3.1,<4", "python-pptx>=1,<2"] + +[project.entry-points."harness2.suites"] +jb = "harness2_benchmarks.suites.jb" +lab = "harness2_benchmarks.suites.lab" +wb-office = "harness2_benchmarks.suites.wb_office" +wb-web = "harness2_benchmarks.suites.wb_web" +wb-code = "harness2_benchmarks.suites.wb_code" + +[tool.setuptools] +include-package-data = false + +[tool.setuptools.packages.find] +include = ["harness2_benchmarks*"] +namespaces = false + +[tool.setuptools.package-data] +harness2_benchmarks = ["third_party/**/*", "dependencies_meta/**/*"] diff --git a/scripts/check_provenance.py b/plugins/harness2-benchmarks/scripts/check_provenance.py similarity index 82% rename from scripts/check_provenance.py rename to plugins/harness2-benchmarks/scripts/check_provenance.py index 84dd8a36..90b28f06 100644 --- a/scripts/check_provenance.py +++ b/plugins/harness2-benchmarks/scripts/check_provenance.py @@ -13,17 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Verify the release's runner and patch files against dependencies/ASSETS.json.""" +"""Verify the release's runner and patch files against harness2_benchmarks/dependencies_meta/ASSETS.json.""" import hashlib import json -from pathlib import Path + +from harness2_benchmarks.config import PKG def main(): - root = Path(__file__).resolve().parents[1] + root = PKG try: - expected = json.loads((root / "dependencies/ASSETS.json").read_text()) + expected = json.loads((root / "dependencies_meta/ASSETS.json").read_text()) failures = [] for name, digest in expected.items(): path = root / name diff --git a/scripts/check_setup.py b/plugins/harness2-benchmarks/scripts/check_setup.py similarity index 79% rename from scripts/check_setup.py rename to plugins/harness2-benchmarks/scripts/check_setup.py index a6d231f4..a32dbe29 100644 --- a/scripts/check_setup.py +++ b/plugins/harness2-benchmarks/scripts/check_setup.py @@ -13,9 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Compatibility entry point for harness2.cli.check.""" +"""Compatibility entry point for harness2 doctor.""" -from harness2.cli.check import main +import subprocess +import sys if __name__ == "__main__": - raise SystemExit(main()) + raise SystemExit(subprocess.call(["harness2", "doctor", *sys.argv[1:]])) diff --git a/scripts/run_jobbench.sh b/plugins/harness2-benchmarks/scripts/run_jobbench.sh similarity index 59% rename from scripts/run_jobbench.sh rename to plugins/harness2-benchmarks/scripts/run_jobbench.sh index b00204e1..732e2551 100644 --- a/scripts/run_jobbench.sh +++ b/plugins/harness2-benchmarks/scripts/run_jobbench.sh @@ -14,23 +14,28 @@ # limitations under the License. # Run JobBench end to end: check, adapt, judge, report. -# bash scripts/run_jobbench.sh # every task -# bash scripts/run_jobbench.sh --only-domains biostatisticians --max-tasks 1 +# bash plugins/harness2-benchmarks/scripts/run_jobbench.sh # every task +# bash plugins/harness2-benchmarks/scripts/run_jobbench.sh --only-domains biostatisticians --max-tasks 1 # Settings: SUBSTRATE=oc|cx K=1 MODE=parallel|sequential # Extra arguments go to the adaptation command; --dry-run stops after it. set -uo pipefail -cd "$(dirname "$0")/.." -PATH="$PWD/.venv/bin:$PATH" +PATH="$(cd "$(dirname "$0")/../../.." && pwd)/.venv/bin:$PATH" SUBSTRATE=${SUBSTRATE:-oc} +case "$SUBSTRATE" in + oc) EXECUTOR=opencode ;; + cx) EXECUTOR=codex ;; + *) exit 2 ;; +esac K=${K:-1} MODE=${MODE:-parallel} -run=(--bench jb --substrate "$SUBSTRATE" --k "$K" --mode "$MODE") +name="jb-${EXECUTOR}-k${K}-${MODE}" +run=(--suite jb --name "$name" --k "$K" --mode "$MODE") -harness2-check --bench jb --substrate "$SUBSTRATE" || exit -harness2 "${run[@]}" "$@" +harness2 -c executor.name="$EXECUTOR" -c suite.name=jb doctor || exit +harness2 -c executor.name="$EXECUTOR" run "${run[@]}" "$@" status=$? [[ " $* " == *" --dry-run "* ]] && exit $status -harness2-judge "${run[@]}" +harness2 judge "$name" judged=$? -harness2-report "${run[@]}" +harness2 report "$name" exit $((status ? status : judged)) diff --git a/scripts/run_lab.sh b/plugins/harness2-benchmarks/scripts/run_lab.sh similarity index 59% rename from scripts/run_lab.sh rename to plugins/harness2-benchmarks/scripts/run_lab.sh index df94d803..7e81aad0 100644 --- a/scripts/run_lab.sh +++ b/plugins/harness2-benchmarks/scripts/run_lab.sh @@ -14,23 +14,28 @@ # limitations under the License. # Run LAB end to end: check, adapt, judge, report. -# bash scripts/run_lab.sh # every task -# bash scripts/run_lab.sh --only-domains antitrust-competition --max-tasks 1 +# bash plugins/harness2-benchmarks/scripts/run_lab.sh # every task +# bash plugins/harness2-benchmarks/scripts/run_lab.sh --only-domains antitrust-competition --max-tasks 1 # Settings: SUBSTRATE=oc|cx K=1 MODE=parallel|sequential # Extra arguments go to the adaptation command; --dry-run stops after it. set -uo pipefail -cd "$(dirname "$0")/.." -PATH="$PWD/.venv/bin:$PATH" +PATH="$(cd "$(dirname "$0")/../../.." && pwd)/.venv/bin:$PATH" SUBSTRATE=${SUBSTRATE:-oc} +case "$SUBSTRATE" in + oc) EXECUTOR=opencode ;; + cx) EXECUTOR=codex ;; + *) exit 2 ;; +esac K=${K:-1} MODE=${MODE:-parallel} -run=(--bench lab --substrate "$SUBSTRATE" --k "$K" --mode "$MODE") +name="lab-${EXECUTOR}-k${K}-${MODE}" +run=(--suite lab --name "$name" --k "$K" --mode "$MODE") -harness2-check --bench lab --substrate "$SUBSTRATE" || exit -harness2 "${run[@]}" "$@" +harness2 -c executor.name="$EXECUTOR" -c suite.name=lab doctor || exit +harness2 -c executor.name="$EXECUTOR" run "${run[@]}" "$@" status=$? [[ " $* " == *" --dry-run "* ]] && exit $status -harness2-judge "${run[@]}" +harness2 judge "$name" judged=$? -harness2-report "${run[@]}" +harness2 report "$name" exit $((status ? status : judged)) diff --git a/scripts/run_workbuddy.sh b/plugins/harness2-benchmarks/scripts/run_workbuddy.sh similarity index 60% rename from scripts/run_workbuddy.sh rename to plugins/harness2-benchmarks/scripts/run_workbuddy.sh index 8df4d798..8c3668d7 100644 --- a/scripts/run_workbuddy.sh +++ b/plugins/harness2-benchmarks/scripts/run_workbuddy.sh @@ -14,24 +14,29 @@ # limitations under the License. # Run WorkBuddy-Bench end to end: check, adapt, judge, report. -# bash scripts/run_workbuddy.sh # every task -# bash scripts/run_workbuddy.sh --only-domains doc-ops --max-tasks 1 +# bash plugins/harness2-benchmarks/scripts/run_workbuddy.sh # every task +# bash plugins/harness2-benchmarks/scripts/run_workbuddy.sh --only-domains doc-ops --max-tasks 1 # Settings: SUBSTRATE=oc|cx SUBSET=office|web|code K=1 MODE=parallel|sequential # Extra arguments go to the adaptation command; --dry-run stops after it. set -uo pipefail -cd "$(dirname "$0")/.." -PATH="$PWD/.venv/bin:$PATH" +PATH="$(cd "$(dirname "$0")/../../.." && pwd)/.venv/bin:$PATH" SUBSTRATE=${SUBSTRATE:-oc} SUBSET=${SUBSET:-office} +case "$SUBSTRATE" in + oc) EXECUTOR=opencode ;; + cx) EXECUTOR=codex ;; + *) exit 2 ;; +esac K=${K:-1} MODE=${MODE:-parallel} -run=(--bench wb --subset "$SUBSET" --substrate "$SUBSTRATE" --k "$K" --mode "$MODE") +name="wb-${SUBSET}-${EXECUTOR}-k${K}-${MODE}" +run=(--suite "wb-$SUBSET" --name "$name" --k "$K" --mode "$MODE") -harness2-check --bench wb --subset "$SUBSET" --substrate "$SUBSTRATE" || exit -harness2 "${run[@]}" "$@" +harness2 -c executor.name="$EXECUTOR" -c suite.name="wb-$SUBSET" doctor || exit +harness2 -c executor.name="$EXECUTOR" run "${run[@]}" "$@" status=$? [[ " $* " == *" --dry-run "* ]] && exit $status -harness2-judge "${run[@]}" +harness2 judge "$name" judged=$? -harness2-report "${run[@]}" +harness2 report "$name" exit $((status ? status : judged)) diff --git a/scripts/setup_benchmark.py b/plugins/harness2-benchmarks/scripts/setup_benchmark.py similarity index 86% rename from scripts/setup_benchmark.py rename to plugins/harness2-benchmarks/scripts/setup_benchmark.py index 871d1cb9..81a03ffc 100644 --- a/scripts/setup_benchmark.py +++ b/plugins/harness2-benchmarks/scripts/setup_benchmark.py @@ -13,9 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Compatibility entry point for harness2.cli.setup.""" +"""Compatibility entry point for harness2_benchmarks.setup.""" -from harness2.cli.setup import main +from harness2_benchmarks.setup import main if __name__ == "__main__": raise SystemExit(main()) diff --git a/scripts/split_tasks.py b/plugins/harness2-benchmarks/scripts/split_tasks.py similarity index 98% rename from scripts/split_tasks.py rename to plugins/harness2-benchmarks/scripts/split_tasks.py index 8a778e7c..59c11203 100644 --- a/scripts/split_tasks.py +++ b/plugins/harness2-benchmarks/scripts/split_tasks.py @@ -23,7 +23,7 @@ from collections import defaultdict from pathlib import Path -from harness2.benchmarks.tasks import discover +from harness2_benchmarks.tasks import discover def main(argv=None) -> int: diff --git a/plugins/harness2-benchmarks/tests/test_plugin.py b/plugins/harness2-benchmarks/tests/test_plugin.py new file mode 100644 index 00000000..c24b68ee --- /dev/null +++ b/plugins/harness2-benchmarks/tests/test_plugin.py @@ -0,0 +1,410 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline coverage for the benchmark distribution boundary.""" + +from __future__ import annotations + +import importlib +import json +import os +import pkgutil +import shutil +import subprocess +import sys +import tempfile +import unittest +from importlib.metadata import entry_points +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import harness2_benchmarks +from harness2_benchmarks import config, install, jb_opencode_runner, preflight + +SUITES = ("jb", "lab", "wb-office", "wb-web", "wb-code") +PLUGIN = Path(__file__).resolve().parents[1] + + +class PluginTests(unittest.TestCase): + def context(self, root, executor="opencode"): + return SimpleNamespace( + settings={"executor.name": executor}, + project=root, + run_root=root / "runs" / "one", + split_file=root / "split.json", + ) + + def suites(self): + return [ + importlib.import_module("harness2_benchmarks.suites." + name.replace("-", "_")) + for name in SUITES + ] + + def test_import_every_module_and_entry_points(self): + for module in pkgutil.walk_packages( + harness2_benchmarks.__path__, harness2_benchmarks.__name__ + "." + ): + importlib.import_module(module.name) + entries = {e.name: e for e in entry_points(group="harness2.suites")} + for name, suite in zip(SUITES, self.suites()): + self.assertIs(entries[name].load(), suite) + self.assertEqual(install.missing_runners(), "") + + def test_suites_reject_unsupported_executor(self): + for name, suite in zip(SUITES, self.suites()): + for executor in ("claude", "command", "unknown"): + with self.subTest(suite=name, executor=executor): + with self.assertRaisesRegex( + config.ConfigError, + f"the {name} suite runs on the opencode or codex executor", + ): + suite.make_adapter(self.context(Path("/project"), executor)) + + def test_suite_paths_and_substrates(self): + with tempfile.TemporaryDirectory() as tmp: + for name, suite in zip(SUITES, self.suites()): + for executor, substrate in (("opencode", "oc"), ("codex", "cx")): + with self.subTest(suite=name, executor=executor): + ctx = self.context(Path(tmp), executor) + adapter = suite.make_adapter(ctx) + self.assertEqual(adapter.spec.name, name) + self.assertEqual(adapter.spec.runs_root, ctx.run_root) + self.assertEqual(adapter.substrate.name, substrate) + self.assertEqual(adapter.split_file, ctx.split_file) + if name == "jb": + self.assertEqual(adapter.store, ctx.run_root / "dataset") + elif name == "lab": + self.assertEqual( + adapter._results_root, ctx.run_root / "results" / substrate + ) + else: + self.assertEqual( + adapter._label_dir("example"), ctx.run_root / "rollouts/example" + ) + + def test_suite_check_and_setup(self): + for name, suite in zip(SUITES, self.suites()): + ctx = self.context(Path("/project")) + with self.subTest(suite=name), patch.object(suite, "make_adapter") as adapter: + with patch.object(preflight, "check", return_value=["missing"]) as check: + self.assertEqual(suite.check(ctx), ["missing"]) + adapter.assert_called_once_with(ctx) + check.assert_called_once_with(adapter.return_value, judge=True) + bench, _, subset = name.partition("-") + args = [bench] + (["--subset", subset] if subset else []) + ["--dry-run"] + with patch.object(suite, "main", return_value=7) as setup: + self.assertEqual(suite.setup(["--dry-run"]), 7) + setup.assert_called_once_with(args) + + def test_channels_keep_legacy_validation(self): + with patch.object(config, "PROPOSER_MODEL_BY_CHANNEL", {"opencode": ""}): + self.assertEqual(config.resolve_channel_model(None, None).model, "") + self.assertEqual( + config.resolve_channel_model("opencode", "provider/model").model, "provider/model" + ) + self.assertEqual(config.resolve_channel_model("claude_cli", "claude-x").model, "claude-x") + for kind, model in ( + (None, "x"), + ("opencode", None), + ("unknown", "x"), + ("opencode", ""), + ("opencode", "bare"), + ("claude_cli", "provider/model"), + ): + with self.subTest(kind=kind, model=model), self.assertRaises(ValueError): + config.resolve_channel_model(kind, model) + + def test_channels_supply_pinned_opencode_and_vertex_provider(self): + with ( + patch.object(config, "VERTEX_PROJECT", "test-project"), + patch.object(config, "VERTEX_REGION", "test-region"), + ): + ch = config.resolve_channel_model("opencode", "provider/model", steps=3, timeout=4) + self.assertEqual( + ch.bin, + f"{config.BUN} run --cwd {config.OPENCODE_DIR}/packages/opencode " + "--conditions=browser src/index.ts", + ) + self.assertEqual(ch.provider.project, "test-project") + self.assertEqual(ch.provider.region, "test-region") + self.assertEqual((ch.steps, ch.timeout), (3, 4)) + self.assertEqual(config.resolve_channel_model("claude_cli", "claude-x").bin, "") + with patch.object(config, "VERTEX_PROJECT", ""): + self.assertIsNone(config.resolve_channel_model("opencode", "provider/model").provider) + + def test_dependency_paths_use_cache_or_env(self): + with tempfile.TemporaryDirectory() as tmp: + env = {**os.environ, "XDG_CACHE_HOME": tmp} + env.pop("HARNESS2_BENCH_DEPS", None) + code = "from harness2_benchmarks.config import DEPENDENCIES; print(DEPENDENCIES)" + for override, expected in ( + (None, Path(tmp) / "harness2/benchmarks"), + (str(Path(tmp) / "custom"), Path(tmp) / "custom"), + ): + if override: + env["HARNESS2_BENCH_DEPS"] = override + result = subprocess.run( + [sys.executable, "-B", "-c", code], + env=env, + cwd=tmp, + capture_output=True, + text=True, + check=True, + ) + self.assertEqual(result.stdout.strip(), str(expected.resolve())) + self.assertEqual( + config.THIRD_PARTY, Path(harness2_benchmarks.__file__).parent / "third_party" + ) + self.assertFalse(hasattr(config, "RUNS")) + + def test_preflight_checks_pinned_assets_without_core_opencode_preflight(self): + with tempfile.TemporaryDirectory() as tmp: + deps = Path(tmp) + oc = deps / "opencode" + bun = deps / "bun" + (oc / "node_modules").mkdir(parents=True) + index = oc / "packages/opencode/src/index.ts" + index.parent.mkdir(parents=True) + index.touch() + bun.touch() + adapter = SimpleNamespace( + spec=SimpleNamespace(name="jb"), tasks=Mock(), preflight=Mock() + ) + with ( + patch.multiple( + config, + DEPENDENCIES=deps, + OPENCODE_DIR=oc, + BUN=bun, + SOLVER_MODEL_OC="provider/model", + IMPROVER_MODEL="provider/model", + validate_models=Mock(), + ), + patch( + "harness2_benchmarks.preflight.subprocess.run", + return_value=SimpleNamespace(returncode=0), + ), + patch( + "harness2.substrate.opencode.OpencodeSubstrate.preflight", + side_effect=AssertionError, + ), + ): + self.assertEqual(preflight.check(adapter), []) + index.unlink() + bun.unlink() + errors = preflight.check(adapter) + self.assertTrue(any("OpenCode checkout missing" in e for e in errors)) + self.assertTrue(any("bun not found" in e for e in errors)) + + def test_codex_preflight_uses_plugin_paths_and_validates_endpoint(self): + with tempfile.TemporaryDirectory() as tmp: + binary = Path(tmp) / "codex" + binary.touch() + with ( + patch.multiple( + config, + CODEX_DIST=tmp, + CODEX_BASE_URL="http://localhost:4000/v1", + CODEX_CONTAINER_URL="http://host.docker.internal:4000/v1", + ), + patch.dict( + os.environ, + HARNESS2_CODEX_BASE_URL="http://localhost:4000/v1", + HARNESS2_CODEX_API_KEY="test-key", + ), + patch("harness2_benchmarks.preflight.socket.create_connection") as connect, + ): + preflight.codex(require_bridge=True) + connect.assert_called_once_with(("localhost", 4000), timeout=5) + with patch.object(config, "CODEX_BASE_URL", "file:///tmp/key"): + with self.assertRaises(config.ConfigError): + preflight.codex() + with patch.object(config, "CODEX_CONTAINER_URL", ""): + with self.assertRaisesRegex(RuntimeError, "HARNESS2_CODEX_CONTAINER_URL"): + preflight.codex(require_bridge=True) + binary.unlink() + with self.assertRaisesRegex(RuntimeError, "Codex is not installed"): + preflight.codex() + + def test_jobbench_runner_defaults_use_plugin_dependencies(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + render = root / "render" + (render / ".opencode").mkdir(parents=True) + oc = root / "opencode" + (oc / "packages/opencode").mkdir(parents=True) + with ( + patch.dict( + os.environ, + { + "MODEL_ID": "provider/model", + "LABEL": "test", + "TASK_IDS": "task", + "HARNESS_RENDER_DIR": str(render), + "OPENCODE_DIR": "", + "TASKS_BASE_DIR": "", + }, + ), + patch.object(config, "OPENCODE_DIR", oc), + patch.object(jb_opencode_runner, "JOBBENCH", root / "jb"), + ): + settings = jb_opencode_runner._settings() + self.assertEqual(settings.opencode_dir, oc) + self.assertEqual(settings.tasks_base, root / "jb/dataset/main") + + def test_lab_overlay_paths_resolve_in_downloaded_checkout(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + with patch.object(config, "DEPENDENCIES", root): + install.overlays("harvey-labs") + repo = root / "harvey-labs" + parsers = repo / "sandbox/parsers" + parsers.mkdir(parents=True) + (parsers / "parse_doc.py").touch() + code = ( + "import json, runpy, sys; from pathlib import Path; repo = Path(sys.argv[1]); " + "r = runpy.run_path(str(repo / 'external/adapt_runs_cx/run_codex_lab.py')); " + "p = runpy.run_path(str(repo / 'opencode/scripts/read_doc.py')); " + "print(json.dumps([str(r['REPO']), str(r['C'].BENCH_ROOT), " + "str(r['READ_DOC']), str(p['_find_parsers_dir']())]))" + ) + env = dict(os.environ) + env.pop("LAB_BENCH_ROOT", None) + result = subprocess.run( + [sys.executable, "-B", "-c", code, str(repo)], + capture_output=True, + text=True, + check=True, + env=env, + ) + self.assertEqual( + json.loads(result.stdout), + [str(repo), str(repo), str(repo / "opencode/scripts/read_doc.py"), str(parsers)], + ) + + def test_codex_shell_runner_defaults_use_benchmark_cache(self): + runner = config.THIRD_PARTY / "jb_runner/run_benchmark_codex_cli_adapt.sh" + with tempfile.TemporaryDirectory() as tmp: + env = {**os.environ, "XDG_CACHE_HOME": tmp, "BENCHMARK_MODELS": "test"} + env.pop("TASKS_BASE_DIR", None) + env.pop("HARNESS2_BENCH_DEPS", None) + env.pop("SPLIT", None) + for override, expected in ( + (None, Path(tmp) / "harness2/benchmarks/job-bench-eval/dataset/main"), + (str(Path(tmp) / "custom"), Path(tmp) / "custom/job-bench-eval/dataset/main"), + ): + if override: + env["HARNESS2_BENCH_DEPS"] = override + result = subprocess.run( + [ + "bash", + "-c", + 'source "$1"; printf "%s" "$TASKS_BASE_DIR"', + "bash", + str(runner), + ], + env=env, + capture_output=True, + text=True, + check=True, + ) + self.assertEqual(result.stdout, str(expected)) + + def test_scripts_use_new_cli_and_keep_project_directory(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + scripts = root / "plugins/harness2-benchmarks/scripts" + scripts.mkdir(parents=True) + binary = root / ".venv/bin/harness2" + binary.parent.mkdir(parents=True) + binary.write_text( + f"#!{sys.executable}\nimport json, os, sys\n" + "with open(os.environ['CALLS'], 'a') as f:\n" + " f.write(json.dumps([os.getcwd(), sys.argv[1:]]) + '\\n')\n" + ) + binary.chmod(0o755) + for filename, suite in ( + ("run_jobbench.sh", "jb"), + ("run_lab.sh", "lab"), + ("run_workbuddy.sh", "wb-web"), + ): + script = scripts / filename + shutil.copy2(PLUGIN / "scripts" / filename, script) + for substrate, executor in (("oc", "opencode"), ("cx", "codex")): + for dry in (False, True): + calls = root / "calls.jsonl" + calls.unlink(missing_ok=True) + env = { + **os.environ, + "CALLS": str(calls), + "SUBSTRATE": substrate, + "SUBSET": "web", + "K": "3", + "MODE": "sequential", + } + result = subprocess.run( + ["bash", str(script), *(["--dry-run"] if dry else [])], + cwd=root, + env=env, + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + rows = [json.loads(line) for line in calls.read_text().splitlines()] + self.assertTrue(all(row[0] == str(root.resolve()) for row in rows)) + self.assertEqual( + rows[0][1], + [ + "-c", + f"executor.name={executor}", + "-c", + f"suite.name={suite}", + "doctor", + ], + ) + self.assertEqual( + rows[1][1], + [ + "-c", + f"executor.name={executor}", + "run", + "--suite", + suite, + "--name", + f"{suite}-{executor}-k3-sequential", + "--k", + "3", + "--mode", + "sequential", + ] + + (["--dry-run"] if dry else []), + ) + self.assertEqual(len(rows), 2 if dry else 4) + if not dry: + self.assertEqual( + [r[1] for r in rows[2:]], + [ + ["judge", f"{suite}-{executor}-k3-sequential"], + ["report", f"{suite}-{executor}-k3-sequential"], + ], + ) + result = subprocess.run( + ["bash", str(script)], env={**os.environ, "SUBSTRATE": "bad"} + ) + self.assertEqual(result.returncode, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/pyproject.toml b/pyproject.toml index d5892bea..3e143b60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,9 +3,9 @@ requires = ["setuptools>=77", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "harness2" +name = "harness2-cli" version = "0.1.0" -description = "Harness²: Recursive Agent Harnessing for an Open World" +description = "General-purpose CLI for Harness²: tune any coding agent's harness on your own tasks" readme = "README.md" license = "Apache-2.0" requires-python = ">=3.12,<3.14" @@ -16,21 +16,29 @@ classifiers = [ ] authors = [ { name = "Google LLC" }, + { name = "The Harness2 Authors" }, ] dependencies = [] +[project.urls] +Homepage = "https://github.com/3ae3ae/harness2-cli" +Upstream = "https://github.com/google-research/harness2" +Paper = "https://arxiv.org/abs/2610.14224" + [project.optional-dependencies] -render = ["openpyxl>=3.1,<4", "python-pptx>=1,<2"] bridge = ["litellm[proxy]==1.95.0", "fastapi==0.139.2", "google-auth==2.56.3"] [project.scripts] -harness2 = "harness2.cli.run_sweep:main" -harness2-judge = "harness2.cli.judge:main" -harness2-report = "harness2.cli.report:main" -harness2-setup = "harness2.cli.setup:main" -harness2-check = "harness2.cli.check:main" -harness2-demo = "harness2.cli.demo:main" -harness2-bridge = "harness2.cli.bridge:main" +harness2 = "harness2.cli.main:main" +harness2-judge = "harness2.cli.main:main" +harness2-report = "harness2.cli.main:main" +harness2-check = "harness2.cli.main:main" +harness2-demo = "harness2.cli.main:main" +harness2-bridge = "harness2.cli.main:main" +harness2-setup = "harness2.cli.main:main" + +[tool.setuptools] +include-package-data = false [tool.setuptools.packages.find] include = ["harness2*"] diff --git a/tests/fixtures/claude-stream.jsonl b/tests/fixtures/claude-stream.jsonl new file mode 100644 index 00000000..d37b3e89 --- /dev/null +++ b/tests/fixtures/claude-stream.jsonl @@ -0,0 +1,7 @@ +{"type":"system","subtype":"init","cwd":"/workspace","session_id":"fixture-session","tools":["Bash","Read"],"model":"claude-fixture","permissionMode":"bypassPermissions","apiKeySource":"ANTHROPIC_API_KEY","claude_code_version":"2.1.0","uuid":"init-1"} +{"type":"assistant","message":{"id":"msg_1","type":"message","role":"assistant","model":"claude-fixture","content":[{"type":"thinking","thinking":"Inspect the input first.","signature":"fixture"},{"type":"text","text":"I will inspect the input."},{"type":"tool_use","id":"tool_1","name":"Bash","input":{"command":"cat input.txt"}}],"stop_reason":"tool_use","usage":{"input_tokens":20,"output_tokens":15}},"parent_tool_use_id":null,"session_id":"fixture-session","uuid":"assistant-1"} +{"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"tool_1","content":[{"type":"text","text":"input contents"}]}]},"parent_tool_use_id":null,"session_id":"fixture-session","uuid":"user-1"} +{"type":"assistant","message":{"id":"msg_2","type":"message","role":"assistant","content":[{"type":"tool_use","id":"tool_2","name":"Read","input":{"file_path":"missing.txt"}}]},"parent_tool_use_id":null,"session_id":"fixture-session","uuid":"assistant-2"} +{"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"tool_2","content":"File does not exist.","is_error":true}]},"parent_tool_use_id":null,"session_id":"fixture-session","uuid":"user-2"} +{"type":"assistant","message":{"id":"msg_3","type":"message","role":"assistant","content":[{"type":"text","text":"Finished the task."}],"stop_reason":"end_turn"},"parent_tool_use_id":null,"session_id":"fixture-session","uuid":"assistant-3"} +{"type":"result","subtype":"success","is_error":false,"duration_ms":1000,"duration_api_ms":900,"num_turns":2,"result":"Task complete.","total_cost_usd":0.0123,"usage":{"input_tokens":30,"output_tokens":25},"modelUsage":{},"permission_denials":[],"session_id":"fixture-session","uuid":"result-1"} diff --git a/tests/test_agent_ops.py b/tests/test_agent_ops.py new file mode 100644 index 00000000..d34c04e2 --- /dev/null +++ b/tests/test_agent_ops.py @@ -0,0 +1,353 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline checks for machine output and run process controls.""" + +from __future__ import annotations + +import contextlib +import io +import json +import os +import signal +import subprocess +import sys +import tempfile +import unittest +from dataclasses import replace +from datetime import datetime +from pathlib import Path +from unittest.mock import Mock, patch + +from harness2.cli import _adopt, diff, main, run +from harness2.cli import _common as C +from harness2.core import events +from harness2.core.tree import diff_summary +from harness2.demo import DemoAdapter, DemoProposer + + +class AgentOpsTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.project = Path(temporary.name).resolve() + (self.project / ".harness2").mkdir() + env = patch.dict(os.environ, {"XDG_CONFIG_HOME": str(self.project / "config")}, clear=True) + env.start() + self.addCleanup(env.stop) + self.addCleanup(events.start, None) + self.root = self.project / ".harness2/runs/claude-k1" + + def make_adapter(ctx): + adapter = DemoAdapter(ctx.run_root / "fixture") + adapter.spec = replace(adapter.spec, runs_root=ctx.run_root) + return adapter + + self.suite = Mock(make_adapter=Mock(side_effect=make_adapter), check=Mock(return_value=[])) + registry = patch("harness2.registry.load", return_value=self.suite) + registry.start() + self.addCleanup(registry.stop) + + def invoke(self, *args): + stdout, stderr = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + code = main.main(["-C", str(self.project), *args]) + return code, stdout.getvalue(), stderr.getvalue() + + def pid_record(self, root=None): + root = root or self.root + root.mkdir(parents=True, exist_ok=True) + record = {"pid": os.getpid(), "started": "2026-10-19T00:00:00+00:00", "argv": ["run"]} + (root / "run.pid").write_text(json.dumps(record)) + return record + + def test_json_errors_and_text_behavior(self): + for exception in ( + OSError("io"), + RuntimeError("runtime"), + ValueError("value"), + KeyError("key"), + ): + with ( + self.subTest(exception=exception), + patch.object(diff, "run", side_effect=exception), + ): + code, out, err = self.invoke("--json", "--quiet", "diff") + self.assertEqual(code, 1) + self.assertEqual( + out.splitlines(), [json.dumps({"error": str(exception), "exit_code": 1})] + ) + self.assertEqual(err, "") + self.assertEqual(self.invoke("diff"), (1, "", f"error: {exception}\n")) + self.assertEqual(self.invoke("--quiet", "diff"), (1, "", "")) + with patch.object(diff, "run", side_effect=KeyboardInterrupt): + self.assertEqual(self.invoke("diff"), (130, "", "")) + code, out, err = self.invoke("--json", "diff") + self.assertEqual( + (code, json.loads(out), err), (130, {"error": "interrupted", "exit_code": 130}, "") + ) + + def test_argparse_errors_still_exit_two(self): + with ( + contextlib.redirect_stdout(io.StringIO()) as out, + contextlib.redirect_stderr(io.StringIO()) as err, + ): + with self.assertRaises(SystemExit) as raised: + main.main(["--json", "run", "--unknown"]) + self.assertEqual(raised.exception.code, 2) + self.assertEqual(out.getvalue(), "") + self.assertIn("usage: harness2", err.getvalue()) + + def test_detach_preserves_in_process_argv_and_appends_logs(self): + child = Mock(pid=12345) + child.wait.side_effect = subprocess.TimeoutExpired("harness2", 2) + + def popen(argv, **kwargs): + self.assertEqual( + argv, + [ + sys.executable, + "-m", + "harness2", + "-C", + str(self.project), + "--json", + "run", + "--seed", + "0", + "--name", + "claude-k1-s0", + ], + ) + self.assertEqual(kwargs["stdin"], subprocess.DEVNULL) + self.assertEqual(kwargs["stderr"], subprocess.STDOUT) + self.assertTrue(kwargs["start_new_session"]) + self.assertNotIn("cwd", kwargs) + self.assertNotIn("env", kwargs) + kwargs["stdout"].write("child log\n") + return child + + root = self.root.with_name("claude-k1-s0") + root.mkdir(parents=True) + (root / "run.log").write_text("previous log\n") + with patch.object(run.subprocess, "Popen", side_effect=popen): + code, out, err = self.invoke("--json", "run", "--seed", "0", "--detach") + self.assertEqual( + (code, json.loads(out), err), + (0, {"name": root.name, "pid": 12345, "log": str(root / "run.log")}, ""), + ) + self.assertEqual((root / "run.log").read_text(), "previous log\nchild log\n") + child.wait.assert_called_once_with(timeout=2) + self.suite.make_adapter.assert_not_called() + self.assertFalse((root / "run.pid").exists()) + + def test_detach_explicit_name_and_exact_text(self): + child = Mock(pid=12345) + child.wait.return_value = 0 + with patch.object(run.subprocess, "Popen", return_value=child) as popen: + code, out, err = self.invoke("run", "--detach", "--name=chosen") + log = self.root.with_name("chosen") / "run.log" + self.assertEqual( + (code, out, err), + ( + 0, + f"Started run chosen in the background (pid 12345).\nFollow it: harness2 status chosen --watch\nStop it: harness2 stop chosen\nLog: {log}\n", + "", + ), + ) + self.assertEqual(popen.call_args.args[0][-2:], ["run", "--name=chosen"]) + + def test_detach_early_failure_reports_last_twenty_lines(self): + child = Mock(pid=12345) + child.wait.return_value = 7 + + def popen(argv, **kwargs): + kwargs["stdout"].write("".join(f"line {i}\n" for i in range(25))) + return child + + with patch.object(run.subprocess, "Popen", side_effect=popen): + code, out, err = self.invoke("--json", "run", "--detach") + self.assertEqual((code, out), (7, "")) + self.assertEqual(err.splitlines(), [f"line {i}" for i in range(5, 25)]) + + def test_detach_dry_run_is_rejected(self): + with patch.object(run.subprocess, "Popen") as popen: + code, out, _ = self.invoke("--json", "run", "--detach", "--dry-run") + self.assertEqual( + (code, json.loads(out)), + (1, {"error": "--detach starts a real run; drop --dry-run", "exit_code": 1}), + ) + popen.assert_not_called() + self.suite.make_adapter.assert_not_called() + + def test_already_running_refuses_both_start_modes(self): + record = self.pid_record() + for extra in ([], ["--detach"]): + with self.subTest(extra=extra), patch.object(run.subprocess, "Popen") as popen: + code, out, _ = self.invoke("--json", "run", *extra) + self.assertEqual( + (code, json.loads(out)), + ( + 1, + { + "error": f"run claude-k1 is already running (pid {record['pid']})", + "exit_code": 1, + }, + ), + ) + self.assertEqual(C.run_process(self.root), record) + popen.assert_not_called() + self.suite.make_adapter.assert_not_called() + + def test_run_process_live_stale_invalid_and_permission_denied(self): + self.assertIsNone(C.run_process(self.root)) + record = self.pid_record() + self.assertEqual(C.run_process(self.root), record) + with patch.object(C.os, "kill", side_effect=PermissionError): + self.assertEqual(C.run_process(self.root), record) + with patch.object(C.os, "kill", side_effect=ProcessLookupError): + self.assertIsNone(C.run_process(self.root)) + self.assertFalse((self.root / "run.pid").exists()) + for invalid in ( + "broken", + "[]", + "{}", + json.dumps(record | {"pid": 0}), + json.dumps(record | {"pid": True}), + json.dumps(record | {"argv": [1]}), + ): + with self.subTest(invalid=invalid), patch.object(C.os, "kill") as kill: + (self.root / "run.pid").write_text(invalid) + self.assertIsNone(C.run_process(self.root)) + self.assertFalse((self.root / "run.pid").exists()) + kill.assert_not_called() + + def test_pid_lifecycle_success_failure_and_interrupt(self): + argv = ["-C", str(self.project), "--json", "run"] + for exception, expected in ( + (None, 0), + (RuntimeError("crash"), 1), + (KeyboardInterrupt(), 130), + (events.Stopped("interrupted"), 130), + (events.Stopped("max_calls"), 4), + ): + + def sweep(*args, **kwargs): + record = C.run_process(self.root) + self.assertEqual(record["pid"], os.getpid()) + self.assertEqual(record["argv"], argv) + datetime.fromisoformat(record["started"]) + self.assertTrue((self.root / "run.json").is_file()) + if exception is not None: + raise exception + return {"domains": []} + + with ( + self.subTest(exception=exception), + patch.object(run.schedule, "run_sweep", side_effect=sweep), + patch.object(run, "Proposer", DemoProposer), + ): + code, out, _ = self.invoke("--json", "run") + self.assertEqual(code, expected) + self.assertFalse((self.root / "run.pid").exists()) + if expected in (1, 130): + self.assertEqual( + json.loads(out), + {"error": "crash" if expected == 1 else "interrupted", "exit_code": expected}, + ) + code, _, _ = self.invoke("run", "--dry-run") + self.assertEqual(code, 0) + self.assertFalse((self.root / "run.pid").exists()) + + def test_json_sweep_error(self): + with ( + patch.object( + run.schedule, "run_sweep", return_value={"domains": [{"error": "failed"}]} + ), + patch.object(run, "Proposer", DemoProposer), + ): + code, out, _ = self.invoke("--json", "run") + self.assertEqual((code, json.loads(out)), (1, {"error": "failed", "exit_code": 1})) + + def test_stop_signals_group_and_defaults_to_latest_run(self): + self.pid_record() + (self.root / "run.json").write_text("{}") + with ( + patch("harness2.cli.stop.os.getpgid", return_value=9876), + patch("harness2.cli.stop.os.killpg") as killpg, + patch("harness2.cli.stop.os.kill") as kill, + ): + self.assertEqual(self.invoke("stop")[0], 0) + killpg.assert_not_called() # not a group leader: the launcher's group is left alone + kill.assert_any_call(os.getpid(), signal.SIGINT) + for json_flag in ([], ["--json"]): + with ( + patch("harness2.cli.stop.os.getpgid", return_value=os.getpid()) as getpgid, + patch("harness2.cli.stop.os.killpg") as killpg, + ): + code, out, err = self.invoke(*json_flag, "stop") + self.assertEqual((code, err), (0, "")) + getpgid.assert_called_once_with(os.getpid()) + killpg.assert_called_once_with(os.getpid(), signal.SIGINT) + if json_flag: + self.assertEqual( + json.loads(out), {"name": self.root.name, "pid": os.getpid(), "signalled": True} + ) + else: + self.assertEqual( + out, + f"Asked run claude-k1 (pid {os.getpid()}) to stop. It ends the calls in flight, then exits; run the same command again to resume.\n", + ) + + def test_stop_not_running_and_process_exit_race(self): + self.root.mkdir(parents=True) + (self.root / "run.json").write_text("{}") + self.assertEqual( + self.invoke("stop", self.root.name), (0, "Run claude-k1 is not running.\n", "") + ) + code, out, err = self.invoke("--json", "stop") + self.assertEqual( + (code, json.loads(out), err), + (0, {"name": self.root.name, "pid": None, "signalled": False}, ""), + ) + self.pid_record() + with patch("harness2.cli.stop.os.getpgid", side_effect=ProcessLookupError): + self.assertEqual(self.invoke("stop"), (0, "Run claude-k1 is not running.\n", "")) + + def test_diff_json_and_unchanged_text(self): + before = {"a": "old\n", "d": "removed\n"} + after = {"a": "new\n", "b": "added\n"} + adapter = DemoAdapter(self.project / "diff-fixture") + changes = [["a", "M"], ["b", "A"], ["d", "D"]] + diffs = {rel: _adopt.unified(rel, before, after) for rel, _ in changes} + with ( + patch.object(_adopt, "open_run", return_value=(adapter, self.root)), + patch.object(_adopt, "harness_for", return_value=("task-1", after)), + patch.object(_adopt, "base_tree", return_value=before), + ): + for extra in ([], ["--stat"]): + code, out, err = self.invoke("--json", "diff", *extra) + self.assertEqual( + (code, json.loads(out), err), + (0, {"label": "task-1", "changes": changes, "diffs": diffs}, ""), + ) + self.assertEqual(self.invoke("diff"), (0, "".join(diffs.values()), "")) + self.assertEqual( + self.invoke("diff", "--stat"), + (0, str(diff_summary(adapter.substrate.surface(), before, after)) + "\n", ""), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_bridge.py b/tests/test_bridge.py new file mode 100644 index 00000000..d48b3d57 --- /dev/null +++ b/tests/test_bridge.py @@ -0,0 +1,136 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline configuration and endpoint persistence checks for the provider bridge.""" + +from __future__ import annotations + +import contextlib +import io +import json +import os +import tempfile +import tomllib +import unittest +from pathlib import Path +from unittest.mock import patch + +from harness2 import providers, settings +from harness2.cli import bridge + + +class BridgeTests(unittest.TestCase): + def setUp(self): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + self.root = Path(temp.name) + self.project = self.root / "project" + (self.project / ".harness2").mkdir(parents=True) + self.binary = self.root / "bin/litellm" + self.binary.parent.mkdir() + self.binary.write_text("fixture") + + def test_endpoint_reuses_key_and_port_and_has_private_permissions(self): + path = self.root / "state/bridge.json" + with patch.object(bridge, "_free_port", return_value=12345) as free: + initial = bridge._endpoint(path, "127.0.0.1", 0) + self.assertEqual(initial["port"], 12345) + self.assertEqual(bridge._endpoint(path, "127.0.0.1", 0), initial) + self.assertEqual(free.call_count, 1) + updated = bridge._endpoint(path, "127.0.0.1", 23456) + self.assertEqual(updated, {"port": 23456, "key": initial["key"]}) + self.assertEqual(path.stat().st_mode & 0o777, 0o600) + self.assertEqual(json.loads(path.read_text()), updated) + self.assertGreater(bridge._free_port("127.0.0.1"), 0) + + def test_provider_config_and_state_without_launching_server(self): + for kind, extra in ( + ("anthropic", 'api_key_env="TEST_KEY"\nbase_url="https://example.test"'), + ("openai", 'api_key_env="TEST_KEY"\nbase_url="https://example.test/v1"'), + ("vertex", 'project="test-project"\nregion="test-region"'), + ("bedrock", 'region="test-region"'), + ): + with self.subTest(kind=kind): + (self.project / ".harness2/config.toml").write_text( + f'[providers.custom]\ntype="{kind}"\n{extra}\n' + ) + stdout = io.StringIO() + cache = self.root / kind + with ( + patch.dict( + os.environ, + { + "XDG_CONFIG_HOME": str(self.root / "config"), + "XDG_CACHE_HOME": str(cache), + "TEST_KEY": "secret", + }, + ), + patch.object(bridge.settings, "find_project", return_value=self.project), + patch.object(bridge.sys, "executable", str(self.binary.with_name("python"))), + patch.object(bridge.os, "execve") as execute, + contextlib.redirect_stdout(stdout), + ): + bridge.main(["--model", "custom/test-model", "--port", "12345"]) + expected = providers.litellm_params( + settings.load(self.project).provider("custom"), "test-model" + ) + root = cache / "harness2/bridge" + config = json.loads((root / "bridge.yaml").read_text()) + self.assertEqual( + config["model_list"], [{"model_name": "test-model", "litellm_params": expected}] + ) + self.assertEqual( + config["general_settings"]["master_key"], "os.environ/HARNESS2_BRIDGE_KEY" + ) + binary, argv, env = execute.call_args.args + self.assertEqual(binary, str(self.binary)) + self.assertEqual( + argv, + [ + str(self.binary), + "--config", + str(root / "bridge.yaml"), + "--host", + "127.0.0.1", + "--port", + "12345", + ], + ) + endpoint = json.loads((root / "bridge.json").read_text()) + self.assertEqual(env["HARNESS2_BRIDGE_KEY"], endpoint["key"]) + self.assertIn("[providers.bridge]", stdout.getvalue()) + self.assertIn('api_key_env = "HARNESS2_BRIDGE_KEY"', stdout.getvalue()) + text = stdout.getvalue() + snippet = text[text.index("[providers.bridge]") : text.index("\n\nand put its key")] + self.assertEqual(tomllib.loads(snippet)["providers"]["bridge"]["type"], "openai") + self.assertNotIn("secret", (root / "bridge.yaml").read_text()) + + def test_model_required_and_unknown_provider_fails(self): + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as raised: + bridge.main([]) + self.assertEqual(raised.exception.code, 2) + with ( + patch.object(bridge.sys, "executable", str(self.binary.with_name("python"))), + patch.object(bridge.settings, "find_project", return_value=self.project), + patch.dict(os.environ, {"XDG_CONFIG_HOME": str(self.root / "config")}), + contextlib.redirect_stderr(io.StringIO()), + ): + with self.assertRaises(SystemExit) as raised: + bridge.main(["--model", "unknown/model"]) + self.assertEqual(raised.exception.code, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 00000000..78c9b2fc --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,637 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline command parsing, snapshots, adoption, and demo-backed CLI checks.""" + +from __future__ import annotations + +import contextlib +import io +import json +import os +import shutil +import subprocess +import sys +import tempfile +import types +import unittest +from dataclasses import replace +from pathlib import Path +from unittest.mock import Mock, patch + +from harness2 import settings +from harness2.cli import _adopt, _final, completion, judge, main, run, runs +from harness2.cli import _common as C +from harness2.core import events +from harness2.core.judge import Scored +from harness2.core.proposer import Reply +from harness2.demo import DemoAdapter, DemoProposer +from harness2.substrate.claude import ClaudeSubstrate + + +class CliTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.project = self.root / "project" + (self.project / ".harness2").mkdir(parents=True) + self.env = patch.dict( + os.environ, {"XDG_CONFIG_HOME": str(self.root / "config")}, clear=True + ) + self.env.start() + self.addCleanup(self.env.stop) + self.addCleanup(events.start, None) + self.config_path = self.project / ".harness2/config.toml" + self.config_path.write_text('[improver]\nmodel="openai/fake"\nchannel="opencode"\n') + self.adapters = [] + + def make_adapter(ctx): + adapter = DemoAdapter(ctx.run_root / "fixture") + adapter.spec = replace(adapter.spec, runs_root=ctx.run_root) + adapter.has_judge = lambda task: task == "ledger-1" + adapter.score = lambda path, task: {"n_passed": 1, "n_criteria": 1} + self.adapters.append(adapter) + return adapter + + self.suite = types.SimpleNamespace(make_adapter=make_adapter, check=lambda ctx: []) + self.registry = patch("harness2.registry.load", return_value=self.suite) + self.registry.start() + self.addCleanup(self.registry.stop) + + def invoke(self, *args, project=True): + stdout, stderr = io.StringIO(), io.StringIO() + argv = (["-C", str(self.project)] if project else []) + list(args) + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + code = main.main(argv) + return code, stdout.getvalue(), stderr.getvalue() + + def context(self, *args): + parsed = main.build_parser().parse_args(["-C", str(self.project), *args]) + return C.Context(parsed, getattr(parsed._module, "FLAGS", {})) + + def execute(self, k=1, *extra): + with ( + patch.object(run, "Proposer", DemoProposer), + patch.object(_final, "Proposer", DemoProposer), + ): + return self.invoke("--json", "run", "--k", str(k), *extra) + + def test_all_command_arguments(self): + commands = [ + [ + "run", + "--suite", + "tasks", + "--executor", + "claude", + "--name", + "x", + "--k", + "2", + "--mode", + "parallel,sequential", + "--seed", + "0", + "--split", + "train", + "--split-file", + "split.json", + "--only-domains", + "d", + "--only-tasks", + "a", + "--max-tasks", + "1", + "--workers", + "2", + "--budget-usd", + "1", + "--max-calls", + "20", + "--selector", + "human", + "--steer", + "@guide", + "--dry-run", + ], + ["doctor", "--live"], + ["setup", "tasks", "--flag"], + ["judge", "x", "--bases"], + ["report", "x", "y"], + ["runs", "ls"], + ["runs", "show", "x"], + ["diff", "x", "--task", "a", "--mode", "sequential", "--stat"], + ["task", "new", "a", "--files", "--domain", "d", "--check", "true"], + ["task", "ls"], + ["task", "import", "a.csv", "--force"], + ["config", "show", "--origin"], + ["config", "get", "run.k"], + ["config", "set", "run.k", "2", "--global"], + ["config", "path"], + ["demo", "--output", "demo", "--k", "1", "--mode", "both"], + ["bridge", "--model", "openai/model", "--host", "localhost", "--port", "1234"], + ["completion", "zsh"], + ["init"], + ["status"], + ["chat"], + ["apply"], + ] + parser = main.build_parser() + for argv in commands: + with self.subTest(argv=argv): + args = parser.parse_args(argv) + self.assertEqual(args.command, argv[0]) + self.assertTrue(callable(args._module.run)) + self.assertEqual(parser.parse_args(["run", "--bench", "jb"]).suite, "jb") + self.assertEqual(parser.parse_args(["run", "--modes", "seq"]).modes, "seq") + + def test_context_overrides_origins_and_output(self): + ctx = self.context( + "-c", "run.k=4", "run", "--k", "2", "--selector", "human", "--workers", "3" + ) + self.assertEqual(ctx.settings.get("run.k"), 2) + self.assertEqual(ctx.settings.origin("run.k"), "flag --k") + self.assertEqual(ctx.settings.role("selector")["channel"], "human") + self.assertEqual(ctx.settings.get("run.workers"), 3) + self.assertEqual(C.default_name(ctx.settings), "claude-k2") + ctx = self.context("run", "--seed", "0") + self.assertEqual(C.default_name(ctx.settings), "claude-k1-s0") + code, stdout, _ = self.invoke("--json", "config", "show", "--origin") + self.assertEqual(code, 0) + self.assertEqual(json.loads(stdout)["run.k"]["origin"], "default") + with self.assertRaises(settings.ConfigError): + self.context("-c", "oops", "config", "show").settings + with self.assertRaises(settings.ConfigError): + C.parse_modes("") + self.assertEqual(C.parse_modes("seq,agg,parallel"), ("agg", "seq")) + self.assertEqual(C.mode_label("seq"), "sequential") + + def test_lazy_project_and_errors(self): + with patch.object(settings, "find_project", return_value=None): + code, _, error = self.invoke("runs", "ls", project=False) + self.assertEqual(code, 1) + self.assertIn("not inside a harness2 project", error) + self.assertEqual( + self.invoke("--json", "config", "get", "run.k", project=False)[:2], (0, "1\n") + ) + self.assertEqual(self.invoke("--quiet", "runs", "show", "missing")[2], "") + self.assertIn("command: config", self.invoke("--verbose", "config", "path")[2]) + for command in ("status", "apply"): + code, _, error = self.invoke(command, project=False) + self.assertEqual(code, 1) + self.assertIn("not inside a harness2 project", error) + with self.assertRaises(settings.ConfigError): + C.run_path(self.context("runs", "ls"), "../escape") + for command in ("--help", "--version"): + with contextlib.redirect_stdout(io.StringIO()), self.assertRaises(SystemExit) as raised: + main.main([command]) + self.assertEqual(raised.exception.code, 0) + + def test_run_identity_and_name_resolution(self): + ctx = self.context("run") + root = C.run_path(ctx, C.default_name(ctx.settings)) + plan = {"k": 1, "modes": "parallel", "seed": None} + first = C.save_run(root, ctx.settings, [["d", "t"]], plan) + self.assertNotIn("run.k", first["settings"]) + self.assertNotIn("judge.model", first["settings"]) + self.assertIn("executor.model", first["settings"]) + changed = self.context("run", "--workers", "9", "--k", "2") + second = C.save_run(root, changed.settings, [["d", "t"]], {**plan, "k": 2}) + self.assertEqual(first["created"], second["created"]) + with self.assertRaisesRegex(settings.ConfigError, "executor.name.*claude.*codex"): + C.save_run( + root, self.context("run", "--executor", "codex").settings, [["d", "t"]], plan + ) + with self.assertRaisesRegex(settings.ConfigError, "stream"): + C.save_run(root, ctx.settings, [["d", "different"]], plan) + recent = root.parent / "new" + C.save_run(recent, ctx.settings, [["d", "t"]], plan) + os.utime(root, ns=(1, 1)) + self.assertEqual(C.resolve_run(ctx, None), recent) + self.assertEqual(C.resolve_run(ctx, root.name), root) + (recent / "run.json").write_text("[]") + with self.assertRaises(settings.ConfigError): + C.load_run(recent) + self.assertEqual(run._csv(" a, b, "), ("a", "b")) + + def test_snapshot_ignores_later_identity_settings(self): + ctx = self.context("run") + root = C.run_path(ctx, "snapshot") + record = C.save_run( + root, ctx.settings, [["d", "t"]], {"k": 1, "modes": "parallel", "seed": None} + ) + self.config_path.write_text( + '[executor]\nname="codex"\n[selector]\nchannel="human"\n[improver]\nmodel="openai/changed"\n' + ) + rebuilt = C.run_settings(self.context("-c", "judge.model=openai/new", "judge"), record) + self.assertEqual(C.identity(rebuilt), record["settings"]) + self.assertEqual(rebuilt.role("selector")["channel"], "opencode") + self.assertEqual(rebuilt.role("judge")["model"], "openai/new") + + def test_dry_run_estimate_and_stream_filters(self): + code, stdout, _ = self.invoke( + "--json", + "run", + "--dry-run", + "--k", + "1", + "--mode", + "parallel,sequential", + "--only-tasks", + "ledger-2", + ) + self.assertEqual(code, 0) + data = json.loads(stdout) + self.assertEqual(data["stream"], [["accounting", "ledger-2"]]) + self.assertEqual((data["estimate"]["rollouts"], data["estimate"]["sessions"]), (7, 10)) + self.assertEqual(data["estimate"]["per_task_per_mode"]["reflector"], 1) + self.assertEqual(list((self.project / ".harness2/runs").rglob("run.json")), []) + self.assertEqual(list((self.project / ".harness2/runs").rglob("events.jsonl")), []) + code, stdout, _ = self.invoke("run", "--dry-run", "--k", "2", "--max-tasks", "1") + self.assertEqual(code, 0) + self.assertIn("would refine 1 task(s)", stdout) + self.assertIn("accounting", stdout) + self.assertIn("Model calls: 8 agent runs + 9", stdout) + self.assertEqual( + run._estimate(1, 2, ("agg",), {"spent_usd": 4, "calls": 2})["average_cost_per_call"], 2 + ) + self.suite.check = lambda ctx: ["fixture error"] + code, _, error = self.invoke("run", "--dry-run") + self.assertEqual(code, 1) + self.assertIn("fixture error", error) + self.suite.check = lambda ctx: [] + self.suite.make_adapter = lambda ctx: types.SimpleNamespace(tasks=lambda split: []) + self.assertEqual(self.invoke("run", "--dry-run")[0], 1) + + def test_demo_backed_run_judge_report_diff_for_each_scale(self): + for k in (1, 2): + with self.subTest(k=k): + code, stdout, _ = self.execute(k, "--mode", "parallel,sequential") + self.assertEqual(code, 0) + data = json.loads(stdout) + self.assertEqual( + data["totals"]["calls"], 2 * (k + 6 * k + 6 * k + 2 * (1 + int(k > 1))) + 2 + ) + name = f"claude-k{k}" + root = self.project / ".harness2/runs" / name + record = C.load_run(root) + self.assertEqual( + record["stream"], [["accounting", "ledger-1"], ["accounting", "ledger-2"]] + ) + self.assertEqual(record["schedule"]["k"], k) + self.assertEqual( + record["settings"], C.identity(self.context("run", "--k", str(k)).settings) + ) + stored_events = [ + json.loads(line) for line in (root / "events.jsonl").read_text().splitlines() + ] + self.assertEqual(stored_events[0]["kind"], "run_start") + self.assertEqual(stored_events[-1]["status"], "ok") + code, stdout, error = self.invoke("--json", "judge", name) + self.assertEqual(code, 0) + self.assertEqual(len(json.loads(stdout)), 2) + self.assertEqual(error.count("no judge; skipped"), 1) + judged = next((root / "adapt").glob("*/judged.json")) + self.assertEqual(json.loads(judged.read_text())["judge_model"], "openai/fake") + self.assertEqual( + self.invoke("-c", "judge.model=openai/different", "judge", name)[0], 1 + ) + code, stdout, _ = self.invoke("--json", "report", name) + self.assertEqual((code, len(json.loads(stdout))), (0, 2)) + code, stdout, _ = self.invoke("--json", "diff", name) + self.assertEqual(code, 0) + diff = json.loads(stdout) + self.assertEqual(diff["label"], "consolidated") + self.assertIn("-total = sum", diff["diffs"]["scripts/summarize.py"]) + self.assertIn("+total = sum", diff["diffs"]["scripts/summarize.py"]) + code, stdout, _ = self.invoke( + "--json", "diff", name, "--task", "ledger-1", "--mode", "sequential", "--stat" + ) + self.assertEqual(code, 0) + self.assertIn(["scripts/summarize.py", "M"], json.loads(stdout)["changes"]) + self.assertEqual(self.invoke("diff", name, "--task", "missing")[0], 1) + self.assertEqual(self.invoke("--json", "judge", name, "--bases")[0], 0) + code, stdout, _ = self.invoke("--json", "runs", "show", name) + self.assertEqual((code, json.loads(stdout)["done"]), (0, 2)) + with ( + patch.object(run, "Proposer", DemoProposer), + patch.object(_final, "Proposer", DemoProposer), + ): + code, stdout, _ = self.invoke( + "--json", "run", "--k", str(k), "--mode", "parallel,sequential" + ) + self.assertEqual((code, self.adapters[-1].execution_count), (0, 0)) + code, stdout, _ = self.invoke("--json", "runs", "ls") + self.assertEqual((code, len(json.loads(stdout))), (0, 2)) + self.assertIn("claude-k1", self.invoke("runs", "ls")[1]) + code, stdout, _ = self.invoke("report", "claude-k1", "claude-k2") + self.assertEqual(code, 0) + self.assertIn("claude-k1", stdout) + self.assertIn("claude-k2", stdout) + self.assertIn("score 100.00%", stdout) + + def test_budget_resume_and_failure_exit_events(self): + code, stdout, _ = self.execute(1, "--max-calls", "1") + self.assertEqual(code, 4) + self.assertEqual(json.loads(stdout)["stopped"], "max_calls") + root = self.project / ".harness2/runs/claude-k1" + self.assertEqual( + json.loads((root / "events.jsonl").read_text().splitlines()[-1])["status"], "stopped" + ) + code, _, _ = self.execute(1, "--max-calls", "100") + self.assertEqual(code, 0) + for exception, expected, status in ( + (KeyboardInterrupt(), 130, "interrupted"), + (events.Stopped("interrupted"), 130, "interrupted"), + (events.Stopped("usage_limit"), 4, "stopped"), + (RuntimeError("fixture crash"), 1, "error"), + ): + with ( + patch.object(run.schedule, "run_sweep", side_effect=exception), + patch.object(run, "Proposer", DemoProposer), + patch.object(run, "consolidate") as final, + ): + code, out, error = self.invoke("run") + final.assert_not_called() + self.assertEqual(code, expected) + self.assertEqual( + json.loads((root / "events.jsonl").read_text().splitlines()[-1])["status"], status + ) + if isinstance(exception, events.Stopped) and str(exception) == "usage_limit": + self.assertIn("reached the model provider's usage limit", out) + self.assertIn("Run the same command once the limit resets to resume.", out) + if expected == 130: + self.assertIn("Run the same command to resume", out) + for result, expected in ( + ({"domains": [{"failed": 1}]}, 3), + ({"domains": [{"error": "dead"}]}, 1), + ): + with ( + patch.object(run.schedule, "run_sweep", return_value=result), + patch.object(run, "Proposer", DemoProposer), + patch.object(_final, "Proposer", DemoProposer), + ): + self.assertEqual(self.invoke("run")[0], expected) + + def test_human_selector_is_loaded_lazily(self): + self.assertNotIn("harness2.cli.human", sys.modules) + from harness2.core.spec import ChannelSpec + + original = self.suite.make_adapter + + def make(ctx): + adapter = original(ctx) + adapter.spec = replace( + adapter.spec, + channels={**adapter.spec.channels, "selector": ChannelSpec("human", "")}, + ) + return adapter + + self.suite.make_adapter = make + with ( + patch.object(run.schedule, "run_sweep", return_value={"domains": [{"done": 2}]}), + patch.object(run, "Proposer", DemoProposer), + ): + self.assertEqual(self.invoke("run", "--selector", "human")[0], 0) + self.assertIn("harness2.cli.human", sys.modules) + + def test_json_stays_parseable_when_adapter_prints(self): + self.execute() + original = self.suite.make_adapter + + def make(ctx): + print("adapter progress") + adapter = original(ctx) + + def score(path, task): + print("judge progress") + return {"n_passed": 1, "n_criteria": 1} + + adapter.score = score + return adapter + + self.suite.make_adapter = make + code, stdout, stderr = self.invoke("--json", "judge") + self.assertEqual((code, len(json.loads(stdout))), (0, 1)) + self.assertIn("judge progress", stderr) + code, stdout, stderr = self.invoke("--json", "--quiet", "judge") + self.assertEqual((code, len(json.loads(stdout)), stderr), (0, 1, "")) + + def test_config_task_setup_and_json(self): + self.assertEqual(self.invoke("--json", "config", "set", "run.k", "3")[0], 0) + self.assertEqual(self.invoke("--json", "config", "get", "run.k")[:2], (0, "3\n")) + self.assertEqual(self.invoke("config", "set", "run.k", "0")[0], 1) + self.assertEqual(self.invoke("config", "set", "run.k", "4", "--global")[0], 0) + self.assertTrue(settings.global_path().is_file()) + rows = [ + {"id": "x", "domain": "d", "split": "train", "workspace": "files", "has_judge": True} + ] + fake = types.ModuleType("harness2.suite") + fake.new_task = Mock(return_value=self.project / "task.md") + fake.list_tasks = Mock(return_value=rows) + fake.import_tasks = Mock(return_value=["x"]) + with ( + patch.dict(sys.modules, {"harness2.suite": fake}), + patch("harness2.suite", fake, create=True), + ): + self.assertEqual( + self.invoke( + "--json", "task", "new", "x", "--files", "--domain", "d", "--check", "true" + )[0], + 0, + ) + fake.new_task.assert_called_once_with( + self.project / ".harness2/tasks", "x", files=True, domain="d", check="true" + ) + code, stdout, _ = self.invoke("--json", "task", "ls") + self.assertEqual((code, json.loads(stdout)), (0, rows)) + self.assertRegex(self.invoke("task", "ls")[1], r"x +d +train +files +yes") + self.assertEqual(self.invoke("--json", "task", "import", "tasks.csv", "--force")[0], 0) + fake.import_tasks.assert_called_once_with( + self.project / ".harness2/tasks", Path("tasks.csv"), force=True + ) + self.assertEqual(self.invoke("setup", "demo")[0], 1) + self.suite.setup = Mock(return_value=7) + self.assertEqual(self.invoke("setup", "demo", "--example")[0], 7) + self.suite.setup.assert_called_once_with(["--example"]) + + def test_doctor_fake_binary_and_live_deduplication(self): + binary = self.root / "bin/opencode" + binary.parent.mkdir() + binary.write_text("#!/bin/sh\nprintf 'fixture 1.0\\n'\n") + binary.chmod(0o755) + os.environ["PATH"] = str(binary.parent) + os.environ["OPENAI_API_KEY"] = "fixture" + self.config_path.write_text( + '[executor]\nname="opencode"\nmodel="openai/fake"\n[improver]\nchannel="opencode"\nmodel="openai/fake"\n' + ) + with patch( + "harness2.cli.doctor.Proposer.call", return_value=Reply("doctor", "ok", True) + ) as call: + code, stdout, _ = self.invoke("--json", "doctor", "--live") + self.assertEqual(code, 0) + self.assertEqual(call.call_count, 1) + rows = json.loads(stdout) + self.assertEqual(sum(row["check"].endswith(" CLI") for row in rows), 1) + self.assertTrue(any("fixture 1.0" in row["detail"] for row in rows)) + os.environ.pop("OPENAI_API_KEY") + binary.unlink() + code, stdout, _ = self.invoke("--json", "doctor") + self.assertEqual(code, 1) + self.assertTrue( + any(not row["ok"] and row["check"].endswith(" CLI") for row in json.loads(stdout)) + ) + self.config_path.write_text('[improver]\nmodel="invalid"\n') + self.assertEqual(self.invoke("doctor")[0], 1) + + def test_completion_aliases_and_module_entry(self): + for shell in ("bash", "zsh"): + script = completion.generate(main.build_parser(), shell) + for command in main.COMMANDS: + self.assertIn(":" + command, script) + self.assertIn("--only-tasks", script) + binary = shutil.which(shell) + if binary is None: # e.g. no zsh on a Linux CI runner: only the text is checked + continue + result = subprocess.run([binary, "-n"], input=script, text=True, capture_output=True) + self.assertEqual(result.returncode, 0, result.stderr) + if shell == "bash": + probe = ( + script + + '\nCOMP_WORDS=(harness2 run --on); COMP_CWORD=2; _harness2_complete; printf "%s\\n" "${COMPREPLY[@]}"' + ) + else: + probe = ( + 'compdef() { :; }; compadd() { print -l -- "$@"; };\n' + + script + + "\nwords=(harness2 run --on); CURRENT=3; _harness2_complete" + ) + result = subprocess.run([binary], input=probe, text=True, capture_output=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("--only-tasks", result.stdout) + for executable, command in main.ALIASES.items(): + with ( + patch.object(sys, "argv", [executable]), + patch.object(main, "build_parser", wraps=main.build_parser), + ): + with ( + contextlib.redirect_stdout(io.StringIO()) as stdout, + self.assertRaises(SystemExit), + ): + main.main(["--help"]) + self.assertIn("harness2 " + command, stdout.getvalue()) + with patch.object(sys, "argv", ["harness2-check"]): + self.assertEqual(self.invoke("--json", "--live")[0], 1) + result = subprocess.run( + [sys.executable, "-B", "-m", "harness2", "--version"], capture_output=True, text=True + ) + self.assertEqual((result.returncode, result.stdout.strip()), (0, "0.1.0")) + + def test_judge_legacy_cache_and_merge(self): + good = Scored("t", "agg", 1, "agg1", "id", False, n_passed=1, n_criteria=1) + bad = Scored("t", "agg", 1, "agg1", "id", False, error="judge failed") + self.assertEqual(judge._merge([good], [bad]), [good]) + changed = replace(bad, run_id="new") + self.assertEqual(judge._merge([good], [changed]), [changed]) + path = self.root / "judged.json" + self.assertEqual(judge._load_rows(path), []) + row = good.__dict__ | {"graded": True, "frac": 1.0} + row["arm"] = row.pop("mode") + path.write_text(json.dumps([row])) + self.assertEqual(judge._load_rows(path), [good]) + judge._judge_model(path, "new") + path.write_text(json.dumps({"judge_model": "old", "rows": [row]})) + with self.assertRaises(settings.ConfigError): + judge._judge_model(path, "new") + path.write_text("broken") + with self.assertRaises(ValueError): + judge._load_rows(path) + with self.assertRaises(settings.ConfigError): + judge._judge_model(path, "new") + path = self.root / "ns/t/summary.json" + path.parent.mkdir(parents=True) + path.write_text(json.dumps({"selections": {"agg": row}})) + self.assertEqual(judge.collect_selections(path.parent.parent, ("agg",))[0]["mode"], "agg") + path.write_text("broken") + with self.assertRaises(settings.ConfigError): + judge.collect_selections(path.parent.parent, ("agg",)) + + def test_adoption_changes_diff_and_native_export(self): + before = {"a": "old", "d": "removed\n", "same": "same"} + after = {"a": "new", "b": "added\n", "same": "same"} + self.assertEqual(_adopt.changes(before, after), [("a", "M"), ("b", "A"), ("d", "D")]) + self.assertIn("No newline at end of file", _adopt.unified("a", before, after)) + self.assertIn("--- /dev/null", _adopt.unified("b", before, after)) + self.assertIn("+++ /dev/null", _adopt.unified("d", before, after)) + adapter = types.SimpleNamespace(substrate=ClaudeSubstrate()) + config = self.context("config", "show").settings + (self.project / "CLAUDE.md").write_text("old\n") + base = _adopt.project_tree(adapter, config) + self.assertEqual(base, {"CLAUDE.md": "old\n"}) + self.assertEqual(_adopt.dirty_paths(config, adapter), []) + + def git(*args): + return subprocess.run( + ["/usr/bin/git", "-C", str(self.project), *args], + check=True, + capture_output=True, + text=True, + ) + + git("init", "-q") + git("add", "CLAUDE.md") + git("-c", "user.name=test", "-c", "user.email=test@example.test", "commit", "-qm", "base") + (self.project / "unrelated.txt").write_text("keep") + (self.project / "CLAUDE.md").write_text("dirty") + (self.project / ".claude/scripts").mkdir(parents=True) + (self.project / ".claude/scripts/new.py").write_text("pass\n") + self.assertEqual( + _adopt.dirty_paths(config, adapter), [".claude/scripts/new.py", "CLAUDE.md"] + ) + result = _adopt.perform( + config, adapter, {"CLAUDE.md": "adopted\n"}, base=base, branch="adopt" + ) + self.assertEqual(result, {"written": ["CLAUDE.md"], "deleted": []}) + self.assertEqual(git("branch", "--show-current").stdout.strip(), "adopt") + self.assertEqual((self.project / "unrelated.txt").read_text(), "keep") + with self.assertRaises(settings.ConfigError): + _adopt.perform(config, adapter, {}, base=base, branch="adopt") + self.assertEqual( + _adopt.perform(config, adapter, {}, base=base, branch=None)["deleted"], ["CLAUDE.md"] + ) + + def test_missing_and_corrupt_state(self): + self.assertEqual(self.invoke("report")[0], 1) + self.assertEqual(self.invoke("--json", "runs", "ls")[:2], (0, "[]\n")) + self.execute() + ctx = self.context("runs", "ls") + adapter, root = _adopt.open_run(ctx, None) + data = C.load_run(root) + for _, task in data["stream"]: + path = root / "adapt/demo-oc-agg1" / task / "summary.json" + summary = json.loads(path.read_text()) + summary["selections"]["agg"]["ok"] = False + path.write_text(json.dumps(summary)) + with self.assertRaises(settings.ConfigError): + _adopt.adopted_tree(adapter, root, None, "parallel") + (root / "events.jsonl").write_text("broken") + with self.assertRaises(settings.ConfigError): + C.event_totals(root) + with self.assertRaises(settings.ConfigError): + runs.describe(root) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_consolidate.py b/tests/test_consolidate.py new file mode 100644 index 00000000..0c28b556 --- /dev/null +++ b/tests/test_consolidate.py @@ -0,0 +1,448 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline consolidation, adoption, and evaluation through a fake Claude CLI.""" + +from __future__ import annotations + +import contextlib +import io +import json +import os +import shlex +import sys +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock, patch + +from harness2.cli import _adopt, _common, _final, eval, run +from harness2.cli.main import COMMANDS, build_parser, main +from harness2.core import events +from harness2.settings import ConfigError +from harness2.suite import GenericAdapter + +FAKE_CLAUDE = r"""import json +import os +import sys +from pathlib import Path + +if "--version" in sys.argv: + print("fake claude 1.0") + sys.exit(0) + +prompt = sys.stdin.read() +assert prompt.strip() +assert os.environ["ANTHROPIC_API_KEY"] == "offline-fixture" +cwd = Path.cwd() +context = cwd / "context" +output_format = sys.argv[sys.argv.index("--output-format") + 1] +if output_format == "stream-json": + output = cwd / "output/result.txt" + output.parent.mkdir(exist_ok=True) + output.write_text("pass" if "Use shared procedure." in (cwd / "CLAUDE.md").read_text() else "fail") + print(json.dumps({"type": "result", "subtype": "success", "is_error": False, + "result": "Saved the result.", "total_cost_usd": 0.125})) + sys.exit(0) + +harness = context / "harness/CLAUDE.md" +if "You are consolidating an autonomous agent" in prompt: + counter = Path(os.environ["WC_CALLS"]) + call = int(counter.read_text()) + 1 if counter.exists() else 1 + counter.write_text(str(call)) + task_dirs = sorted((context / "tasks").iterdir()) + index = (context / "INDEX.md").read_text().splitlines() + assert len(index) == len(task_dirs) + assert f"run on {len(task_dirs)} task(s)" in prompt + for task in task_dirs: + assert (task / "task.md").read_text() == f"Complete {task.name}.\n" + assert "Task note." in (task / "harness/CLAUDE.md").read_text() + assert (task / "EXPERIENCE.md").is_file() == any( + line.startswith(f"- {task.name} ") and line.endswith("lesson: yes") for line in index) + assert all("(ops/review): adopted harness differs from the start; lesson:" in line for line in index) + assert not list(context.rglob("rubric.md")) + behavior = os.environ.get("WC_BEHAVIOR", "retry") + invalid = context / "harness/.claude/scripts/broken.py" + if behavior == "missing": + import shutil + shutil.rmtree(context / "harness") + elif behavior == "invalid" or (behavior == "retry" and call == 1): + invalid.parent.mkdir(parents=True, exist_ok=True) + invalid.write_text("def broken(") + else: + if behavior == "retry" and call == 2: + assert "# YOUR PREVIOUS ATTEMPT WAS REJECTED" in prompt + assert "broken.py" in prompt + assert invalid.is_file() + invalid.unlink(missing_ok=True) + harness.write_text("Read sources.\n" if behavior == "unchanged" else "Use shared procedure.\n") + text = "a-one and b-two support the shared procedure." +elif (context / "adoption.json").is_file(): + text = ("\ncomponent: CLAUDE.md\ncandidates: inv1\n" + "change: added a procedure\nobserved: the output was produced\nverdict: helped\n\n" + "\ncue: When doing project work\nlesson: Use a shared procedure.\n" + "evidence: A task note accompanied the completed output.\n" + "confidence: specific to this task\n") +else: + harness.write_text(harness.read_text() + "Task note.\n") + text = "Added a task note." +print(json.dumps({"type": "result", "subtype": "success", "is_error": False, + "result": text, "total_cost_usd": 0.01})) +""" + + +class ConsolidateTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.project = self.root / "project" + self.state = self.project / ".harness2" + self.state.mkdir(parents=True) + (self.project / "CLAUDE.md").write_text("Read sources.\n") + (self.state / "config.toml").write_text( + '[executor]\nname = "claude"\nmodel = "anthropic/x"\n' + '[improver]\nchannel = "claude"\nmodel = "anthropic/y"\n' + '[providers.anthropic]\napi_key_env = "WC_KEY"\n' + ) + binaries = self.root / "bin" + binaries.mkdir() + binary = binaries / "claude" + binary.write_text(f"#!{sys.executable}\n" + FAKE_CLAUDE) + binary.chmod(0o755) + self.calls = self.root / "consolidate-calls" + environment = patch.dict( + os.environ, + { + "PATH": f"{binaries}{os.pathsep}{os.defpath}", + "XDG_CONFIG_HOME": str(self.root / "config"), + "XDG_CACHE_HOME": str(self.root / "cache"), + "WC_KEY": "offline-fixture", + "WC_CALLS": str(self.calls), + "PYTHONDONTWRITEBYTECODE": "1", + }, + clear=True, + ) + environment.start() + self.addCleanup(environment.stop) + self.addCleanup(events.start, None) + for task, split in (("a-one", "train"), ("b-two", "test")): + self.new_task(task, split=split) + self.run_dir = self.state / "runs/trial" + + def invoke(self, *argv, code=0): + stdout, stderr = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + result = main(["-C", str(self.project), *argv]) + self.assertEqual(result, code, f"{argv!r}\n{stdout.getvalue()}\n{stderr.getvalue()}") + return stdout.getvalue(), stderr.getvalue() + + def new_task(self, task, *, split="train", judge=True): + check = shlex.join( + [ + sys.executable, + "-B", + "-c", + "from pathlib import Path; assert Path('output/result.txt').read_text() == 'pass'", + ] + ) + self.invoke( + "task", + "new", + task, + "--files", + "--domain", + "ops/review", + *(["--check", check] if judge else []), + ) + directory = self.state / "tasks" / task + (directory / "task.md").write_text(f"Complete {task}.\n") + with (directory / "task.toml").open("a") as stream: + stream.write(f'split = "{split}"\n') + + def execute(self, *argv, code=0): + return self.invoke("run", "--name", "trial", *argv, code=code) + + def adapter(self): + args = build_parser().parse_args(["-C", str(self.project), "diff", "trial"]) + return _adopt.open_run(_common.Context(args, {}), "trial")[0] + + def test_two_tasks_retry_adoption_eval_and_json(self): + text, _ = self.execute() + before = _common.event_totals(self.run_dir) + self.assertEqual(self.calls.read_text(), "2") + path = self.run_dir / "consolidated/parallel/harness" + self.assertEqual((path / "CLAUDE.md").read_text(), "Use shared procedure.\n") + self.assertIn(f"Final harness (parallel): {path}", text) + self.assertIn( + "Next: harness2 diff trial shows it; harness2 apply trial writes it into the project; " + "harness2 eval trial measures it first.", + text, + ) + meta = json.loads((path.parent / "consolidation.json").read_text()) + self.assertEqual(meta["tasks"], ["a-one", "b-two"]) + self.assertEqual((meta["model"], meta["cost_usd"]), ("anthropic/y", 0.02)) + self.assertTrue(meta["created"]) + self.assertEqual( + (path.parent / "notes.md").read_text(), + "a-one and b-two support the shared procedure.\n", + ) + data = json.loads(self.invoke("--json", "run", "--name", "trial")[0]) + self.assertEqual(set(data), {"result", "totals", "final"}) + self.assertEqual(data["final"], {"parallel": str(path)}) + self.assertEqual(data["totals"]["calls"], before["calls"]) + self.assertEqual(data["totals"]["spent_usd"], before["spent_usd"]) + self.assertEqual(self.calls.read_text(), "2") + self.assertTrue(list((self.run_dir / "consolidated/_audit").glob("*.response.md"))) + consolidated = json.loads(self.invoke("--json", "diff", "trial")[0]) + self.assertEqual(consolidated["label"], "consolidated") + self.assertIn("+Use shared procedure.", consolidated["diffs"]["CLAUDE.md"]) + task = json.loads(self.invoke("--json", "diff", "trial", "--task", "a-one")[0]) + self.assertEqual(task["label"], "a-one") + self.assertIn("+Task note.", task["diffs"]["CLAUDE.md"]) + applied, _ = self.invoke("apply", "trial", "--dry-run", "--task", "a-one") + self.assertIn("Harness adopted for a-one (run trial, parallel)", applied) + applied, _ = self.invoke("apply", "trial", "--yes") + self.assertIn("Final harness of run trial (parallel)", applied) + self.assertEqual((self.project / "CLAUDE.md").read_text(), "Use shared procedure.\n") + text, _ = self.invoke("eval", "trial", "--repeat", "2") + self.assertIn( + "Comparing the final harness with the starting one on 2 task(s), 2 run(s) each.", + text, + ) + self.assertRegex(text, r"TASK +START +FINAL\n +a-one +0% +100%\n +b-two +0% +100%") + self.assertRegex(text, r"mean +0% +100% \(\+100 points\)") + self.assertIn(f"Saved to {self.run_dir / 'eval/parallel/results.json'}", text) + results = json.loads((self.run_dir / "eval/parallel/results.json").read_text()) + self.assertEqual(results["mean"], {"start": 0.0, "consolidated": 1.0}) + for row in results["tasks"].values(): + self.assertEqual( + row["start"], {"runs": [{"n_passed": 0, "n_criteria": 1}] * 2, "mean": 0.0} + ) + self.assertEqual( + row["consolidated"], {"runs": [{"n_passed": 1, "n_criteria": 1}] * 2, "mean": 1.0} + ) + event_rows = [ + json.loads(line) for line in (self.run_dir / "events.jsonl").read_text().splitlines() + ] + eval_rows = [row for row in event_rows if row.get("run_id", "").startswith("eval/")] + self.assertEqual(len(eval_rows), 8) + self.assertEqual({row["cost_usd"] for row in eval_rows}, {0.125}) + self.assertAlmostEqual( + _common.event_totals(self.run_dir)["spent_usd"], before["spent_usd"] + 1.0 + ) + self.assertEqual( + {row["run_id"] for row in eval_rows}, + { + f"eval/{name}/{task}/{i}" + for task in ("a-one", "b-two") + for name in ("start", "consolidated") + for i in range(2) + }, + ) + + def test_eval_filters_skipped_failed_grades_and_limits(self): + self.new_task("c-no-judge", judge=False) + self.execute() + text, _ = self.invoke("eval", "trial") + self.assertIn("Skipped 1 task(s) with no check or rubric: c-no-judge", text) + data = json.loads( + self.invoke( + "--json", + "eval", + "trial", + "--split", + "test", + "--only-tasks", + "b-two", + "--repeat", + "2", + )[0] + ) + self.assertEqual(list(data["tasks"]), ["b-two"]) + self.assertEqual(data["skipped"], []) + self.assertEqual(data["repeat"], 2) + self.assertEqual(json.loads(Path(data["path"]).read_text()), data) + with patch.object(GenericAdapter, "score", return_value=None): + text, _ = self.invoke("eval", "trial", "--only-tasks", "a-one", code=3) + self.assertRegex(text, r"a-one +- +-") + self.assertRegex(text, r"mean +- +- \(- points\)") + data = json.loads((self.run_dir / "eval/parallel/results.json").read_text()) + self.assertEqual(data["tasks"]["a-one"]["start"], {"runs": [None], "mean": None}) + text, _ = self.invoke("-c", "run.max_calls=1", "eval", "trial", code=4) + self.assertIn("Run trial stopped: reached --max-calls 1", text) + with patch.object(GenericAdapter, "execute", side_effect=events.Stopped("usage_limit")): + self.invoke("eval", "trial", code=4) + _, err = self.invoke("eval", "trial", "--repeat", "0", code=1) + self.assertIn("--repeat must be a positive integer", err) + _, err = self.invoke("eval", "trial", "--only-tasks", "missing", code=1) + self.assertIn("unknown task id(s)", err) + data = json.loads(self.invoke("--json", "eval", "trial", "--only-tasks", "c-no-judge")[0]) + self.assertEqual(data["tasks"], {}) + self.assertEqual(data["mean"], {"start": None, "consolidated": None}) + self.assertEqual(data["skipped"], ["c-no-judge"]) + + def test_modes_and_unchanged_harness(self): + text, _ = self.execute("--mode", "parallel,sequential") + self.assertEqual(self.calls.read_text(), "3") + adapter = self.adapter() + for mode in ("parallel", "sequential"): + path = self.run_dir / "consolidated" / mode / "harness" + self.assertIn(f"Final harness ({mode}): {path}", text) + self.assertEqual( + _adopt.harness_for(adapter, self.run_dir, None, mode)[0], "consolidated" + ) + self.assertEqual( + _adopt.harness_for(adapter, self.run_dir, "a-one", "sequential")[0], "a-one" + ) + with patch.dict(os.environ, {"WC_BEHAVIOR": "unchanged"}): + self.invoke("run", "--name", "unchanged") + self.assertEqual(self.invoke("diff", "unchanged")[0], "\n") + + def test_validation_failure_keeps_exit_code_and_can_retry(self): + with patch.dict(os.environ, {"WC_BEHAVIOR": "invalid"}): + text, err = self.execute() + self.assertIn("Could not build the final harness (parallel): ", err) + self.assertIn("broken.py", err) + self.assertTrue(err.rstrip().endswith(". Run the same command again to retry.")) + self.assertNotIn("Final harness (parallel):", text) + self.assertFalse((self.run_dir / "consolidated/parallel/harness").exists()) + self.assertEqual(self.calls.read_text(), "2") + _, err = self.invoke("eval", "trial", code=1) + self.assertIn("this run has no final harness yet; run `harness2 run` to finish it", err) + with patch.dict(os.environ, {"WC_BEHAVIOR": "valid"}): + self.execute() + self.assertEqual(self.calls.read_text(), "3") + path = self.run_dir / "consolidated/parallel/harness/CLAUDE.md" + self.assertEqual(path.read_text(), "Use shared procedure.\n") + for failure, expected in ( + ("failure", 0), + ("failure", 3), + ("failure", 1), + (events.Stopped("usage_limit"), 0), + ): + error = failure if isinstance(failure, Exception) else ConfigError(failure) + result = { + "domains": [ + {"failed": int(expected == 3), **({"error": "dead"} if expected == 1 else {})} + ] + } + with ( + patch.object(run.schedule, "run_sweep", return_value=result), + patch.object(run, "consolidate", side_effect=error), + ): + text, err = self.execute(code=expected) + self.assertEqual( + err, + f"Could not build the final harness (parallel): {error}. " + "Run the same command again to retry.\n", + ) + self.assertEqual(path.read_text(), "Use shared procedure.\n") + + def test_missing_workspace_failure_keeps_exit_code(self): + with patch.dict(os.environ, {"WC_BEHAVIOR": "missing"}): + _, err = self.execute() + self.assertIn( + "Could not build the final harness (parallel): " + "the harness directory did not come back from the session. " + "Run the same command again to retry.", + err, + ) + self.assertEqual(self.calls.read_text(), "2") + + def test_context_stream_order_optional_lessons_and_errors(self): + self.execute() + adapter = self.adapter() + record = _common.load_run(self.run_dir) + plan = record["schedule"] + lesson = ( + _common.library_root_for(adapter, plan["k"], seed=plan.get("seed")) + / "agg/ops__review/harnesses/b-two/EXPERIENCE.md" + ) + lesson.unlink() + record["stream"].reverse() + dest = self.root / "context" + base = _adopt.base_tree(adapter) + task_path, row = next( + (path, row) + for path, row in _common.summaries(self.run_dir) + if row["task_id"] == "b-two" + ) + row["selections"]["agg"]["chosen_tree_is_base"] = True + task_path.write_text(json.dumps(row)) + tasks = _final._context(adapter, self.run_dir, record, "parallel", dest, base) + self.assertEqual(tasks, ["b-two", "a-one"]) + self.assertEqual( + (dest / "INDEX.md").read_text(), + "- b-two (ops/review): adopted harness same as the start; lesson: no\n" + "- a-one (ops/review): adopted harness differs from the start; lesson: yes\n", + ) + self.assertEqual((dest / "tasks/b-two/task.md").read_text(), "Complete b-two.\n") + self.assertFalse((dest / "tasks/b-two/EXPERIENCE.md").exists()) + for path, row in _common.summaries(self.run_dir): + row["selections"]["agg"]["ok"] = False + path.write_text(json.dumps(row)) + with self.assertRaisesRegex(ConfigError, "no successful selection"): + _final._context(adapter, self.run_dir, record, "parallel", self.root / "empty", base) + with ( + patch.object(run.schedule, "run_sweep", return_value={"domains": [{"failed": 2}]}), + patch.object(run, "consolidate") as final, + ): + data = json.loads(self.invoke("--json", "run", "--name", "trial", code=3)[0]) + final.assert_not_called() + self.assertEqual(data["final"], {}) + + def test_rollout_grades_and_optional_cost(self): + adapter = SimpleNamespace(execute=Mock(return_value=self.root), score=Mock()) + for grade in ( + None, + {}, + {"n_passed": 1, "n_criteria": 0}, + {"n_passed": True, "n_criteria": 1}, + {"n_passed": 2, "n_criteria": 1}, + ): + adapter.score.return_value = grade + self.assertIsNone(eval._rollout(adapter, "task", None, "eval/start/task/0")) + adapter.score.side_effect = ValueError("failed judge") + self.assertIsNone(eval._rollout(adapter, "task", None, "eval/start/task/0")) + adapter.score.side_effect = events.Stopped("usage_limit") + with self.assertRaisesRegex(events.Stopped, "usage_limit"): + eval._rollout(adapter, "task", None, "eval/start/task/0") + self.assertEqual(eval._mean([0.0, None, 1.0]), 0.5) + self.assertIsNone(eval._mean([None])) + data = { + "tasks": {}, + "repeat": 1, + "skipped": [], + "path": "results.json", + "mean": {"start": 1.0, "consolidated": 0.0}, + } + self.assertRegex(eval._text(data), r"mean +100% +0% \(-100 points\)") + + def test_command_order_and_help(self): + i = COMMANDS.index("diff") + self.assertEqual(COMMANDS[i + 1], "eval") + self.assertNotIn("consolidate", COMMANDS) + parser = build_parser() + sub = parser._subparsers._group_actions[0].choices["eval"] + for action in sub._actions: + self.assertTrue(action.help) + if action.dest != "help": + self.assertEqual(action.help, action.help.lower()) + self.assertFalse(action.help.endswith(".")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_e2e.py b/tests/test_e2e.py new file mode 100644 index 00000000..c41f3c63 --- /dev/null +++ b/tests/test_e2e.py @@ -0,0 +1,343 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Real CLI and tasks-suite workflow against one offline Claude executable.""" + +from __future__ import annotations + +import contextlib +import io +import json +import os +import shlex +import subprocess +import sys +import tempfile +import unittest +from collections import Counter +from pathlib import Path +from unittest.mock import patch + +from harness2.cli import _adopt, _common +from harness2.cli.main import build_parser, main +from harness2.core import events, runmark + +FAKE_CLAUDE = r"""import json +import os +import sys +from pathlib import Path + +if "--version" in sys.argv: + print("fake claude 1.0") + sys.exit(0) + +prompt = sys.stdin.read() +assert prompt.strip(), "missing stdin prompt" +assert os.environ["ANTHROPIC_API_KEY"] == os.environ["E2E_API_KEY"] +cwd = Path.cwd() +output_format = sys.argv[sys.argv.index("--output-format") + 1] +model = sys.argv[sys.argv.index("--model") + 1] +context = cwd / "context" +if output_format == "stream-json": + role = "executor" +elif (context / "candidates").is_dir(): + role = "selector" +elif (context / "adoption.json").is_file(): + role = "reflector" +else: + role = "improver" +with open(os.environ["E2E_INVOCATIONS"], "a") as log: + log.write(json.dumps({"role": role, "model": model, "cwd": str(cwd)}) + "\n") + +if output_format == "stream-json": + assert model == "x" + assert not (cwd / ".harness2").exists(), "project state leaked into the workspace" + assert (cwd / ".claude/skills/checks/SKILL.md").is_file() + hooks = json.loads((cwd / ".claude/settings.json").read_text())["hooks"] + assert "PreToolUse" in hooks + if (cwd / "tiny_pkg/arithmetic.py").is_file(): + assert "Fix add" in prompt + target = cwd / "tiny_pkg/arithmetic.py" + target.write_text(target.read_text().replace("return a - b", "return a + b")) + result = "Repaired add in tiny_pkg/arithmetic.py." + else: + assert "Uppercase" in prompt + target = cwd / "output/result.txt" + target.parent.mkdir(exist_ok=True) + target.write_text((cwd / "message.txt").read_text().upper()) + result = "Saved output/result.txt." + for event in ( + {"type": "system", "subtype": "init", "model": model, "cwd": str(cwd)}, + {"type": "assistant", "message": {"role": "assistant", "content": [ + {"type": "tool_use", "id": "edit-1", "name": "Write", + "input": {"file_path": str(target)}}]}}, + {"type": "user", "message": {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "edit-1", "content": "File saved."}]}}, + {"type": "assistant", "message": {"role": "assistant", "content": [ + {"type": "text", "text": result}]}}, + {"type": "result", "subtype": "success", "is_error": False, + "result": result, "total_cost_usd": 0.125}, + ): + print(json.dumps(event)) +else: + assert output_format == "json" and model == "y" + harness = context / "harness/CLAUDE.md" + if harness.is_file(): + guidance = "Use the checks skill before finishing.\n" + if guidance not in harness.read_text(): + harness.write_text(harness.read_text() + guidance) + if role == "selector": + chosen = sorted((context / "candidates").glob("*.md"))[0].stem + text = "The requested change is present.\n```choice\n" + chosen + "\n```" + elif role == "reflector": + text = ( + "\ncomponent: CLAUDE.md\ncandidates: inv1\n" + "change: named the checks skill\nobserved: the requested output was produced\n" + "verdict: helped\n\n\n" + "\ncue: When completing a task\n" + "lesson: Use the checks skill before finishing.\n" + "evidence: CLAUDE.md named the checks skill and the run produced its output.\n" + "confidence: specific to this task\n" + ) + else: + text = "Name the checks skill in CLAUDE.md." + print(json.dumps({"type": "result", "subtype": "success", "is_error": False, + "result": text, "total_cost_usd": 0.01})) +""" + + +class EndToEndTests(unittest.TestCase): + def test_tasks_cli_run_judge_adopt_and_resume(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + project = root / "project" + project.mkdir() + binary_dir = root / "bin" + binary_dir.mkdir() + binary = binary_dir / "claude" + binary.write_text(f"#!{sys.executable}\n" + FAKE_CLAUDE) + binary.chmod(0o755) + invocation_log = root / "invocations.jsonl" + env = { + "PATH": f"{binary_dir}{os.pathsep}{os.defpath}", + "XDG_CONFIG_HOME": str(root / "config"), + "XDG_CACHE_HOME": str(root / "cache"), + "E2E_API_KEY": "offline-fixture", + "E2E_INVOCATIONS": str(invocation_log), + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": os.devnull, + "PYTHONDONTWRITEBYTECODE": "1", + } + self.addCleanup(events.start, None) + with patch.dict(os.environ, env, clear=True): + package = project / "tiny_pkg" + package.mkdir() + (package / "__init__.py").write_text("") + (package / "arithmetic.py").write_text("def add(a, b):\n return a - b\n") + tests = project / "tests" + tests.mkdir() + (tests / "test_arithmetic.py").write_text( + "import unittest\nfrom tiny_pkg.arithmetic import add\n\n" + "class ArithmeticTests(unittest.TestCase):\n" + " def test_add(self):\n self.assertEqual(add(2, 3), 5)\n" + ) + base_instructions = "Read the source before editing.\n" + (project / "CLAUDE.md").write_text(base_instructions) + skill = project / ".claude/skills/checks/SKILL.md" + skill.parent.mkdir(parents=True) + skill_text = ( + "---\nname: checks\ndescription: Run the project tests.\n---\n" + "Run python -m unittest discover -s tests.\n" + ) + skill.write_text(skill_text) + native_settings = { + "hooks": { + "PreToolUse": [ + {"matcher": "Bash", "hooks": [{"type": "command", "command": "true"}]} + ] + }, + "permissions": {"allow": ["Read"]}, + } + native_path = project / ".claude/settings.json" + native_path.write_text(json.dumps(native_settings)) + subprocess.run(["git", "init", "-q", str(project)], check=True) + subprocess.run(["git", "add", "."], cwd=project, check=True) + subprocess.run( + [ + "git", + "-c", + "user.name=fixture", + "-c", + "user.email=fixture@localhost", + "-c", + "commit.gpgsign=false", + "commit", + "-qm", + "fixture", + ], + cwd=project, + check=True, + ) + state = project / ".harness2" + state.mkdir() + (state / "config.toml").write_text( + '[executor]\nname = "claude"\nmodel = "anthropic/x"\n' + '[improver]\nchannel = "claude"\nmodel = "anthropic/y"\n' + '[providers.anthropic]\napi_key_env = "E2E_API_KEY"\n' + ) + + def invoke(*argv): + stdout, stderr = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + code = main(["-C", str(project), *argv]) + self.assertEqual(code, 0, f"{argv!r}\n{stdout.getvalue()}\n{stderr.getvalue()}") + return stdout.getvalue() + + project_check = shlex.join( + [sys.executable, "-B", "-m", "unittest", "discover", "-s", "tests", "-v"] + ) + invoke("task", "new", "fix-add", "--check", project_check) + (state / "tasks/fix-add/task.md").write_text("Fix add so 2 plus 3 returns 5.\n") + files_check = shlex.join( + [ + sys.executable, + "-B", + "-c", + "from pathlib import Path; assert Path('output/result.txt').read_text() == 'HELLO\\n'", + ] + ) + invoke("task", "new", "write-file", "--files", "--check", files_check) + (state / "tasks/write-file/task.md").write_text( + "Uppercase message.txt and save output/result.txt.\n" + ) + (state / "tasks/write-file/files/message.txt").write_text("hello\n") + + invoke("run", "--k", "1") + invoke("run", "--k", "2", "--name", "k2") + all_calls = [json.loads(line) for line in invocation_log.read_text().splitlines()] + self.assertEqual( + Counter(call["role"] for call in all_calls), + { + "executor": 24, + "improver": 20, + "selector": 2, + "reflector": 4, + }, + ) + self.assertTrue( + all( + call["model"] == ("x" if call["role"] == "executor" else "y") + for call in all_calls + ) + ) + for name, k in (("claude-k1", 1), ("k2", 2)): + run_dir = state / "runs" / name + record = _common.load_run(run_dir) + self.assertEqual(record["schedule"]["k"], k) + self.assertEqual( + record["stream"], [["default", "fix-add"], ["default", "write-file"]] + ) + event_rows = [ + json.loads(line) + for line in (run_dir / "events.jsonl").read_text().splitlines() + ] + self.assertEqual( + (event_rows[0]["kind"], event_rows[-1]["status"]), ("run_start", "ok") + ) + self.assertEqual(sum(row["kind"] == "rollout" for row in event_rows), 8 * k) + self.assertTrue( + all( + row["cost_usd"] == 0.125 + for row in event_rows + if row["kind"] == "rollout" + ) + ) + self.assertTrue( + all( + row["cost_usd"] == 0.01 + for row in event_rows + if row["kind"] == "session" + ) + ) + run_rows = json.loads(invoke("--json", "runs", "ls")) + self.assertEqual( + { + row["name"]: (row["done"], row["total"], row["spent_usd"]) + for row in run_rows + }, + { + "claude-k1": (2, 2, 1.09), + "k2": (2, 2, 2.17), + }, + ) + invoke("judge") + rows = json.loads(invoke("--json", "report")) + self.assertEqual({row["task_id"] for row in rows}, {"fix-add", "write-file"}) + self.assertTrue( + all(row["name"] == "k2" and row["graded"] and row["all_pass"] for row in rows) + ) + self.assertTrue(all(row["raw"]["exit_code"] == 0 for row in rows)) + self.assertTrue( + any( + "Ran 1 test" in row["raw"]["output_tail"] + for row in rows + if row["task_id"] == "fix-add" + ) + ) + diff = invoke("diff") + self.assertIn("--- a/CLAUDE.md", diff) + self.assertIn("+Use the checks skill before finishing.", diff) + + args = build_parser().parse_args(["-C", str(project), "diff"]) + ctx = _common.Context(args, {}) + adapter, run_dir = _adopt.open_run(ctx, None) + self.assertEqual(run_dir.name, "k2") + chosen_task, adopted = _adopt.adopted_tree(adapter, run_dir, None, "parallel") + self.assertEqual(chosen_task, "write-file") + base = _adopt.base_tree(adapter) + self.assertEqual(base["CLAUDE.md"], base_instructions) + self.assertEqual(_adopt.changes(base, adopted), [("CLAUDE.md", "M")]) + for _, summary in _common.summaries(run_dir): + selected = summary["selections"]["agg"] + rollout = runmark.resolve_path(selected["chosen_run_dir"], run_dir) + self.assertTrue(adapter.has_judge(summary["task_id"])) + self.assertEqual( + json.loads((rollout / "judge.json").read_text())["n_passed"], 1 + ) + if summary["task_id"] == "fix-add": + deliverable = (rollout / "deliverable.patch").read_text() + self.assertIn("+ return a + b", deliverable) + self.assertNotIn("CLAUDE.md", deliverable) + self.assertNotIn(".claude/", deliverable) + self.assertNotIn(".harness2/", deliverable) + else: + self.assertEqual((rollout / "output/result.txt").read_text(), "HELLO\n") + applied = _adopt.perform(ctx.settings, adapter, adopted, base=base, branch=None) + self.assertEqual(applied, {"written": ["CLAUDE.md"], "deleted": []}) + self.assertEqual((project / "CLAUDE.md").read_text(), adopted["CLAUDE.md"]) + self.assertEqual(json.loads(native_path.read_text()), native_settings) + self.assertEqual(skill.read_text(), skill_text) + self.assertIn("return a - b", (package / "arithmetic.py").read_text()) + before_resume = invocation_log.read_bytes() + invoke("run", "--k", "1") + self.assertEqual(invocation_log.read_bytes(), before_resume) + worktrees = subprocess.check_output( + ["git", "worktree", "list", "--porcelain"], cwd=project, text=True + ) + self.assertEqual(worktrees.count("worktree "), 1) + print("E2E: 2 task types; k=1/k=2; 50 calls; 2 graded; adopted; resume=0 calls") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_executors.py b/tests/test_executors.py new file mode 100644 index 00000000..ffc20f51 --- /dev/null +++ b/tests/test_executors.py @@ -0,0 +1,592 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline executor contracts, native-file safety and documented Claude stream shapes.""" + +from __future__ import annotations + +import json +import os +import shlex +import shutil +import subprocess +import sys +import tempfile +import unittest +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from harness2 import providers +from harness2.core import prompts +from harness2.core import tree as HT +from harness2.core.spec import INSTALL_MODES, BenchPrompts +from harness2.settings import ConfigError +from harness2.substrate import ( + CLAUDE_SURFACE, + CODEX_SURFACE, + OPENCODE_SURFACE, + ClaudeSubstrate, + CodexSubstrate, + CommandSubstrate, + OpencodeSubstrate, +) +from harness2.substrate.claude import parse_events + + +class SurfaceTests(unittest.TestCase): + def test_script_dir_and_references(self): + for surface in (CLAUDE_SURFACE, CODEX_SURFACE, OPENCODE_SURFACE): + with self.subTest(surface=surface.name): + script = f"{surface.script_dir}/helper.py" + self.assertTrue(HT.is_allowed(surface, script)) + if surface.script_dir != "scripts": + self.assertFalse(HT.is_allowed(surface, "scripts/helper.py")) + tree = {surface.always_on[0]: f"python3 {script}", script: "print('ok')"} + HT.validate(surface, tree) + del tree[script] + with self.assertRaisesRegex(ValueError, "but that file is not in the harness"): + HT.validate(surface, tree) + tree = { + surface.always_on[0]: "helper", + f"{surface.skill_dir}/helper/SKILL.md": f"---\nname: helper\ndescription: Help\n---\npython3 {surface.skill_dir}/helper/scripts/tool.py", + } + with self.assertRaisesRegex(ValueError, "tool.py.*not in the harness"): + HT.validate(surface, tree) + tree[f"{surface.skill_dir}/helper/scripts/tool.py"] = "pass" + HT.validate(surface, tree) + + def test_skills_can_be_discovered_without_trigger(self): + tree = {".claude/skills/helper/SKILL.md": "---\ndescription: Help\n---\nDo it."} + before = dict(tree) + HT.validate(CLAUDE_SURFACE, tree) + self.assertEqual(HT.autotrigger(CLAUDE_SURFACE, tree), []) + self.assertEqual(tree, before) + required = replace(CLAUDE_SURFACE, skills_need_trigger=True) + with self.assertRaisesRegex(ValueError, "never named"): + HT.validate(required, tree) + tree[".claude/skills/helper/SKILL.md"] = "missing description" + with self.assertRaisesRegex(ValueError, "description"): + HT.validate(CLAUDE_SURFACE, tree) + + def test_globs_override_component_rules_but_keep_caps_and_compile(self): + substrate = CommandSubstrate("true", ("custom/*",)) + surface = substrate.surface() + self.assertEqual(substrate.NATIVE_PATHS, (":(glob)custom/*",)) + self.assertTrue(HT.is_allowed(surface, "custom/a.txt")) + self.assertFalse(HT.is_allowed(surface, "sub_agents/helper/AGENT.md")) + self.assertFalse(HT.is_allowed(surface, "../custom/a.txt")) + HT.validate(surface, {"custom/hook.json": "bad json", "ignored.py": "bad syntax!"}) + with self.assertRaisesRegex(ValueError, "does not compile"): + HT.validate(surface, {"custom/tool.py": "bad syntax!"}) + special = replace(surface, editable_globs=("**",), hooks_file="hooks.json") + HT.validate( + special, + {"sub_agents/a/AGENT.md": "no description", "hooks.json": "bad"}, + allow_subagents=False, + ) + capped = replace(surface, always_on=("custom/a.txt",), always_on_max=2) + with self.assertRaisesRegex(ValueError, "over the 2 cap"): + HT.validate(capped, {"custom/a.txt": "long"}) + with self.assertRaises(ValueError): + CommandSubstrate("true", ()) + + def test_new_install_modes_and_all_surface_prompts(self): + self.assertIn("claude_root", INSTALL_MODES) + self.assertIn("command_files", INSTALL_MODES) + for surface in ( + CLAUDE_SURFACE, + CODEX_SURFACE, + OPENCODE_SURFACE, + CommandSubstrate("true", ("*.md",)).surface(), + ): + with self.subTest(surface=surface.name): + text = prompts.propose(BenchPrompts(work="x"), "INV", surface) + self.assertNotIn("__", text) + + def test_glob_surface_accepts_empty_directory_as_tree(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + command = CommandSubstrate("true", ("custom/*",)).surface() + self.assertTrue(HT.is_tree(command, root)) + self.assertFalse(HT.is_tree(command, root / "missing")) + file = root / "file.txt" + file.write_text("content") + self.assertFalse(HT.is_tree(command, file)) + for surface in (CLAUDE_SURFACE, CODEX_SURFACE, OPENCODE_SURFACE): + self.assertFalse(HT.is_tree(surface, root)) + always_on = root / surface.always_on[0] + always_on.write_text("") + self.assertTrue(HT.is_tree(surface, root)) + always_on.unlink() + + +class NativeTests(unittest.TestCase): + def setUp(self): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + self.root = Path(temp.name) + + def write(self, rel, text): + path = self.root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + def native_project(self, substrate): + surface = substrate.surface() + self.write(surface.always_on[0], "Use helper.") + self.write(f"{surface.skill_dir}/helper/SKILL.md", "---\ndescription: Help\n---\nBody") + self.write(f"{surface.skill_dir}/helper/scripts/tool.py", "pass\n") + self.write(f"{surface.skill_dir}/helper/reference.md", "keep reference") + self.write(f"{surface.script_dir}/tool.py", "pass\n") + self.write(f"{surface.script_dir}/extra.txt", "keep extra") + hooks = {"hooks": {"Stop": [{"hooks": [{"type": "command", "command": "true"}]}]}} + if surface.name == "claude": + self.write( + ".claude/settings.json", json.dumps({"permissions": {"allow": ["Read"]}, **hooks}) + ) + self.write(".claude/agents/helper.md", "---\ndescription: Help\n---\nDo it.") + self.write(".claude/agents/notes.txt", "keep notes") + else: + self.write(surface.hooks_file, json.dumps(hooks)) + self.write(".codex/config.toml", "keep = true") + return hooks + + def test_claude_round_trip_hooks_merge_and_bounded_deletions(self): + sub = ClaudeSubstrate() + hooks = self.native_project(sub) + rejected = [] + tree = sub.import_tree(self.root, rejected=rejected) + self.assertEqual( + set(rejected), + { + ".claude/skills/helper/reference.md", + ".claude/scripts/extra.txt", + ".claude/agents/notes.txt", + }, + ) + self.assertIn("sub_agents/helper/AGENT.md", tree) + self.assertEqual(json.loads(tree[".claude/hooks.json"]), hooks) + dest = self.root / "destination" + settings = dest / ".claude/settings.json" + settings.parent.mkdir(parents=True) + settings.write_text('{"model": "keep"}') + result = sub.export_tree(tree, dest, base={}) + self.assertIn(".claude/agents/helper.md", result["written"]) + self.assertIn("name: helper", (dest / ".claude/agents/helper.md").read_text()) + self.assertEqual(json.loads(settings.read_text()), {"model": "keep", **hooks}) + imported = sub.import_tree(dest) + self.assertEqual( + sub.export_tree(imported, dest, base=imported), {"written": [], "deleted": []} + ) + self.assertEqual(tree.keys(), imported.keys()) + changed = {k: v for k, v in tree.items() if k not in (".claude/hooks.json", "CLAUDE.md")} + result = sub.export_tree(changed, self.root, base=tree) + self.assertEqual(result["deleted"], ["CLAUDE.md"]) + self.assertIn(".claude/settings.json", result["written"]) + self.assertEqual( + json.loads((self.root / ".claude/settings.json").read_text()), + {"permissions": {"allow": ["Read"]}}, + ) + self.assertEqual( + (self.root / ".claude/skills/helper/reference.md").read_text(), "keep reference" + ) + + def test_claude_agent_name_inserted_only_when_missing(self): + sub = ClaudeSubstrate() + for name, body in ( + ("present", "---\nname: present\ndescription: Help\n---\nBody"), + ("plain", "Body"), + ("empty", "---\nname: \ndescription: Help\n---\nBody"), + ("header", "---\n---\nBody"), + ): + sub.export_tree({f"sub_agents/{name}/AGENT.md": body}, self.root, base={}) + exported = (self.root / f".claude/agents/{name}.md").read_text() + self.assertIn(f"name: {name}\n", exported) + self.assertEqual(sum(line.startswith("name:") for line in exported.splitlines()), 1) + if name == "present": + self.assertEqual(exported, body) + + def test_codex_round_trip_no_stubs_or_subagents_and_bounded_deletions(self): + sub = CodexSubstrate() + self.native_project(sub) + rejected = [] + tree = sub.import_tree(self.root, rejected=rejected) + self.assertIn(".agents/skills/helper/reference.md", rejected) + dest = self.root / "destination" + candidate = { + **tree, + "sub_agents/helper/AGENT.md": "description: Help", + **dict(CODEX_SURFACE.stubs), + } + result = sub.export_tree(candidate, dest, base={}) + self.assertEqual(sub.import_tree(dest), tree) + self.assertFalse((dest / "sub_agents").exists()) + self.assertFalse(any(dest.rglob("README.md"))) + self.assertEqual(set(result["written"]), set(tree)) + self.assertEqual(sub.export_tree(tree, dest, base=tree), {"written": [], "deleted": []}) + result = sub.export_tree({}, self.root, base=tree) + self.assertEqual(set(result["deleted"]), set(tree)) + self.assertTrue((self.root / ".codex/config.toml").is_file()) + self.assertTrue((self.root / ".agents/skills/helper/reference.md").is_file()) + + def test_strip_native_preserves_unmanaged_files_and_settings(self): + for sub in (ClaudeSubstrate(), CodexSubstrate()): + with self.subTest(substrate=sub.name): + self.native_project(sub) + sub.strip_native(self.root) + self.assertEqual(sub.import_tree(self.root), {}) + surface = sub.surface() + self.assertTrue((self.root / f"{surface.skill_dir}/helper/reference.md").is_file()) + self.assertTrue((self.root / f"{surface.script_dir}/extra.txt").is_file()) + if sub.name == "claude": + self.assertIn( + "permissions", json.loads((self.root / ".claude/settings.json").read_text()) + ) + self.assertTrue((self.root / ".claude/agents/notes.txt").is_file()) + + def test_materialise_and_install_native_layout(self): + for sub, mode in ( + (ClaudeSubstrate(), "claude_root"), + (CodexSubstrate(), "codex_root"), + (CommandSubstrate("true", ("custom/*",)), "command_files"), + ): + with self.subTest(mode=mode): + canonical = self.root / mode + surface = sub.surface() + if mode == "command_files": + tree = {"custom/file.txt": "custom"} + else: + tree = { + surface.always_on[0]: "instructions", + f"{surface.script_dir}/tool.py": "pass", + } + if mode == "claude_root": + tree["sub_agents/helper/AGENT.md"] = "---\ndescription: Help\n---\nBody" + tree[".claude/hooks.json"] = '{"hooks":{"Stop":[]}}' + HT.write_tree(surface, tree, canonical) + (canonical / "unmanaged.txt").write_text("do not stage") + install = sub.materialise(canonical, SimpleNamespace(install_mode=mode), {}) + self.addCleanup(shutil.rmtree, install.staging_dir, True) + self.assertTrue(install.fingerprint) + self.assertFalse((install.staging_dir / "unmanaged.txt").exists()) + workspace = self.root / f"{mode}-workspace" + if mode == "claude_root": + (workspace / ".claude").mkdir(parents=True) + (workspace / ".claude/settings.json").write_text('{"model":"keep"}') + sub.install_into(install, workspace) + imported = sub.import_tree(workspace) + self.assertEqual(set(imported), set(tree)) + if mode == "claude_root": + self.assertIn("name: helper", imported["sub_agents/helper/AGENT.md"]) + self.assertEqual( + json.loads((workspace / ".claude/settings.json").read_text())["model"], + "keep", + ) + else: + self.assertEqual(imported, tree) + with self.assertRaises(ValueError): + sub.materialise(canonical, SimpleNamespace(install_mode="wrong"), {}) + + def test_native_exports_reject_unmanaged_paths_and_symlinks_before_writes(self): + external = self.write("external.txt", "keep") + (self.root / "CLAUDE.md").symlink_to(external) + with self.assertRaisesRegex(ConfigError, "symlink"): + ClaudeSubstrate().export_tree({"CLAUDE.md": "overwrite"}, self.root, base={}) + self.assertEqual(external.read_text(), "keep") + with self.assertRaisesRegex(ConfigError, "symlink"): + ClaudeSubstrate().import_tree(self.root) + (self.root / "CLAUDE.md").unlink() + (self.root / ".agents").symlink_to(self.root, target_is_directory=True) + with self.assertRaisesRegex(ConfigError, "symlink"): + CodexSubstrate().export_tree( + {"AGENTS.md": "new", ".agents/scripts/tool.py": "pass"}, self.root, base={} + ) + self.assertFalse((self.root / "AGENTS.md").exists()) + for sub in (ClaudeSubstrate(), CodexSubstrate(), CommandSubstrate("true", ("*.md",))): + with self.subTest(substrate=sub.name): + with self.assertRaises(ValueError): + sub.export_tree({"unmanaged.txt": "bad"}, self.root, base={}) + with self.assertRaises(ValueError): + sub.export_tree({"../escape.md": "bad"}, self.root, base={}) + + def test_malformed_settings_and_hooks_never_overwrite_data(self): + settings = self.write(".claude/settings.json", '{"broken":') + sub = ClaudeSubstrate() + for text in ('{"broken":', "[]"): + settings.write_text(text) + with self.assertRaises(ConfigError): + sub.export_tree( + {"CLAUDE.md": "new", ".claude/hooks.json": '{"hooks":{}}'}, self.root, base={} + ) + self.assertEqual(settings.read_text(), text) + self.assertFalse((self.root / "CLAUDE.md").exists()) + settings.write_text('{"hooks":[]}') + with self.assertRaises(ConfigError): + sub.import_tree(self.root) + for content in ("bad", "[]", '{"hooks":[]}'): + with self.assertRaises(ConfigError): + sub.export_tree({".claude/hooks.json": content}, self.root, base={}) + + def test_command_native_files_and_run_with_quoted_paths(self): + sub = CommandSubstrate( + 'cat {prompt_file}; printf "diagnostic" >&2; ' + "mkdir -p {output}; cp {prompt_file} {output}/answer.txt; " + 'printf "%s" {workspace} > {output}/workspace.txt', + ("custom/*",), + ) + self.write("custom/rules.txt", "rules") + self.write("keep.txt", "keep") + base = sub.import_tree(self.root) + self.assertEqual(base, {"custom/rules.txt": "rules"}) + sub.strip_native(self.root) + self.assertTrue((self.root / "keep.txt").exists()) + self.assertFalse((self.root / "custom/rules.txt").exists()) + sub.export_tree(base, self.root, base={}) + prompt = self.write("prompt ' $file.md", "hello") + workspace = self.root / "work ' $dir" + workspace.mkdir() + run_dir = self.root / "run" + self.assertEqual(sub.run(prompt, workspace, run_dir, 5, os.environ), 0) + self.assertEqual((workspace / "output/answer.txt").read_text(), "hello") + self.assertEqual((workspace / "output/workspace.txt").read_text(), str(workspace)) + stream = run_dir / "stream.jsonl" + self.assertEqual(sub.final_message(stream), "hellodiagnostic") + self.assertEqual( + sub.read_trajectory(stream, obs_trunc=10, max_chars=100), "[OBSERVATION] diagnostic" + ) + self.assertEqual(sub.cost_usd(stream), 0.0) + self.assertEqual( + CommandSubstrate("exit 7", ("*.md",)).run(prompt, workspace, run_dir, 5, os.environ), 7 + ) + with self.assertRaises(subprocess.TimeoutExpired): + CommandSubstrate("sleep 2", ("*.md",)).run(prompt, workspace, run_dir, 0.01, os.environ) + + +class ClaudeStreamTests(unittest.TestCase): + fixture = Path(__file__).parent / "fixtures/claude-stream.jsonl" + + def test_documented_message_and_result_fields(self): + sub = ClaudeSubstrate() + rendered = sub.read_trajectory(self.fixture, obs_trunc=1000, max_chars=10000) + for text in ( + "[HARNESS] init | model=claude-fixture cwd=/workspace", + "[REASONING] Inspect the input first.", + "[ASSISTANT] I will inspect the input.", + '[ACTION] Bash: {"command": "cat input.txt"}', + "[OBSERVATION] input contents", + "[OBSERVATION ERROR] File does not exist.", + ): + self.assertIn(text, rendered) + self.assertEqual(sub.final_message(self.fixture), "Task complete.") + self.assertAlmostEqual(sub.cost_usd(self.fixture), 0.0123) + clipped = sub.read_trajectory(self.fixture, obs_trunc=5, max_chars=10000, args_trunc=5) + self.assertIn("…(+", clipped) + self.assertIn("render truncation)", clipped) + capped = sub.read_trajectory(self.fixture, obs_trunc=1000, max_chars=25) + self.assertTrue(capped.startswith(rendered[:25])) + self.assertIn("render truncation)", capped) + + def test_errors_fallback_junk_and_missing_capture(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "stream.jsonl" + path.write_text( + 'junk\n[]\n{"type":\n' + + json.dumps( + { + "type": "assistant", + "message": {"content": [{"type": "text", "text": "fallback"}]}, + } + ) + + "\n" + + json.dumps( + { + "type": "result", + "subtype": "error_during_execution", + "is_error": True, + "errors": ["failed to execute"], + "total_cost_usd": 0.1, + } + ) + + "\n" + ) + sub = ClaudeSubstrate() + self.assertEqual(len(parse_events(path)), 2) + self.assertEqual(sub.final_message(path), "fallback") + self.assertIn( + "[ERROR] failed to execute", + sub.read_trajectory(path, obs_trunc=100, max_chars=1000), + ) + path.write_text("junk\n{}\n") + self.assertEqual(sub.final_message(path), "") + self.assertEqual(sub.cost_usd(path), 0.0) + self.assertEqual( + sub.read_trajectory(path, obs_trunc=100, max_chars=1000), + "(no parseable events in trajectory)", + ) + path.unlink() + self.assertIn( + "failed to read trajectory", + sub.read_trajectory(path, obs_trunc=100, max_chars=1000), + ) + self.assertEqual(sub.final_message(path), "") + self.assertEqual(sub.cost_usd(path), 0.0) + + +class CliTests(unittest.TestCase): + def setUp(self): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + self.root = Path(temp.name) + self.capture = self.root / "capture.json" + script = ( + "#!" + + sys.executable + + "\n" + + """import json, os, sys +from pathlib import Path +if '--version' in sys.argv: + print('fixture 1.0') + raise SystemExit(int(os.environ.get('VERSION_EXIT', '0'))) +Path(os.environ['CAPTURE']).write_text(json.dumps({'argv': sys.argv[1:], 'env': dict(os.environ), 'stdin': sys.stdin.read(), 'cwd': os.getcwd()})) +print('{"type":"result","result":"ok","total_cost_usd":0.01}') +""" + ) + for name in ("claude", "codex", "opencode", "wrapper cli"): + binary = self.root / name + binary.write_text(script) + binary.chmod(0o755) + self.patch = patch.dict( + os.environ, + { + "PATH": str(self.root) + os.pathsep + os.environ["PATH"], + "CAPTURE": str(self.capture), + }, + ) + self.patch.start() + self.addCleanup(self.patch.stop) + + def test_argv_environment_and_stdin_on_fake_clis(self): + base = {**os.environ, **dict.fromkeys(providers.STRIP_ENV, "stale")} + provider = providers.Provider("test", "anthropic", base_url="https://example.test") + prompt = self.root / "prompt.md" + prompt.write_text("task input") + for sub in (ClaudeSubstrate(), CodexSubstrate()): + with self.subTest(substrate=sub.name): + if sub.name == "claude": + home = self.root / "claude-home" + env = sub.env(base, provider, home) + self.assertTrue(home.is_dir()) + self.assertEqual(env["CLAUDE_CONFIG_DIR"], str(home)) + self.assertEqual(env["ANTHROPIC_BASE_URL"], provider.base_url) + self.assertTrue( + all(k not in env for k in providers.STRIP_ENV if k != "ANTHROPIC_BASE_URL") + ) + else: + home = self.root / "codex-home" + codex_provider = providers.Provider( + "test", "openai", base_url="https://example.test/v1", api_key_env="TEST_KEY" + ) + sub.write_home(home, codex_provider, "test-model", "high") + self.assertEqual( + (home / "config.toml").read_text(), + providers.codex_config(codex_provider, "test-model", "high"), + ) + env = sub.env(base, home) + self.assertEqual(env["CODEX_HOME"], str(home)) + self.assertTrue(all(k not in env for k in providers.STRIP_ENV)) + self.assertEqual(sub.exec_env(base, codex_home=home), env) + self.assertEqual(sub.cost_usd(prompt), 0.0) + argv = sub.argv(prompt, self.root, "test-model", "high") + with prompt.open() as stdin: + subprocess.run( + argv, cwd=self.root, env=env, stdin=stdin, check=True, capture_output=True + ) + capture = json.loads(self.capture.read_text()) + self.assertEqual(capture["stdin"], "task input") + self.assertEqual(Path(capture["cwd"]), self.root.resolve()) + self.assertEqual(capture["env"]["PATH"], base["PATH"]) + if sub.name == "claude": + self.assertEqual( + capture["argv"], + [ + "-p", + "--output-format", + "stream-json", + "--verbose", + "--dangerously-skip-permissions", + "--setting-sources", + "project,local", + "--model", + "test-model", + "--effort", + "high", + ], + ) + else: + self.assertEqual( + capture["argv"], + [ + "exec", + "--json", + "--skip-git-repo-check", + "--dangerously-bypass-approvals-and-sandbox", + "--dangerously-bypass-hook-trust", + "--ephemeral", + "-C", + str(self.root), + "-", + ], + ) + + def test_wrapped_binary_overrides_legacy_argv_and_preflight(self): + wrapper = shlex.quote(str(self.root / "wrapper cli")) + " --wrapper" + for cls in (ClaudeSubstrate, CodexSubstrate, OpencodeSubstrate): + sub = cls(bin=wrapper) + sub.preflight() + with patch.dict(os.environ, {"VERSION_EXIT": "2"}): + with self.assertRaisesRegex(ConfigError, "version check failed"): + sub.preflight() + with self.assertRaisesRegex(ConfigError, "not found"): + cls(bin="nonexistent-harness2-binary").preflight() + cx = CodexSubstrate(bin="codex") + argv = cx.exec_argv("task", self.root, "model", reasoning_effort="low", bin=wrapper) + self.assertEqual(argv[:3], [str(self.root / "wrapper cli"), "--wrapper", "exec"]) + self.assertIn('model_reasoning_effort="low"', argv) + oc = OpencodeSubstrate(bin=wrapper) + argv = oc.run_argv( + "task", + self.root, + "test/model", + title="title", + agent="helper", + variant="high", + session="session", + ) + self.assertEqual( + argv[:5], [str(self.root / "wrapper cli"), "--wrapper", "run", "task", "--dir"] + ) + self.assertEqual( + argv[-6:], ["--agent", "helper", "--variant", "high", "--session", "session"] + ) + self.assertEqual( + oc.run_argv("task", self.root, "test/model", bin="opencode")[0], + str(self.root / "opencode"), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_providers.py b/tests/test_providers.py new file mode 100644 index 00000000..a58f67f7 --- /dev/null +++ b/tests/test_providers.py @@ -0,0 +1,329 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Provider translations without network or model calls.""" + +from __future__ import annotations + +import os +import tomllib +import unittest +from dataclasses import FrozenInstanceError +from unittest.mock import patch + +from harness2 import providers +from harness2.providers import Provider +from harness2.settings import ConfigError + + +class ProviderTests(unittest.TestCase): + def test_model_split(self): + self.assertEqual(providers.split_model("anthropic/claude-x"), ("anthropic", "claude-x")) + self.assertEqual(providers.split_model("local/team/model"), ("local", "team/model")) + for ref in ("bare", "", "/model", "provider/"): + with self.subTest(ref=ref), self.assertRaises(ConfigError) as caught: + providers.split_model(ref) + self.assertEqual( + str(caught.exception), + (f"model {ref!r} must be written provider/model, e.g. anthropic/claude-sonnet-5-5"), + ) + + def test_provider_is_frozen_and_strip_env_is_exact(self): + provider = Provider("custom", "anthropic") + with self.assertRaises(FrozenInstanceError): + provider.region = "changed" + self.assertEqual( + providers.PROVIDER_TYPES, + ("anthropic", "openai", "vertex", "bedrock", "claude-login", "chatgpt-login"), + ) + self.assertEqual( + providers.STRIP_ENV, + ( + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_BASE_URL", + "ANTHROPIC_MODEL", + "ANTHROPIC_BEDROCK_BASE_URL", + "ANTHROPIC_VERTEX_PROJECT_ID", + "CLAUDE_CODE_USE_BEDROCK", + "CLAUDE_CODE_USE_VERTEX", + "CLOUD_ML_REGION", + "OPENAI_BASE_URL", + ), + ) + + def test_claude_anthropic_env_and_optional_values(self): + with patch.dict(os.environ, {"CUSTOM_KEY": "secret"}, clear=True): + self.assertEqual( + providers.claude_env( + Provider( + "custom", + "anthropic", + "https://example.com", + "CUSTOM_KEY", + ) + ), + {"ANTHROPIC_API_KEY": "secret", "ANTHROPIC_BASE_URL": "https://example.com"}, + ) + self.assertEqual(providers.claude_env(Provider("a", "anthropic")), {}) + self.assertEqual( + providers.claude_env(Provider("a", "anthropic", api_key_env="MISSING")), + { + "ANTHROPIC_API_KEY": "", + }, + ) + + def test_claude_vertex_and_bedrock_env(self): + self.assertEqual( + providers.claude_env(Provider("v", "vertex", project="project")), + { + "CLAUDE_CODE_USE_VERTEX": "1", + "ANTHROPIC_VERTEX_PROJECT_ID": "project", + "CLOUD_ML_REGION": "global", + }, + ) + self.assertEqual( + providers.claude_env( + Provider( + "v", + "vertex", + project="project", + region="us-east1", + ) + )["CLOUD_ML_REGION"], + "us-east1", + ) + self.assertEqual( + providers.claude_env(Provider("b", "bedrock")), + { + "CLAUDE_CODE_USE_BEDROCK": "1", + }, + ) + self.assertEqual( + providers.claude_env(Provider("b", "bedrock", region="us-east-1")), + { + "CLAUDE_CODE_USE_BEDROCK": "1", + "AWS_REGION": "us-east-1", + }, + ) + with self.assertRaisesRegex( + ConfigError, "Claude Code needs an Anthropic-compatible provider" + ): + providers.claude_env(Provider("o", "openai")) + + def test_codex_config_is_valid_toml_and_escapes_values(self): + provider = Provider('local."name 😀', "openai", "http://localhost:8000/v1", "LOCAL_KEY") + parsed = tomllib.loads(providers.codex_config(provider, 'model"\nnext 😀\x7f', "high")) + self.assertEqual( + parsed, + { + "model": 'model"\nnext 😀\x7f', + "model_provider": f"harness2-{provider.name}", + "model_reasoning_effort": "high", + "model_providers": { + f"harness2-{provider.name}": { + "name": provider.name, + "base_url": provider.base_url, + "env_key": "LOCAL_KEY", + "wire_api": "responses", + } + }, + }, + ) + for kind in ("anthropic", "vertex", "bedrock"): + with self.subTest(kind=kind), self.assertRaises(ConfigError) as caught: + providers.codex_config(Provider("p", kind), "model", "high") + self.assertEqual( + str(caught.exception), + ( + "Codex needs an OpenAI Responses-compatible provider; serve one with " + "`harness2 bridge --model /` and point an openai-type " + "provider at it." + ), + ) + + def test_opencode_openai_and_anthropic(self): + for kind, npm in ( + ("openai", "@ai-sdk/openai-compatible"), + ("anthropic", "@ai-sdk/anthropic"), + ): + with self.subTest(kind=kind): + provider = Provider("local", kind, "http://localhost:8000", "MY_KEY") + self.assertEqual( + providers.opencode_config(provider, "model"), + ( + { + "provider": { + "local": { + "npm": npm, + "name": "local", + "options": { + "baseURL": "http://localhost:8000", + "apiKey": "{env:MY_KEY}", + }, + "models": {"model": {}}, + } + } + }, + {}, + ), + ) + self.assertEqual( + providers.opencode_config(Provider("a", "anthropic"), "model")[0], + { + "provider": { + "a": { + "npm": "@ai-sdk/anthropic", + "name": "a", + "options": {}, + "models": {"model": {}}, + } + }, + }, + ) + self.assertEqual( + providers.opencode_config(Provider("o", "openai"), "model")[0]["provider"]["o"][ + "options" + ], + {"baseURL": "", "apiKey": "{env:}"}, + ) + + def test_opencode_vertex_and_bedrock(self): + for region in ("", "europe-west1"): + with self.subTest(region=region): + self.assertEqual( + providers.opencode_config( + Provider( + "v", + "vertex", + project="project", + region=region, + ), + "model", + ), + ( + {}, + { + "GOOGLE_VERTEX_PROJECT": "project", + "GOOGLE_VERTEX_LOCATION": region or "global", + "VERTEXAI_LOCATION": region or "global", + }, + ), + ) + with self.assertRaisesRegex( + ConfigError, "Bedrock is not supported on the opencode channel" + ): + providers.opencode_config(Provider("b", "bedrock"), "model") + + def test_litellm_params_for_each_type(self): + for kind in ("anthropic", "openai"): + with self.subTest(kind=kind): + self.assertEqual( + providers.litellm_params( + Provider( + "local", + kind, + "https://example.com", + "CUSTOM_KEY", + ), + "model", + ), + { + "model": f"{kind}/model", + "api_key": "os.environ/CUSTOM_KEY", + "api_base": "https://example.com", + }, + ) + self.assertEqual( + providers.litellm_params(Provider("local", kind), "model"), + { + "model": f"{kind}/model", + "api_key": "os.environ/", + **({"api_base": ""} if kind == "openai" else {}), + }, + ) + self.assertEqual( + providers.litellm_params( + Provider( + "v", + "vertex", + project="project", + region="us-east1", + ), + "model", + ), + { + "model": "vertex_ai/model", + "vertex_project": "project", + "vertex_location": "us-east1", + }, + ) + self.assertEqual( + providers.litellm_params(Provider("v", "vertex"), "model")["vertex_location"], "global" + ) + self.assertEqual( + providers.litellm_params(Provider("b", "bedrock", region="us-west-2"), "model"), + { + "model": "bedrock/model", + "aws_region_name": "us-west-2", + }, + ) + + def test_unknown_provider_types_raise_config_error(self): + for function, args in ( + (providers.claude_env, ()), + (providers.codex_config, ("model", "high")), + (providers.opencode_config, ("model",)), + (providers.litellm_params, ("model",)), + ): + with self.subTest(function=function.__name__), self.assertRaises(ConfigError): + function(Provider("p", "unknown"), *args) + + def test_doctor_checks_missing_env_project_and_invalid_urls(self): + with patch.dict(os.environ, {}, clear=True): + provider = Provider("v", "vertex", "ftp://example.com", "CUSTOM_KEY") + self.assertEqual( + providers.check(provider), + [ + "v: set CUSTOM_KEY", + "v: set project", + "v: base_url must be an HTTP(S) URL", + ], + ) + os.environ["CUSTOM_KEY"] = "" + self.assertEqual( + providers.check(Provider("a", "anthropic", api_key_env="CUSTOM_KEY")), + [ + "a: set CUSTOM_KEY", + ], + ) + os.environ["CUSTOM_KEY"] = "present" + self.assertEqual( + providers.check(Provider("a", "anthropic", api_key_env="CUSTOM_KEY")), [] + ) + for url in ("https://example.com/v1", "http://localhost:8080", ""): + with self.subTest(url=url): + self.assertEqual(providers.check(Provider("o", "openai", url)), []) + for url in ("file:///tmp/endpoint", "localhost:8080", "https://", "http://[bad"): + with self.subTest(url=url): + self.assertEqual( + providers.check(Provider("o", "openai", url)), + [ + "o: base_url must be an HTTP(S) URL", + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_registry.py b/tests/test_registry.py new file mode 100644 index 00000000..71dfd1c0 --- /dev/null +++ b/tests/test_registry.py @@ -0,0 +1,84 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Suite discovery with isolated, fake entry points.""" + +from __future__ import annotations + +import unittest +from dataclasses import FrozenInstanceError +from importlib.metadata import EntryPoint +from pathlib import Path +from types import ModuleType +from unittest.mock import patch + +from harness2 import registry +from harness2.settings import ConfigError + + +class RegistryTests(unittest.TestCase): + def test_builtin_is_listed_first_and_imported_lazily(self): + entries = [ + EntryPoint("zeta", "fake.zeta", "harness2.suites"), + EntryPoint("alpha", "fake.alpha", "harness2.suites"), + EntryPoint("tasks", "fake.tasks", "harness2.suites"), + ] + module = ModuleType("harness2.suite") + with ( + patch("importlib.metadata.entry_points", return_value=entries) as discover, + patch("importlib.import_module", return_value=module) as import_module, + ): + self.assertEqual(registry.names(), ["tasks", "alpha", "zeta"]) + import_module.assert_not_called() + self.assertIs(registry.load("tasks"), module) + import_module.assert_called_once_with("harness2.suite") + discover.assert_called_once_with(group="harness2.suites") + + def test_plugin_module_is_loaded_through_entry_point(self): + module = ModuleType("fake_suite") + module.make_adapter = lambda ctx: ctx.project + entry = EntryPoint("custom", "fake_suite", "harness2.suites") + with ( + patch("importlib.metadata.entry_points", return_value=[entry]) as discover, + patch.dict("sys.modules", {"fake_suite": module}), + ): + self.assertEqual(registry.names(), ["tasks", "custom"]) + self.assertIs(registry.load("custom"), module) + discover.assert_called_with(group="harness2.suites") + context = registry.SuiteContext(None, Path("/project"), Path("/project/runs/name")) + self.assertEqual(module.make_adapter(context), Path("/project")) + + def test_unknown_suite_lists_available_names(self): + entry = EntryPoint("example", "fake.example", "harness2.suites") + with patch("importlib.metadata.entry_points", return_value=[entry]): + with self.assertRaisesRegex( + ConfigError, "unknown suite 'missing'; available suites: tasks, example" + ): + registry.load("missing") + with patch("importlib.metadata.entry_points", return_value=[]): + self.assertEqual(registry.names(), ["tasks"]) + + def test_suite_context_is_frozen_with_optional_split_file(self): + context = registry.SuiteContext(None, Path("/project"), Path("/runs")) + self.assertIsNone(context.split_file) + with self.assertRaises(FrozenInstanceError): + context.project = Path("/changed") + split = Path("/split.json") + self.assertEqual( + registry.SuiteContext(None, Path("/p"), Path("/r"), split).split_file, split + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_runtime.py b/tests/test_runtime.py new file mode 100644 index 00000000..da0d86bc --- /dev/null +++ b/tests/test_runtime.py @@ -0,0 +1,620 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline runtime contracts: channels, accounting, stop/resume, and relocation.""" + +from __future__ import annotations + +import json +import os +import shutil +import sys +import tempfile +import threading +import tomllib +import unittest +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from unittest.mock import Mock, patch + +from harness2.core import configure, events, judge, recursion, runmark, schedule +from harness2.core.proposer import SYSTEM_ARG_MAX, Proposer, ProposerError, _call_fingerprint +from harness2.core.spec import CHANNELS, ChannelSpec +from harness2.demo import DemoAdapter, DemoProposer +from harness2.providers import STRIP_ENV, Provider +from harness2.substrate.codex import parse_items, parse_items_text + + +class EventsTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.path = Path(temporary.name) / "nested/events.jsonl" + self.addCleanup(events.start, None) + events.start(None) + + def test_disabled_until_started_and_request_stop(self): + with patch.object(events, "_active", False): + events.emit("session", cost_usd=2) + events.check() + self.assertEqual(events.totals()["calls"], 0) + events.request_stop("requested") + with self.assertRaisesRegex(events.Stopped, "requested"): + events.check() + events.start(None) + events.check() + self.assertIsNone(events.totals()["stopped"]) + + def test_budget_resume_and_cached_sessions(self): + events.start(self.path, budget_usd=0.5, max_calls=9) + events.emit("task_start", task="one") + events.emit("session", cost_usd=0.2, cached=False) + events.emit("session", cost_usd=0, cached=True) + events.emit("rollout", cost_usd=0.3) + self.assertEqual( + events.totals(), + { + "spent_usd": 0.5, + "calls": 2, + "budget_usd": 0.5, + "max_calls": 9, + "stopped": "budget_usd", + }, + ) + with self.assertRaisesRegex(events.Stopped, "budget_usd"): + events.check() + events.start(self.path, max_calls=3) + events.check() + events.emit("session") + with self.assertRaisesRegex(events.Stopped, "max_calls"): + events.check() + records = [json.loads(line) for line in self.path.read_text().splitlines()] + self.assertEqual(len(records), 5) + self.assertTrue(all(record["t"].endswith("Z") for record in records)) + for kwargs in ({"budget_usd": -1}, {"max_calls": -1}): + with self.assertRaises(ValueError): + events.start(None, **kwargs) + + def test_concurrent_events_are_whole_lines_and_counted_once(self): + events.start(self.path) + with ThreadPoolExecutor(max_workers=4) as pool: + list(pool.map(lambda i: events.emit("rollout", task=str(i), cost_usd=0.1), range(40))) + self.assertEqual(len([json.loads(line) for line in self.path.read_text().splitlines()]), 40) + self.assertEqual((events.totals()["calls"], events.totals()["spent_usd"]), (40, 4)) + + +FAKE_CLI = r"""import json +import os +import sys +from pathlib import Path + +cwd = Path.cwd() +kind = Path(sys.argv[0]).name +record = {"argv": sys.argv[1:], "env": dict(os.environ), + "stdin": sys.stdin.read() if kind != "opencode" else "", + "cwd": str(cwd)} +for key, name in (("CODEX_HOME", "config.toml"), ("CLAUDE_CONFIG_DIR", "")): + if key in os.environ: + home = Path(os.environ[key]) + record[key + "_exists"] = home.is_dir() + if name: + record["config"] = (home / name).read_text() +if (cwd / "opencode.json").is_file(): + record["config"] = json.loads((cwd / "opencode.json").read_text()) + record["mission"] = (cwd / "MISSION.md").read_text() +with open(os.environ["FAKE_RECORD"], "a") as output: + output.write(json.dumps(record) + "\n") +if "FAKE_OUTPUT" in os.environ: + print(os.environ["FAKE_OUTPUT"]) +elif kind == "claude": + print(json.dumps({"result": "done", "total_cost_usd": 0.25, "subtype": "success"})) +elif kind == "codex": + print(json.dumps({"type": "item.completed", "item": { + "id": "first", "type": "agent_message", "text": "earlier"}})) + print(json.dumps({"type": "item.completed", "item": { + "id": "last", "type": "agent_message", "text": "done"}})) +else: + print(json.dumps({"type": "text", "part": {"text": "done"}})) + print(json.dumps({"type": "step_finish", "part": {"cost": 0.125}})) +sys.exit(int(os.environ.get("FAKE_RC", "0"))) +""" + + +class ProposerTests(unittest.TestCase): + maxDiff = None + + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + binary = self.root / "bin" + binary.mkdir() + for kind in ("claude", "codex", "opencode"): + path = binary / kind + path.write_text(f"#!{sys.executable}\n" + FAKE_CLI) + path.chmod(0o755) + self.record = self.root / "record.jsonl" + self.base_env = { + "PATH": f"{binary}:{os.defpath}", + "FAKE_RECORD": str(self.record), + "CUSTOM_KEY": "fake-key", + "LC_ALL": "C", + } + self.environment = patch.dict( + os.environ, self.base_env | dict.fromkeys(STRIP_ENV, "stale"), clear=True + ) + self.environment.start() + self.addCleanup(self.environment.stop) + self.addCleanup(events.start, None) + events.start(None) + + def records(self): + return [json.loads(line) for line in self.record.read_text().splitlines()] + + def test_channels_alias_empty_and_human(self): + self.assertEqual(CHANNELS, ("claude", "codex", "opencode", "human")) + self.assertEqual(ChannelSpec("claude_cli", "local/model").kind, "claude") + with self.assertRaises(ValueError): + ChannelSpec("missing", "model") + with self.assertRaises(ValueError): + Proposer(ChannelSpec("claude", "")) + human = Proposer(ChannelSpec("human", "")) + with self.assertRaisesRegex(ProposerError, "interactive selector"): + human.call("choose", None, "one") + self.assertFalse(self.record.exists()) + + def test_channel_argv_env_and_provider_config(self): + providers = { + "claude": Provider("local", "anthropic", "https://claude.invalid", "CUSTOM_KEY"), + "codex": Provider("local", "openai", "https://responses.invalid/v1", "CUSTOM_KEY"), + "opencode": Provider("local", "openai", "https://chat.invalid/v1", "CUSTOM_KEY"), + } + for kind, provider in providers.items(): + with self.subTest(kind=kind): + prop = Proposer(ChannelSpec(kind, "local/test-model", provider=provider)) + reply = prop.call("mission", None, kind, system="system") + self.assertEqual(reply.text, "done") + self.assertEqual( + reply.cost_usd, {"claude": 0.25, "codex": 0, "opencode": 0.125}[kind] + ) + record = self.records()[-1] + cwd = record["cwd"] + expected_env = dict(self.base_env) + if kind == "claude": + self.assertEqual( + record["argv"], + [ + "--dangerously-skip-permissions", + "-p", + "--output-format", + "json", + "--effort", + "high", + "--model", + "test-model", + "--setting-sources", + "project,local", + "--append-system-prompt", + "system", + ], + ) + expected_env |= { + "ANTHROPIC_API_KEY": "fake-key", + "ANTHROPIC_BASE_URL": provider.base_url, + "CLAUDE_CONFIG_DIR": cwd + "/.claude-home", + } + self.assertTrue(record["CLAUDE_CONFIG_DIR_exists"]) + self.assertEqual(record["stdin"], "mission") + elif kind == "codex": + self.assertEqual( + record["argv"], + [ + "exec", + "--json", + "--skip-git-repo-check", + "--dangerously-bypass-approvals-and-sandbox", + "--ephemeral", + "-C", + cwd, + "-", + ], + ) + expected_env["CODEX_HOME"] = cwd + "/.codex-home" + self.assertTrue(record["CODEX_HOME_exists"]) + self.assertEqual( + record["stdin"], "# SYSTEM INSTRUCTIONS\n\nsystem\n\n# END\n\nmission" + ) + config = tomllib.loads(record["config"]) + self.assertEqual(config["model"], "test-model") + self.assertEqual(config["model_provider"], "harness2-local") + self.assertEqual( + config["model_providers"]["harness2-local"]["env_key"], "CUSTOM_KEY" + ) + else: + self.assertEqual( + record["argv"], + [ + "run", + "Read MISSION.md first. Do what it asks. " + "Reply with the requested output as your final message AND save it verbatim to output/answer.md.", + "--dir", + cwd, + "--model", + "local/test-model", + "--format", + "json", + "--variant", + "high", + ], + ) + expected_env |= { + "OPENCODE_PERMISSION": json.dumps( + { + "external_directory": { + "*": "deny", + cwd: "allow", + cwd + "/**": "allow", + } + } + ), + "XDG_DATA_HOME": cwd + "/.ocdata", + "XDG_STATE_HOME": cwd + "/.ocstate", + } + self.assertEqual(record["stdin"], "") + self.assertEqual(record["mission"], "system\n\n---\n\nmission") + self.assertEqual(record["config"]["agent"]["build"]["steps"], 120) + self.assertEqual( + record["config"]["provider"]["local"]["models"], {"test-model": {}} + ) + observed_env = dict(record["env"]) + # macOS injects this into the child after subprocess receives its environment. + observed_env.pop("__CF_USER_TEXT_ENCODING", None) + self.assertEqual(observed_env, expected_env) + normalized = json.dumps( + {"argv": [kind, *record["argv"]], "env": record["env"]}, sort_keys=True + ) + normalized = normalized.replace(cwd, "").replace(str(self.root), "") + print(f"PROPOSER {kind}: {normalized}") + + def test_no_provider_custom_binary_long_system_and_codex_config(self): + system = "s" * (SYSTEM_ARG_MAX + 1) + binary = f'"{self.root}/bin/claude" --wrapper' + Proposer(ChannelSpec("claude", "bare-model", bin=binary)).call( + "mission", None, "long", system=system + ) + record = self.records()[-1] + self.assertEqual(record["argv"][0], "--wrapper") + self.assertEqual(record["stdin"], f"# SYSTEM INSTRUCTIONS\n\n{system}\n\n# END\n\nmission") + self.assertFalse(any(key in record["env"] for key in STRIP_ENV)) + Proposer(ChannelSpec("codex", "bare-model")).call("mission", None, "plain") + self.assertEqual( + tomllib.loads(self.records()[-1]["config"]), + {"model": "bare-model", "model_reasoning_effort": "high"}, + ) + + def test_vertex_env_is_explicit(self): + provider = Provider("vertex", "vertex", project="test-project", region="test-region") + prop = Proposer(ChannelSpec("opencode", "google-vertex/test-model", provider=provider)) + prop.call("mission", None, "vertex") + record = self.records()[-1] + self.assertEqual(record["env"]["GOOGLE_VERTEX_PROJECT"], "test-project") + self.assertEqual(record["env"]["GOOGLE_VERTEX_LOCATION"], "test-region") + self.assertEqual(record["env"]["VERTEXAI_LOCATION"], "test-region") + self.assertNotIn("provider", record["config"]) + + def test_codex_failure_events_and_returncode_emit_failed_sessions(self): + for output, rc in ( + (json.dumps({"type": "turn.failed", "error": {"message": "failed"}}), "0"), + (json.dumps({"type": "error", "message": "bad"}), "0"), + ( + json.dumps( + {"type": "item.completed", "item": {"type": "agent_message", "text": "partial"}} + ), + "2", + ), + ): + with ( + self.subTest(output=output, rc=rc), + patch.dict(os.environ, {"FAKE_OUTPUT": output, "FAKE_RC": rc}), + ): + events.start(None) + prop = Proposer(ChannelSpec("codex", "local/model"), attempts=1) + with self.assertRaises(ProposerError): + prop.call("mission", None, "failure") + self.assertEqual(events.totals()["calls"], 1) + self.assertFalse(prop._calls[-1].ok) + + def test_cache_budget_harvest_and_fingerprint_relocation(self): + context = self.root / "run/context/task" + (context / "harness").mkdir(parents=True) + (context / "harness/CLAUDE.md").write_text("base") + audit = self.root / "run/audit" + prop = Proposer(ChannelSpec("claude", "local/model"), audit_dir=audit) + events.start(None, max_calls=1) + reply = prop.call("mission", context, "one", harvest="context/harness") + self.assertEqual((reply.workspace / "CLAUDE.md").read_text(), "base") + fingerprint = _call_fingerprint("one", "mission", "", context, audit) + moved = self.root / "moved" + shutil.move(self.root / "run", moved) + context, audit = moved / "context/task", moved / "audit" + self.assertEqual(_call_fingerprint("one", "mission", "", context, audit), fingerprint) + prop = Proposer(prop.spec, audit_dir=audit) + self.assertTrue(prop.call("mission", context, "one", harvest="context/harness").cached) + self.assertEqual((len(self.records()), events.totals()["calls"]), (1, 1)) + with self.assertRaises(events.Stopped): + prop.call("different", context, "two") + shutil.rmtree(prop._calls[-1].workspace) + with self.assertRaises(events.Stopped): + prop.call("mission", context, "one", harvest="context/harness") + + def test_jsonl_text_and_path_share_deduplication(self): + text = ( + "junk\n" + + "\n".join( + json.dumps(record) + for record in [ + { + "type": "item.started", + "item": {"id": "x", "type": "agent_message", "text": "old"}, + }, + {"type": "turn.completed"}, + { + "type": "item.completed", + "item": {"id": "x", "type": "agent_message", "text": "last"}, + }, + ] + ) + + "\n{invalid\n" + ) + path = self.root / "stream.jsonl" + path.write_text(text) + self.assertEqual(parse_items_text(text), parse_items(path)) + self.assertEqual( + [record["type"] for record in parse_items_text(text)], + ["item.completed", "turn.completed"], + ) + self.assertEqual(parse_items_text(text)[0]["item"]["text"], "last") + self.assertEqual(parse_items(self.root / "missing")[0]["type"], "_read_error") + + +class RuntimeWorkflowTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) / "original" + self.adapter = DemoAdapter(self.root) + self.prop = DemoProposer( + self.adapter.spec.channels["proposer"], audit_dir=self.adapter.spec.runs_root / "audit" + ) + self.addCleanup(events.start, None) + self.addCleanup(configure, phase_width=2, transport_attempts=2, proposer_concurrency=2) + events.start(None) + + def sweep(self, k=1, **kwargs): + return schedule.run_sweep( + k, + adapter=self.adapter, + prop=self.prop, + runs_root=self.adapter.spec.runs_root, + library_root=self.root / "library", + **kwargs, + ) + + def test_budget_stop_resume_without_repeated_work(self): + event_path = self.adapter.spec.runs_root / "events.jsonl" + events.start(event_path, max_calls=3) + with self.assertRaisesRegex(events.Stopped, "max_calls"): + self.sweep() + self.assertEqual(events.totals()["calls"], 3) + before = self.adapter.execution_count + self.assertEqual(before, 1) + events.start(event_path, max_calls=100) + result = self.sweep() + self.assertEqual(result["domains"][0]["done"], 2) + records = [json.loads(line) for line in event_path.read_text().splitlines()] + rollouts = [record["run_id"] for record in records if record["kind"] == "rollout"] + self.assertEqual(len(rollouts), len(set(rollouts))) + self.assertEqual(self.adapter.execution_count, 14) + self.assertEqual(events.totals()["calls"], 30) + executions, calls = self.adapter.execution_count, len(self.prop._calls) + self.sweep() + self.assertEqual((self.adapter.execution_count, len(self.prop._calls)), (executions, calls)) + print( + f"BUDGET STOP/RESUME: stopped=3 calls; executions={executions}; " + f"total_calls={events.totals()['calls']}; repeated_rollouts=0" + ) + + def test_relocation_skips_executions_and_proposals_and_resolves_judge_paths(self): + self.sweep(k=2) + moved = self.root.with_name("moved") + shutil.move(self.root, moved) + self.root = moved + self.adapter = DemoAdapter(moved) + self.prop = DemoProposer(self.prop.spec, audit_dir=self.adapter.spec.runs_root / "audit") + self.adapter.score = Mock( + side_effect=lambda rd, task: ( + {"n_passed": 1, "n_criteria": 1} if self.adapter.delivered(rd) else None + ) + ) + result = self.sweep(k=2) + self.assertEqual( + [row["status"] for row in result["domains"][0]["tasks"]], ["skipped", "skipped"] + ) + self.assertEqual((self.adapter.execution_count, len(self.prop._calls)), (0, 0)) + selections = [] + for summary in self.adapter.spec.runs_root.glob("demo-oc-agg2/*/summary.json"): + data = json.loads(summary.read_text()) + for row in data["base"]: + self.assertFalse(Path(row["run_dir"]).is_absolute()) + self.assertTrue( + runmark.resolve_path(row["run_dir"], self.adapter.spec.runs_root).is_dir() + ) + for selection in data["selections"].values(): + self.assertFalse(Path(selection["chosen_run_dir"]).is_absolute()) + selections.append(selection) + for record in self.adapter.spec.runs_root.rglob("selection.json"): + selection = json.loads(record.read_text()) + self.assertFalse(Path(selection["chosen_run_dir"]).is_absolute()) + for record in self.adapter.spec.runs_root.rglob(runmark.MARK): + data = json.loads(record.read_text()) + if "rollout" in data: + self.assertFalse(Path(data["rollout"]["run_dir"]).is_absolute()) + rows = judge.judge_selections(self.adapter, selections) + self.assertTrue(all(row.graded for row in rows)) + absolute = dict(selections[0]) + absolute["chosen_run_dir"] = str( + runmark.resolve_path(absolute["chosen_run_dir"], self.adapter.spec.runs_root) + ) + self.assertTrue(judge.judge_selections(self.adapter, [absolute])[0].graded) + print(f"RELOCATION: executions=0; proposer_calls=0; judged={len(rows)}; relative_paths=ok") + + def test_reflector_prop_and_distinct_selector_are_threaded(self): + reflector = DemoProposer(self.prop.spec) + selector = DemoProposer(self.prop.spec) + self.sweep(k=2, selector_prop=selector, reflector_prop=reflector) + self.assertTrue(selector._calls) + self.assertTrue(all(call.label.startswith("select.") for call in selector._calls)) + self.assertEqual(len(reflector._calls), 4) + self.assertTrue(all(call.label.startswith("experience.") for call in reflector._calls)) + self.assertFalse(any(call.label.startswith("experience.") for call in self.prop._calls)) + spec = ChannelSpec("codex", "local/model") + self.adapter.spec.channels["selector"] = spec + resolved = schedule._selector_for(self.adapter, self.prop, None) + self.assertEqual(resolved.spec, spec) + + def test_configure_reaches_existing_functions_without_default_capture(self): + configure(phase_width=1, transport_attempts=1, proposer_concurrency=1) + self.assertEqual( + (recursion.PHASE_WIDTH, schedule.ROLLOUT_PARALLEL, recursion.TRANSPORT_ATTEMPTS), + (1, 1, 1), + ) + import harness2.core.proposer as proposer_module + + self.assertTrue(proposer_module._SEM.acquire(blocking=False)) + self.assertFalse(proposer_module._SEM.acquire(blocking=False)) + proposer_module._SEM.release() + widths = [] + real_pool = ThreadPoolExecutor + + def pool(*args, **kwargs): + widths.append(kwargs["max_workers"]) + return real_pool(*args, **kwargs) + + with patch.object(recursion, "ThreadPoolExecutor", side_effect=pool): + self.sweep() + self.assertTrue(widths) + self.assertEqual(set(widths), {1}) + for kwargs in ( + {"phase_width": 0}, + {"transport_attempts": -1}, + {"proposer_concurrency": False}, + ): + with self.assertRaises(ValueError): + configure(**kwargs) + + def test_rollout_cost_failures_cache_and_stage_stop(self): + self.adapter.rollout_cost = Mock(return_value=0.5) + ledger = recursion.Ledger("agg", "ledger-1", 1) + work = self.adapter.spec.runs_root / "work" + rv = recursion._roll( + self.adapter, ledger, "ledger-1", "base1", None, lambda label: label, work + ) + self.assertEqual((rv.cost_usd, events.totals()["spent_usd"]), (0.5, 0.5)) + recursion._roll(self.adapter, ledger, "ledger-1", "base1", None, lambda label: label, work) + self.assertEqual((self.adapter.execution_count, events.totals()["calls"]), (1, 1)) + events.request_stop("requested") + with self.assertRaises(events.Stopped): + recursion._run_stage( + label="inv1", + dim="INV", + job="edit", + fallback={}, + adapter=self.adapter, + prop=self.prop, + ctx_dir=work, + task_id="ledger-1", + rollouts=[], + harness_dir=None, + library_dir=None, + ) + events.start(None) + with patch.object(self.adapter, "execute", side_effect=RuntimeError("failed")): + failed = recursion._roll( + self.adapter, ledger, "ledger-1", "failed", None, lambda label: label, work + ) + self.assertFalse(failed.ok) + self.assertIn("failed", failed.error) + with patch.object(self.adapter, "execute", side_effect=events.Stopped("requested")): + with self.assertRaises(events.Stopped): + recursion._roll( + self.adapter, ledger, "ledger-1", "stopped", None, lambda label: label, work + ) + + def test_batch_bases_reuse_cache_and_propagate_stop(self): + self.adapter.execute_many = Mock( + side_effect=lambda task, harness, ids: [ + self.adapter.execute(task, harness, rid) for rid in ids + ] + ) + pending = [("base1", "first"), ("base2", "second")] + work = self.adapter.spec.runs_root / "batch" + got = schedule._roll_bases("ledger-1", self.adapter, pending, work / "missing") + self.assertEqual(set(got), {"base1", "base2"}) + self.assertEqual(events.totals()["calls"], 2) + schedule._roll_bases("ledger-1", self.adapter, pending, work / "missing") + self.adapter.execute_many.assert_called_once() + self.adapter.execute_many.side_effect = events.Stopped("batch stopped") + with self.assertRaises(events.Stopped): + schedule._roll_bases( + "ledger-1", + self.adapter, + [("base3", "third"), ("base4", "fourth")], + work / "missing", + ) + + def test_sweep_collects_other_domains_before_raising_and_interrupt_requests_stop(self): + completed = threading.Event() + + def domain(name, *args, **kwargs): + if name == "first": + raise events.Stopped("limit") + completed.set() + return {"domain": name} + + with ( + patch.object(schedule, "task_stream", return_value={"first": ["a"], "second": ["b"]}), + patch.object(schedule, "run_domain", side_effect=domain), + ): + with self.assertRaisesRegex(events.Stopped, "limit"): + self.sweep(domain_parallel=2) + self.assertTrue(completed.is_set()) + with ( + patch.object(schedule, "run_domain", return_value={}), + patch.object(schedule, "as_completed", side_effect=KeyboardInterrupt), + ): + with self.assertRaises(KeyboardInterrupt): + self.sweep() + self.assertEqual(events.totals()["stopped"], "interrupted") + + def test_relative_and_legacy_absolute_paths(self): + inside = self.root / "runs/one" + outside = self.root / "elsewhere" + self.assertEqual(runmark.relative_path(inside, self.root / "runs"), "one") + self.assertEqual(runmark.relative_path(outside, self.root / "runs"), str(outside.resolve())) + self.assertEqual(runmark.resolve_path("one", self.root / "runs"), inside) + self.assertEqual(runmark.resolve_path(outside, self.root / "runs"), outside) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_settings.py b/tests/test_settings.py new file mode 100644 index 00000000..978c30c0 --- /dev/null +++ b/tests/test_settings.py @@ -0,0 +1,437 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline tests for layered settings and edits to project TOML files.""" + +from __future__ import annotations + +import os +import tempfile +import tomllib +import unittest +from pathlib import Path +from unittest.mock import patch + +from harness2 import providers, settings + + +class SettingsTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.project = self.root / "project" + (self.project / ".harness2").mkdir(parents=True) + self.global_file = self.root / "config/harness2/config.toml" + self.project_file = settings.project_path(self.project) + environment = patch.dict( + os.environ, {"XDG_CONFIG_HOME": str(self.root / "config")}, clear=True + ) + environment.start() + self.addCleanup(environment.stop) + + def write(self, path, text): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + def test_defaults_access_and_copy_isolation(self): + resolved = settings.load(self.project) + for section, values in settings.DEFAULTS.items(): + self.assertEqual(resolved.section(section), values) + self.assertEqual(resolved.project, self.project) + self.assertEqual(resolved.origin("run.k"), "default") + self.assertIsNone(resolved.get("run.seed")) + self.assertEqual(resolved.get("run.absent", 4), 4) + for key in ("absent", "run.absent", "run.k.absent"): + with self.subTest(key=key), self.assertRaises(KeyError): + resolved.get(key) + with self.assertRaises(KeyError): + resolved.origin(key) + self.assertEqual(resolved.flat()["executor.editable"], []) + self.assertNotIn("selector", resolved.flat()) + resolved.section("executor")["editable"].append("one.py") + resolved.flat()["executor.editable"].append("two.py") + self.assertEqual(resolved.get("executor.editable"), []) + self.assertEqual(settings.load(None).section("executor"), settings.DEFAULTS["executor"]) + + def test_state_dir_uses_xdg_or_home_without_creating_it(self): + with patch.dict(os.environ, {"HOME": str(self.root)}, clear=True): + self.assertEqual(settings.state_dir(), self.root / ".local/state/harness2") + self.assertFalse(settings.state_dir().exists()) + os.environ["XDG_STATE_HOME"] = str(self.root / "state") + self.assertEqual(settings.state_dir(), self.root / "state/harness2") + self.assertFalse(settings.state_dir().exists()) + os.environ["XDG_STATE_HOME"] = "" + self.assertEqual(settings.state_dir(), self.root / ".local/state/harness2") + + def test_precedence_and_origins_across_all_six_layers(self): + self.write(self.global_file, "[run]\nk = 2\nworkers = 2\n") + self.write( + self.project_file, + "[run]\nk = 3\nphase_width = 3\n[profiles.fast.run]\nk = 4\n", + ) + layers = [ + (settings.load(None), 2, str(self.global_file)), + (settings.load(self.project), 3, str(self.project_file)), + (settings.load(self.project, profile="fast"), 4, "profile fast"), + ] + os.environ["HARNESS2_RUN__K"] = "5" + layers.append((settings.load(self.project, profile="fast"), 5, "env HARNESS2_RUN__K")) + layers.append( + ( + settings.load( + self.project, + profile="fast", + overrides={"run.k": (6, "flag --k")}, + ), + 6, + "flag --k", + ) + ) + for resolved, value, origin in layers: + with self.subTest(origin=origin): + self.assertEqual(resolved.get("run.k"), value) + self.assertEqual(resolved.origin("run.k"), origin) + resolved = layers[-1][0] + self.assertEqual(resolved.origin("run.workers"), str(self.global_file)) + self.assertEqual(resolved.origin("run.phase_width"), str(self.project_file)) + self.assertEqual(resolved.origin("run.obs_trunc"), "default") + + def test_profiles_merge_files_and_explicit_profile_wins(self): + self.write( + self.global_file, + '[profiles.fast.improver]\nmodel = "anthropic/global"\neffort = "low"\n', + ) + self.write( + self.project_file, + '[profiles.fast.improver]\neffort = "medium"\n[profiles.slow.run]\nk = 7\n', + ) + os.environ["HARNESS2_PROFILE"] = "fast" + resolved = settings.load(self.project) + self.assertEqual(resolved.role("improver")["model"], "anthropic/global") + self.assertEqual(resolved.role("improver")["effort"], "medium") + self.assertEqual(resolved.origin("improver.model"), "profile fast") + self.assertEqual(settings.load(self.project, profile="slow").get("run.k"), 7) + with self.assertRaisesRegex(settings.ConfigError, "unknown profile 'missing'"): + settings.load(self.project, profile="missing") + self.write(self.project_file, '[profiles]\nbad = "invalid"\n') + with self.assertRaisesRegex(settings.ConfigError, "profiles.bad must be a table"): + settings.load(self.project, profile="bad") + + def test_env_values_and_dotted_overrides(self): + os.environ.update( + { + "HARNESS2_EXECUTOR__MODEL": "anthropic/model/submodel", + "HARNESS2_EXECUTOR__EDITABLE": '["a.py", "b.py"]', + "HARNESS2_PROVIDERS__LOCAL__TYPE": "openai", + "HARNESS2_PROVIDERS__LOCAL__BASE_URL": "http://localhost:8080/v1", + "HARNESS2_RUN__K": "3", + "HARNESS2_RUN__PLUGIN_BOOL": "true", + "HARNESS2_OLD_OPTION": "ignored", + } + ) + resolved = settings.load(self.project, overrides={"run.plugin.nested": ({"x": 1}, "-c")}) + self.assertEqual(resolved.get("executor.model"), "anthropic/model/submodel") + self.assertEqual(resolved.get("executor.editable"), ["a.py", "b.py"]) + self.assertEqual(resolved.get("run.k"), 3) + self.assertIs(resolved.get("run.plugin_bool"), True) + self.assertEqual(resolved.get("run.plugin.nested.x"), 1) + self.assertEqual(resolved.origin("run.plugin.nested.x"), "-c") + self.assertEqual(resolved.provider("local").base_url, "http://localhost:8080/v1") + self.assertNotIn("old_option", resolved.flat()) + + def test_parse_value(self): + for raw, value in ( + ("8", 8), + ("0.25", 0.25), + ("false", False), + ('"hello"', "hello"), + ('["one", 2, true]', ["one", 2, True]), + ("{x = 1}", {"x": 1}), + ("provider/model", "provider/model"), + ("", ""), + ("[broken", "[broken"), + ): + with self.subTest(raw=raw): + self.assertEqual(settings.parse_value(raw), value) + + def test_role_inheritance_and_human_selector(self): + resolved = settings.load( + self.project, + overrides={ + "improver.model": ("anthropic/root", "-c"), + "improver.steer": ("keep it small", "-c"), + "selector.effort": ("low", "-c"), + "selector.model": ("anthropic/selector", "-c"), + "reflector.steps": (4, "-c"), + "judge.timeout": (20, "-c"), + "judge.plugin_key": ("ignored by role", "-c"), + }, + ) + root = resolved.role("improver") + self.assertEqual(root["steer"], "keep it small") + self.assertEqual( + resolved.role("selector"), + { + key: value + for key, value in {**root, "effort": "low", "model": "anthropic/selector"}.items() + if key != "steer" + }, + ) + self.assertEqual(resolved.role("reflector"), {**resolved.role("selector"), "steps": 4}) + self.assertEqual( + resolved.role("judge"), + {key: value for key, value in {**root, "timeout": 20}.items() if key != "steer"}, + ) + human = settings.load( + self.project, + overrides={ + "selector.channel": ("human", "-c"), + "selector.model": ("unused", "-c"), + "reflector.timeout": (10, "-c"), + }, + ) + self.assertEqual(human.role("selector")["channel"], "human") + self.assertEqual(human.role("reflector")["channel"], "claude") + self.assertEqual(human.role("reflector")["model"], "") + self.assertEqual(human.role("reflector")["timeout"], 10) + with self.assertRaisesRegex(settings.ConfigError, "unknown role"): + resolved.role("executor") + + def test_provider_lookup_and_unknown_plugin_keys(self): + resolved = settings.load( + None, + overrides={ + "providers.local": ( + { + "type": "vertex", + "project": "my-project", + "region": "us-east1", + "plugin": 9, + }, + "-c", + ), + }, + ) + self.assertEqual( + resolved.provider("local"), + providers.Provider( + "local", + "vertex", + project="my-project", + region="us-east1", + ), + ) + with self.assertRaisesRegex(settings.ConfigError, "unknown provider 'absent'"): + resolved.provider("absent") + + def test_validation_aggregates_problems_and_rejects_secrets_anywhere(self): + with self.assertRaises(settings.ConfigError) as caught: + settings.load( + self.project, + overrides={ + "unexpected.key": (1, "-c"), + "executor.api_key": ("never-print-this", "-c"), + "run.k": (0, "-c"), + "run.workers": (True, "-c"), + "run.phase_width": (-1, "-c"), + "run.proposer_concurrency": (1.5, "-c"), + "run.transport_attempts": ("two", "-c"), + "run.budget_usd": (-0.1, "-c"), + "run.max_calls": (-1, "-c"), + "providers.local.type": ("invalid", "-c"), + }, + ) + message = str(caught.exception) + self.assertEqual(len(message.splitlines()), 10) + self.assertIn( + "store secrets in environment variables and reference them with api_key_env", message + ) + self.assertNotIn("never-print-this", message) + self.assertIn("unknown section 'unexpected'", message) + for name in ("k", "workers", "phase_width", "proposer_concurrency", "transport_attempts"): + self.assertIn(f"run.{name} must be a positive integer", message) + self.assertIn("run.budget_usd must be a non-negative number", message) + self.assertIn("run.max_calls must be a non-negative number", message) + self.assertIn("providers.local.type must be one of", message) + for key, value in ( + ("profiles.unused.executor.api_key", "hidden"), + ("executor.plugin", [{"api_key": "hidden"}]), + ): + with ( + self.subTest(key=key), + self.assertRaisesRegex(settings.ConfigError, "store secrets"), + ): + settings.load(None, overrides={key: (value, "-c")}) + + def test_malformed_sections_and_non_numeric_limits(self): + for key, value, message in ( + ("run", "invalid", "run must be a table"), + ("profiles", 3, "profiles must be a table"), + ("providers.local", 3, "providers.local must be a table"), + ("run.max_calls", "many", "run.max_calls must be a non-negative number"), + ("run.budget_usd", float("nan"), "run.budget_usd must be a non-negative number"), + ("run.budget_usd", False, "run.budget_usd must be a non-negative number"), + ): + with self.subTest(key=key), self.assertRaisesRegex(settings.ConfigError, message): + settings.load(None, overrides={key: (value, "-c")}) + with self.assertRaisesRegex(settings.ConfigError, "invalid setting key"): + settings.load(None, overrides={"run..k": (1, "-c")}) + + def test_invalid_toml_and_read_error(self): + self.write(self.project_file, "[broken") + with self.assertRaisesRegex(settings.ConfigError, "invalid TOML in"): + settings.load(self.project) + self.project_file.unlink() + self.project_file.mkdir() + with self.assertRaisesRegex(settings.ConfigError, "cannot read"): + settings.load(self.project) + + def test_locations_find_project_and_path_resolution(self): + self.assertEqual(settings.global_path(), self.global_file) + os.environ["XDG_CACHE_HOME"] = str(self.root / "cache") + self.assertEqual(settings.cache_dir(), self.root / "cache/harness2") + os.environ.pop("XDG_CACHE_HOME") + os.environ.pop("XDG_CONFIG_HOME") + with patch.object(Path, "home", return_value=self.root): + self.assertEqual(settings.global_path(), self.root / ".config/harness2/config.toml") + self.assertEqual(settings.cache_dir(), self.root / ".cache/harness2") + os.environ["XDG_CONFIG_HOME"] = "" + self.assertEqual(settings.global_path(), self.root / ".config/harness2/config.toml") + child = self.project / "src/deep" + child.mkdir(parents=True) + self.assertEqual(settings.find_project(child), self.project) + self.assertEqual(settings.find_project(self.project), self.project) + (child / ".harness2").mkdir() + self.assertEqual(settings.find_project(child), child) + (self.root / ".harness2").touch() + self.assertIsNone(settings.find_project(self.root)) + resolved = settings.load(self.project) + self.assertEqual(resolved.path("paths.tasks"), self.project / ".harness2/tasks") + absolute = settings.load( + self.project, overrides={"paths.runs": (str(self.root / "runs"), "-c")} + ) + self.assertEqual(absolute.path("paths.runs"), self.root / "runs") + self.assertEqual(settings.load(None).path("paths.harness"), Path.cwd().resolve()) + + def test_dotenv_parser_and_load_order(self): + dotenv = self.project / ".harness2/.env" + self.write( + dotenv, + ( + "# comment\n\nexport SHELL_VALUE = ignored\n" + "DOUBLE = \"two # literal\"\nSINGLE = 'one'\nPLAIN = value # note\n" + "EMPTY=\nINVALID-KEY=no\nno assignment\n = invalid\n" + "HARNESS2_RUN__K=4\nHARNESS2_PROFILE=fromenv\n" + ), + ) + self.write(self.project_file, "[profiles.fromenv.run]\nworkers = 2\n") + os.environ["SHELL_VALUE"] = "shell" + resolved = settings.load(self.project) + self.assertEqual(resolved.get("run.k"), 4) + self.assertEqual(resolved.get("run.workers"), 2) + self.assertEqual(resolved.origin("run.k"), "env HARNESS2_RUN__K") + self.assertEqual(os.environ["SHELL_VALUE"], "shell") + self.assertEqual(os.environ["DOUBLE"], "two # literal") + self.assertEqual(os.environ["SINGLE"], "one") + self.assertEqual(os.environ["PLAIN"], "value") + self.assertEqual(os.environ["EMPTY"], "") + self.assertNotIn("INVALID-KEY", os.environ) + settings.load_dotenv(self.root / "missing") + settings.load_dotenv(self.root) + + def test_set_value_preserves_comments_and_other_sections(self): + original = '# heading\n[run] # tuning\nk = 1 # count\n\n[executor]\nmodel = "original"\n' + self.write(self.project_file, original) + settings.set_value(self.project_file, "run.k", 3) + self.assertEqual(self.project_file.read_text(), original.replace("k = 1", "k = 3")) + settings.set_value(self.project_file, "run.workers", 2) + settings.set_value(self.project_file, "providers.local.type", "openai") + parsed = tomllib.loads(self.project_file.read_text()) + self.assertEqual(parsed["run"], {"k": 3, "workers": 2}) + self.assertEqual(parsed["providers"]["local"]["type"], "openai") + self.assertEqual(parsed["executor"], {"model": "original"}) + self.assertIn("# heading\n[run] # tuning\nk = 3 # count\n", self.project_file.read_text()) + + def test_set_value_round_trips_literals_and_creates_file(self): + for name, value in { + "string": 'quote " and \\ and\nnewline 😀\x7f', + "bool": True, + "int": 2, + "float": 0.25, + "array": ["one # literal", False, 2, 0.5], + "empty": [], + }.items(): + with self.subTest(name=name): + path = self.root / "new/nested/config.toml" + settings.set_value(path, "custom." + name, value) + parsed = tomllib.loads(path.read_text()) + self.assertEqual(parsed["custom"][name], value) + settings.set_value(path, "custom.a spaced key", "quoted key") + self.assertEqual(tomllib.loads(path.read_text())["custom"]["a spaced key"], "quoted key") + + def test_set_value_multiline_values_root_keys_and_line_endings(self): + original = ( + '# top\r\nroot = 1\r\n["executor"]\r\neditable = [\r\n' + ' "one#file", # first\r\n "two", # second\r\n] # files\r\n' + 'command = """first\r\n# inside string\r\nlast""" # command\r\n' + "[other]\r\nkeep = 'same'\r\n" + ) + self.project_file.write_bytes(original.encode()) + settings.set_value(self.project_file, "executor.editable", ["new"]) + settings.set_value(self.project_file, "executor.command", 'quoted " # literal') + settings.set_value(self.project_file, "added", False) + result = self.project_file.read_bytes().decode() + self.assertNotIn("one#file", result) + for comment in ("# top", "# first", "# second", "# files", "# command"): + self.assertIn(comment, result) + self.assertNotIn("# inside string", result) + self.assertIn("[other]\r\nkeep = 'same'\r\n", result) + self.assertNotIn("\n", result.replace("\r\n", "")) + parsed = tomllib.loads(result) + self.assertIs(parsed["added"], False) + self.assertEqual(parsed["executor"]["editable"], ["new"]) + self.assertEqual(parsed["executor"]["command"], 'quoted " # literal') + + def test_set_value_no_final_newline_and_invalid_input_do_not_destroy_file(self): + self.write(self.project_file, "[run]\nk = 1") + settings.set_value(self.project_file, "run.workers", 2) + self.assertEqual(tomllib.loads(self.project_file.read_text())["run"]["workers"], 2) + before = self.project_file.read_text() + for value in (None, {}, [[1]], [None], "\ud800"): + with self.subTest(value=value), self.assertRaises(settings.ConfigError): + settings.set_value(self.project_file, "run.value", value) + self.assertEqual(self.project_file.read_text(), before) + with self.assertRaisesRegex(settings.ConfigError, "invalid setting key"): + settings.set_value(self.project_file, "run..k", 2) + self.write(self.project_file, "[invalid") + with self.assertRaisesRegex(settings.ConfigError, "invalid TOML"): + settings.set_value(self.project_file, "run.k", 2) + self.assertEqual(self.project_file.read_text(), "[invalid") + + def test_set_value_multiline_quotes_and_escaped_hashes(self): + for literal in ( + "'literal # hash'", + '"escaped \\" # hash"', + "'''multiline # hash\nends with quote''''", + '"""multiline # hash\nends with quote""""', + ): + with self.subTest(literal=literal): + self.write(self.project_file, f"[suite]\nwork = {literal} # retain\n") + settings.set_value(self.project_file, "suite.work", "new") + self.assertEqual(self.project_file.read_text(), '[suite]\nwork = "new" # retain\n') + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_subscriptions.py b/tests/test_subscriptions.py new file mode 100644 index 00000000..976e41e1 --- /dev/null +++ b/tests/test_subscriptions.py @@ -0,0 +1,756 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Subscription isolation and resumable usage limits against offline CLIs.""" + +from __future__ import annotations + +import contextlib +import io +import json +import os +import subprocess +import sys +import tempfile +import time +import tomllib +import unittest +from pathlib import Path +from unittest.mock import Mock, patch + +from harness2 import providers, settings, suite, workspace +from harness2.cli.main import main +from harness2.core import events, recursion, runmark, schedule +from harness2.core.proposer import Proposer +from harness2.core.spec import ChannelSpec +from harness2.registry import SuiteContext +from harness2.settings import ConfigError + +FAKE_CLI = r"""import json, os, sys +from pathlib import Path + +argv = sys.argv[1:] +kind = Path(sys.argv[0]).name +if "--version" in argv: + print("fixture 1.0") + sys.exit(0) +status = argv in (["login", "status"], ["auth", "status"]) +cwd = Path.cwd() +home = Path(os.environ.get("CODEX_HOME", Path.home() / ".codex")) +config = home / "config.toml" +record = {"kind": kind, "argv": argv, "env": dict(os.environ), "cwd": str(cwd), + "stdin": "" if status else sys.stdin.read(), + "agents_exists": (home / "AGENTS.md").exists(), + "hooks_exists": (home / "hooks.json").exists(), + "config_exists": config.exists(), + "config": config.read_text() if config.exists() else None} +with open(os.environ["FAKE_RECORD"], "a") as log: + log.write(json.dumps(record) + "\n") +if status: + sys.exit(int(os.environ.get("FAKE_LOGIN_RC", "0"))) +improver = ("--output-format" in argv and argv[argv.index("--output-format") + 1] == "json" + or (cwd / "context").is_dir()) +role = "improver" if improver else "executor" +if os.environ.get("LIMIT_ROLE") == role: + limit = os.environ.get("LIMIT_TEXT", "rate-limit reached") + source = os.environ.get("LIMIT_SOURCE", "event") + if source == "event": + if kind == "claude": + print(json.dumps({"type": "result", "subtype": "error_during_execution", + "is_error": True, "errors": [limit]})) + else: + print(json.dumps({"type": "turn.failed", "error": {"message": limit}})) + sys.exit(0) + print(limit, file=sys.stderr if source == "stderr" else sys.stdout) + sys.exit(7) +if improver: + context = cwd / "context" + if (context / "candidates").is_dir(): + chosen = sorted((context / "candidates").glob("*.md"))[0].stem + text = "Complete.\n```choice\n" + chosen + "\n```" + elif (context / "adoption.json").is_file(): + text = "NO_ENTRY" + else: + for name in ("CLAUDE.md", "AGENTS.md"): + harness = context / "harness" / name + if harness.is_file(): + harness.write_text(harness.read_text() + "Check the output.\n") + text = "Check the output." +else: + (cwd / "output").mkdir(exist_ok=True) + (cwd / "output/report.txt").write_text("done\n") + text = "done" +if kind == "claude": + print(json.dumps({"type": "result", "subtype": "success", "result": text, + "total_cost_usd": 0.01})) +else: + print(json.dumps({"type": "item.completed", "item": { + "id": "reply", "type": "agent_message", "text": text}})) +""" + + +class SubscriptionTests(unittest.TestCase): + maxDiff = None + + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.project = self.root / "project" + self.tasks = self.project / ".harness2/tasks" + self.tasks.mkdir(parents=True) + (self.project / "CLAUDE.md").write_text("Read the task.\n") + (self.project / "AGENTS.md").write_text("Read the task.\n") + binary = self.root / "bin" + binary.mkdir() + for name in ("claude", "codex"): + path = binary / name + path.write_text(f"#!{sys.executable}\n" + FAKE_CLI) + path.chmod(0o755) + self.record = self.root / "records.jsonl" + self.user_home = self.root / "user" + for name in (".codex", ".claude"): + path = self.user_home / name + path.mkdir(parents=True) + (path / "config.toml").write_text("private user config") + (path / "auth.json").write_text("private user auth") + (self.user_home / ".codex/AGENTS.md").write_text("Private user instructions.\n") + (self.user_home / ".codex/hooks.json").write_text('{"hooks": {}}\n') + environment = patch.dict( + os.environ, + { + "PATH": f"{binary}{os.pathsep}{os.defpath}", + "HOME": str(self.user_home), + "XDG_CONFIG_HOME": str(self.root / "config"), + "XDG_STATE_HOME": str(self.root / "state"), + "FAKE_RECORD": str(self.record), + "CLAUDE_CODE_OAUTH_TOKEN": "fixture-oauth", + "ANTHROPIC_API_KEY": "ambient-anthropic", + "OPENAI_API_KEY": "ambient-openai", + "CODEX_API_KEY": "ambient-codex", + "ANTHROPIC_BASE_URL": "https://ambient.invalid", + "OPENAI_BASE_URL": "https://ambient.invalid", + "PYTHONDONTWRITEBYTECODE": "1", + }, + clear=True, + ) + environment.start() + self.addCleanup(environment.stop) + events.start(None) + self.addCleanup(events.start, None) + task = suite.new_task(self.tasks, "one", files=True, domain="coding", check="true") + (task / "task.md").write_text("Produce a report.") + + def adapter(self, name="test", **overrides): + values = { + "executor.model": "claude-login/claude-sonnet-5-5", + "improver.model": "claude-login/claude-sonnet-5-5", + } + values.update(overrides) + config = settings.load( + self.project, overrides={key: (value, "test") for key, value in values.items()} + ) + return suite.make_adapter( + SuiteContext(config, self.project, self.project / ".harness2/runs" / name) + ) + + def records(self): + return [json.loads(line) for line in self.record.read_text().splitlines()] + + def launch_record(self, label): + record = self.records()[-1] + env = dict(record["env"]) + env.pop("__CF_USER_TEXT_ENCODING", None) + normalized = json.dumps( + {"argv": [record["kind"], *record["argv"]], "env": env}, sort_keys=True + ) + normalized = normalized.replace(record["cwd"], "") + normalized = normalized.replace(str(self.root), "") + print(f"SUBSCRIPTION {label}: {normalized}") + return record + + def test_builtin_providers_use_cli_auth(self): + config = settings.load(self.project) + for name in ("claude-login", "chatgpt-login"): + self.assertEqual(config.provider(name).type, name) + self.assertEqual(providers.claude_env(config.provider("claude-login")), {}) + with patch.dict(os.environ, {}, clear=True): + self.assertEqual(providers.claude_env(config.provider("claude-login")), {}) + + def test_claude_subscription_executor_and_improver_launches(self): + adapter = self.adapter() + result = adapter.execute("one", None, "claude") + record = self.launch_record("claude-login executor") + self.assertEqual( + record["argv"], + [ + "-p", + "--output-format", + "stream-json", + "--verbose", + "--dangerously-skip-permissions", + "--setting-sources", + "project,local", + "--model", + "claude-sonnet-5-5", + "--effort", + "high", + "--strict-mcp-config", + "--no-session-persistence", + ], + ) + self.assertFalse((result / "claude-home").exists()) + context = self.root / "context" + context.mkdir() + Proposer(adapter.spec.channels["proposer"]).call("mission", context, "improver") + record = self.launch_record("claude-login improver") + self.assertEqual( + record["argv"], + [ + "--dangerously-skip-permissions", + "-p", + "--output-format", + "json", + "--effort", + "high", + "--model", + "claude-sonnet-5-5", + "--setting-sources", + "project,local", + "--strict-mcp-config", + "--no-session-persistence", + ], + ) + for record in self.records(): + self.assertNotIn("CLAUDE_CONFIG_DIR", record["env"]) + self.assertEqual(record["env"]["CLAUDE_CODE_OAUTH_TOKEN"], "fixture-oauth") + self.assertTrue(all(key not in record["env"] for key in providers.STRIP_ENV)) + os.environ.pop("CLAUDE_CODE_OAUTH_TOKEN") + adapter.execute("one", None, "native-login") + self.assertNotIn("CLAUDE_CODE_OAUTH_TOKEN", self.records()[-1]["env"]) + self.assertNotIn("CLAUDE_CONFIG_DIR", self.records()[-1]["env"]) + + def test_chatgpt_subscription_executor_and_improver_launches(self): + adapter = self.adapter( + **{ + "executor.name": "codex", + "executor.model": "chatgpt-login/gpt-6.1-sol", + "improver.channel": "codex", + "improver.model": "chatgpt-login/gpt-6.1-sol", + } + ) + home = self.user_home / ".codex" + before = {p.name: p.read_bytes() for p in home.iterdir()} + result = adapter.execute("one", None, "codex") + record = self.launch_record("chatgpt-login executor") + self.assertEqual( + record["argv"], + [ + "exec", + "--json", + "--skip-git-repo-check", + "--dangerously-bypass-approvals-and-sandbox", + "--dangerously-bypass-hook-trust", + "--ephemeral", + "--ignore-user-config", + "--ignore-rules", + "-m", + "gpt-6.1-sol", + "-c", + 'model_reasoning_effort="high"', + "-C", + record["cwd"], + "-", + ], + ) + self.assertFalse((result / "codex-home").exists()) + context = self.root / "context" + context.mkdir() + Proposer(adapter.spec.channels["proposer"]).call("mission", context, "improver") + record = self.launch_record("chatgpt-login improver") + self.assertEqual( + record["argv"], + [ + "exec", + "--json", + "--skip-git-repo-check", + "--dangerously-bypass-approvals-and-sandbox", + "--ephemeral", + "--ignore-user-config", + "--ignore-rules", + "-m", + "gpt-6.1-sol", + "-c", + 'model_reasoning_effort="high"', + "-C", + record["cwd"], + "-", + ], + ) + for record in self.records(): + self.assertEqual( + record["env"]["CODEX_HOME"], str(self.root / "state/harness2/codex-home") + ) + self.assertFalse(record["config_exists"]) + self.assertFalse(record["agents_exists"]) + self.assertFalse(record["hooks_exists"]) + self.assertTrue( + all( + key not in record["env"] + for key in (*providers.STRIP_ENV, "OPENAI_API_KEY", "CODEX_API_KEY") + ) + ) + self.assertEqual({p.name: p.read_bytes() for p in home.iterdir()}, before) + self.assertFalse((settings.state_dir() / "codex-home").exists()) + + def test_configured_chatgpt_home_is_used_without_writes(self): + for populated in (False, True): + with self.subTest(populated=populated): + home = self.user_home / f"custom codex-{populated}" + if populated: + home.mkdir() + for name in ("auth.json", "config.toml", "AGENTS.md", "hooks.json"): + (home / name).write_text("fixture " + name) + before = {p.name: p.read_bytes() for p in home.iterdir()} if populated else {} + adapter = self.adapter( + f"custom-{populated}", + **{ + "executor.name": "codex", + "executor.model": "chatgpt-login/model", + "improver.channel": "codex", + "improver.model": "chatgpt-login/model", + "providers.chatgpt-login.home": f"~/{home.name}", + }, + ) + with patch.dict(os.environ, {"CODEX_HOME": str(self.user_home / ".codex")}): + result = adapter.execute("one", None, "custom") + self.assertFalse((result / "codex-home").exists()) + context = self.root / f"context-{populated}" + context.mkdir() + Proposer(adapter.spec.channels["proposer"]).call("mission", context, "custom") + for record in self.records()[-2:]: + self.assertEqual(record["env"]["CODEX_HOME"], str(home)) + self.assertEqual(record["config_exists"], populated) + self.assertEqual(record["agents_exists"], populated) + self.assertEqual(record["hooks_exists"], populated) + self.assertNotIn("OPENAI_API_KEY", record["env"]) + self.assertNotIn("CODEX_API_KEY", record["env"]) + self.assertIn("--ignore-user-config", record["argv"]) + self.assertIn("--ignore-rules", record["argv"]) + self.assertIn("--ephemeral", record["argv"]) + after = {p.name: p.read_bytes() for p in home.iterdir()} if home.exists() else {} + self.assertEqual(after, before) + self.assertEqual(home.exists(), populated) + + def test_inherited_claude_home_is_preserved_and_codex_home_is_overridden(self): + context = self.root / "context" + context.mkdir() + for kind, provider_name, variable in ( + ("claude", "claude-login", "CLAUDE_CONFIG_DIR"), + ("codex", "chatgpt-login", "CODEX_HOME"), + ): + with ( + self.subTest(kind=kind), + patch.dict(os.environ, {variable: str(self.user_home / ("." + kind))}), + ): + adapter = self.adapter( + kind, + **{ + "executor.name": kind, + "executor.model": provider_name + "/model", + "improver.channel": kind, + "improver.model": provider_name + "/model", + }, + ) + adapter.execute("one", None, "inherited") + expected = ( + os.environ[variable] + if kind == "claude" + else str(settings.state_dir() / "codex-home") + ) + self.assertEqual(self.records()[-1]["env"][variable], expected) + Proposer(adapter.spec.channels["proposer"]).call("mission", context, "inherited") + self.assertEqual(self.records()[-1]["env"][variable], expected) + + def test_login_status_success_failure_missing_cli_and_timeout(self): + for kind, argv, login in ( + ("claude-login", ["auth", "status"], "claude auth login"), + ("chatgpt-login", ["login", "status"], "harness2 login codex"), + ): + with self.subTest(provider=kind): + provider = providers.Provider(kind, kind) + self.assertEqual(providers.check(provider), []) + record = self.records()[-1] + self.assertEqual(record["argv"], argv) + self.assertNotIn("CLAUDE_CONFIG_DIR", record["env"]) + if kind == "chatgpt-login": + self.assertEqual( + record["env"]["CODEX_HOME"], str(settings.state_dir() / "codex-home") + ) + self.assertNotIn("OPENAI_API_KEY", record["env"]) + self.assertNotIn("CODEX_API_KEY", record["env"]) + else: + self.assertNotIn("CODEX_HOME", record["env"]) + self.assertTrue(all(key not in record["env"] for key in providers.STRIP_ENV)) + where = " for harness2" if kind == "chatgpt-login" else "" + problem = [f"{kind}: not logged in{where}; run `{login}`"] + with patch.dict(os.environ, {"FAKE_LOGIN_RC": "1"}): + self.assertEqual(providers.check(provider), problem) + with patch.dict(os.environ, {"PATH": ""}): + self.assertEqual(providers.check(provider), problem) + with patch( + "harness2.providers.subprocess.run", + side_effect=subprocess.TimeoutExpired("cli", 10), + ): + self.assertEqual(providers.check(provider), problem) + args = providers.codex_args( + providers.Provider("chatgpt-login", "chatgpt-login"), + 'model"\nnext\x7f', + 'effort"\nnext', + ) + self.assertEqual(args[3], 'model"\nnext\x7f') + self.assertEqual(tomllib.loads(args[5]), {"model_reasoning_effort": 'effort"\nnext'}) + + def test_codex_logged_in_checks_only_the_requested_home(self): + home = self.root / "login home" + with patch.dict(os.environ, {"CODEX_HOME": str(self.user_home / ".codex")}): + self.assertTrue(providers.codex_logged_in(home)) + record = self.records()[-1] + self.assertEqual(record["argv"], ["login", "status"]) + self.assertEqual(record["env"]["CODEX_HOME"], str(home)) + self.assertFalse(record["config_exists"]) + self.assertFalse(record["agents_exists"]) + self.assertFalse(record["hooks_exists"]) + self.assertTrue( + all( + key not in record["env"] + for key in (*providers.STRIP_ENV, "OPENAI_API_KEY", "CODEX_API_KEY") + ) + ) + with patch.dict(os.environ, {"FAKE_LOGIN_RC": "1"}): + self.assertFalse(providers.codex_logged_in(home)) + for error in (FileNotFoundError(), subprocess.TimeoutExpired("codex", 10), ValueError()): + with ( + self.subTest(error=type(error).__name__), + patch("harness2.providers.subprocess.run", side_effect=error) as run, + ): + self.assertFalse(providers.codex_logged_in(home)) + self.assertEqual(run.call_args.kwargs["timeout"], 10) + self.assertFalse(home.exists()) + + def test_api_provider_homes_and_ambient_oauth_are_unchanged(self): + for kind, provider_name in (("claude", "anthropic"), ("codex", "openai")): + with self.subTest(kind=kind): + adapter = self.adapter( + kind, + **{ + "executor.name": kind, + "executor.model": provider_name + "/fixture", + "improver.channel": kind, + "improver.model": provider_name + "/fixture", + }, + ) + result = adapter.execute("one", None, "api") + record = self.records()[-1] + self.assertEqual(record["env"]["CLAUDE_CODE_OAUTH_TOKEN"], "fixture-oauth") + if kind == "claude": + self.assertEqual(record["env"]["ANTHROPIC_API_KEY"], "ambient-anthropic") + else: + self.assertEqual(record["env"]["OPENAI_API_KEY"], "ambient-openai") + self.assertEqual(record["env"]["CODEX_HOME"], str(result / "codex-home")) + self.assertEqual( + record["config"], + providers.codex_config( + adapter.settings.provider("openai"), "fixture", "high" + ), + ) + self.assertNotIn("-c", record["argv"]) + Proposer(adapter.spec.channels["proposer"]).call("mission", None, "api-improver") + record = self.records()[-1] + self.assertEqual(record["env"]["CLAUDE_CODE_OAUTH_TOKEN"], "fixture-oauth") + if kind == "codex": + self.assertEqual( + record["config"], + providers.codex_config( + adapter.settings.provider("openai"), "fixture", "high" + ), + ) + self.assertNotEqual(record["env"]["CODEX_HOME"], str(self.user_home / ".codex")) + + def test_wrong_cli_and_bridge_reject_subscription_providers(self): + claude = providers.Provider("claude-login", "claude-login") + chatgpt = providers.Provider("chatgpt-login", "chatgpt-login") + with self.assertRaisesRegex(ConfigError, "cannot use a ChatGPT login"): + providers.claude_env(chatgpt) + with self.assertRaises(ConfigError): + providers.codex_config(claude, "model", "high") + with self.assertRaises(ConfigError): + providers.codex_args(claude, "model", "high") + for provider in (claude, chatgpt): + for operation in (providers.opencode_config, providers.litellm_params): + with self.subTest(provider=provider.type, operation=operation.__name__): + with self.assertRaisesRegex(ConfigError, "API provider"): + operation(provider, "model") + + def test_doctor_per_role_subscription_compatibility(self): + for channel, provider_name in ( + ("claude", "claude-login"), + ("codex", "chatgpt-login"), + ("claude", "chatgpt-login"), + ("codex", "claude-login"), + ("opencode", "claude-login"), + ("opencode", "chatgpt-login"), + ): + with self.subTest(channel=channel, provider=provider_name): + adapter = ( + self.adapter( + **{ + "executor.name": channel, + "executor.model": provider_name + "/model", + "improver.channel": channel, + "improver.model": provider_name + "/model", + } + ) + if channel != "opencode" + else self.adapter( + **{ + "improver.channel": channel, + "improver.model": provider_name + "/model", + } + ) + ) + code, roles = self.doctor(adapter) + compatible = (channel, provider_name) in ( + ("claude", "claude-login"), + ("codex", "chatgpt-login"), + ) + self.assertEqual(code, 0 if compatible else 1) + self.assertEqual(roles["improver"]["ok"], compatible) + self.assertEqual(roles["judge"]["ok"], compatible) + + def doctor(self, adapter): + output = io.StringIO() + argv = ["-C", str(self.project), "--json"] + for key, value in adapter.settings.flat().items(): + if key.startswith(("executor.", "improver.", "providers.")) and not isinstance( + value, list + ): + argv += ["-c", f"{key}={json.dumps(value)}"] + with contextlib.redirect_stdout(output): + code = main([*argv, "doctor"]) + return code, {row["check"]: row for row in json.loads(output.getvalue())} + + def test_doctor_login_status_uses_dedicated_codex_home(self): + adapter = self.adapter( + **{ + "executor.name": "codex", + "executor.model": "chatgpt-login/model", + "improver.channel": "codex", + "improver.model": "chatgpt-login/model", + } + ) + code, rows = self.doctor(adapter) + self.assertEqual(code, 0) + self.assertNotIn("Codex home", rows) + with patch.dict(os.environ, {"FAKE_LOGIN_RC": "1"}): + code, rows = self.doctor(adapter) + self.assertEqual(code, 1) + self.assertEqual( + rows["provider chatgpt-login"]["detail"], + "chatgpt-login: not logged in for harness2; run `harness2 login codex`", + ) + code, rows = self.doctor(self.adapter()) + self.assertEqual(code, 1) + self.assertEqual( + rows["provider claude-login"]["detail"], + "claude-login: not logged in; run `claude auth login`", + ) + _, rows = self.doctor(self.adapter()) + self.assertNotIn("Codex home", rows) + self.assertFalse((settings.state_dir() / "codex-home").exists()) + self.assertTrue( + all( + record["argv"] in (["auth", "status"], ["login", "status"]) + for record in self.records() + ) + ) + + def test_doctor_custom_codex_home_login_and_instruction_files(self): + home = self.user_home / "custom-codex" + home.mkdir() + adapter = self.adapter( + **{ + "executor.name": "codex", + "executor.model": "chatgpt-login/model", + "improver.channel": "codex", + "improver.model": "chatgpt-login/model", + "providers.chatgpt-login.home": "~/custom-codex", + } + ) + with patch.dict(os.environ, {"FAKE_LOGIN_RC": "1"}): + code, rows = self.doctor(adapter) + self.assertEqual(code, 1) + self.assertEqual( + rows["provider chatgpt-login"]["detail"], + f"chatgpt-login: not logged in at {home}; run `codex login` with CODEX_HOME={home}", + ) + self.assertEqual(self.records()[-1]["env"]["CODEX_HOME"], str(home)) + for files in ((), ("hooks.json",), ("AGENTS.md",), ("AGENTS.md", "hooks.json")): + with self.subTest(files=files): + for name in ("AGENTS.md", "hooks.json"): + (home / name).write_text("fixture" if name in files else "") + before = {p.name: p.read_bytes() for p in home.iterdir()} + code, rows = self.doctor(adapter) + self.assertEqual(code, 0) + if files: + self.assertEqual( + rows["Codex home"], + { + "check": "Codex home", + "ok": True, + "detail": " and ".join(str(home / name) for name in files) + + " are part of every Codex run", + }, + ) + else: + self.assertNotIn("Codex home", rows) + self.assertEqual({p.name: p.read_bytes() for p in home.iterdir()}, before) + + def test_executor_usage_limits_stop_and_resume_entire_run(self): + adapter = self.adapter() + self.stop_and_resume(adapter, "executor") + + def test_improver_usage_limits_stop_without_retry_and_resume_entire_run(self): + adapter = self.adapter() + self.stop_and_resume(adapter, "improver") + + def stop_and_resume(self, adapter, role): + run_root = adapter.spec.runs_root + event_path = run_root / "events.jsonl" + prop = Proposer(adapter.spec.channels["proposer"], audit_dir=run_root / "audit") + kwargs = dict( + adapter=adapter, + prop=prop, + runs_root=run_root, + library_root=run_root / "library", + modes=("agg",), + ) + events.start(event_path) + with ( + patch.dict(os.environ, {"LIMIT_ROLE": role}), + patch("harness2.core.proposer.time", wraps=time) as clock, + ): + with self.assertRaisesRegex(events.Stopped, "^usage_limit$"): + schedule.run_sweep(1, **kwargs) + clock.sleep.assert_not_called() + self.assertEqual(events.totals()["stopped"], "usage_limit") + self.assertFalse(list(run_root.glob("tasks-*/**/summary.json"))) + if role == "executor": + self.assertEqual(list((run_root / "rollouts").iterdir()), []) + self.assertEqual(events.totals()["calls"], 0) + else: + self.assertFalse(list((run_root / "audit").glob("*.response.md"))) + events.start(event_path) + resumed = schedule.run_sweep(1, **kwargs) + self.assertEqual((resumed["domains"][0]["done"], resumed["domains"][0]["failed"]), (1, 0)) + self.assertTrue(list(run_root.glob("tasks-*/**/summary.json"))) + rollouts = list((run_root / "rollouts").iterdir()) + self.assertTrue(rollouts) + self.assertTrue(all(runmark.is_complete(path, match_harness=False) for path in rollouts)) + + def test_usage_limit_failure_sources_and_success_mentions(self): + for source in ("event", "stderr", "stdout"): + for kind in ("claude", "codex"): + with self.subTest(source=source, kind=kind): + provider_name = "claude-login" if kind == "claude" else "chatgpt-login" + adapter = self.adapter( + f"{kind}-{source}", + **{ + "executor.name": kind, + "executor.model": provider_name + "/model", + }, + ) + events.start(None) + with patch.dict(os.environ, {"LIMIT_ROLE": "executor", "LIMIT_SOURCE": source}): + with self.assertRaisesRegex(events.Stopped, "usage_limit"): + adapter.execute("one", None, "limited") + self.assertEqual(list((adapter.spec.runs_root / "rollouts").iterdir()), []) + events.start(None) + provider = adapter.settings.provider(provider_name) + with patch.dict(os.environ, {"LIMIT_ROLE": "improver", "LIMIT_SOURCE": source}): + context = self.root / "context" + context.mkdir(exist_ok=True) + with self.assertRaisesRegex(events.Stopped, "usage_limit"): + Proposer( + ChannelSpec(kind, provider_name + "/model", provider=provider) + ).call("mission", context, "limited") + for text in ( + "usage limit", + "RATE_LIMIT", + "too many requests", + "HTTP 429", + "quota", + "credit balance is too low", + ): + self.assertIsNotNone(providers.USAGE_LIMIT.search(text)) + self.assertIsNone(providers.USAGE_LIMIT.search("14290")) + events.start(None) + adapter = self.adapter("success") + (self.root / "bin/claude").write_text( + f"#!{sys.executable}\nimport json\n" + "print(json.dumps({'type':'result', 'subtype':'success', 'result':'rate limit fixed'}))\n" + ) + self.assertTrue( + runmark.is_complete( + adapter.execute("one", None, "success"), harness_dir=adapter.base_harness() + ) + ) + self.assertTrue( + Proposer(adapter.spec.channels["proposer"]).call("mission", None, "success") + ) + + def test_core_does_not_swallow_stopped_from_batched_adapter(self): + adapter = self.adapter() + adapter.execute_many = Mock(side_effect=events.Stopped("usage_limit")) + with self.assertRaisesRegex(events.Stopped, "usage_limit"): + schedule._roll_bases("one", adapter, [("a", "a"), ("b", "b")], self.root / "dead") + with patch.object(adapter, "execute", side_effect=events.Stopped("usage_limit")): + with self.assertRaisesRegex(events.Stopped, "usage_limit"): + recursion._roll(adapter, None, "one", "a", None, lambda label: label, self.root) + + def test_opencode_usage_limit_stops_even_with_a_partial_answer(self): + binary = self.root / "bin/opencode" + binary.write_text( + f"#!{sys.executable}\nimport json\nfrom pathlib import Path\n" + "Path('output/answer.md').write_text('partial')\n" + "print(json.dumps({'type':'error','error':{'data':{'message':'quota exceeded'}}}))\n" + ) + binary.chmod(0o755) + provider = providers.Provider("api", "openai") + prop = Proposer(ChannelSpec("opencode", "api/model", provider=provider)) + with patch("harness2.core.proposer.time", wraps=time) as clock: + with self.assertRaisesRegex(events.Stopped, "usage_limit"): + prop.call("mission", None, "limit") + clock.sleep.assert_not_called() + self.assertEqual(events.totals()["stopped"], "usage_limit") + + def test_project_workspace_prunes_before_adding(self): + subprocess.run(["git", "init", "-q", str(self.project)], check=True) + workspace.commit_all(self.project, "fixture") + with patch("harness2.workspace.subprocess.run", wraps=subprocess.run) as run: + with workspace.project_workspace(self.project, "HEAD") as ws: + self.assertTrue(ws.exists()) + argv = [call.args[0] for call in run.call_args_list] + self.assertEqual(argv[0], ["git", "-C", str(self.project), "worktree", "prune"]) + self.assertEqual(argv[1][4], "add") diff --git a/tests/test_suite.py b/tests/test_suite.py new file mode 100644 index 00000000..daa3bb72 --- /dev/null +++ b/tests/test_suite.py @@ -0,0 +1,670 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The generic suite against fake CLIs, local git projects, and the real scheduler.""" + +from __future__ import annotations + +import csv +import json +import os +import shlex +import shutil +import subprocess +import sys +import tempfile +import unittest +import warnings +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from pathlib import Path +from unittest.mock import patch + +from harness2 import registry, suite, workspace +from harness2.core import events, runmark, schedule, tree +from harness2.core.proposer import Proposer +from harness2.core.spec import BenchmarkAdapter +from harness2.settings import ConfigError, load + +FAKE_CLI = r"""import json +import os +import sys +import time +from pathlib import Path + +if "--version" in sys.argv: + print("fake cli 1.0") + sys.exit(0) +cwd = Path.cwd() +prompt = sys.stdin.read() +record = {"cwd": str(cwd), "argv": sys.argv[1:], "prompt": prompt, + "claude_home": os.environ.get("CLAUDE_CONFIG_DIR"), + "codex_home": os.environ.get("CODEX_HOME"), + "harness": {name: (cwd / name).read_text() for name in ("CLAUDE.md", "AGENTS.md") + if (cwd / name).is_file()}} +if record["codex_home"]: + record["config"] = (Path(record["codex_home"]) / "config.toml").read_text() +with open(os.environ["FAKE_RECORD"], "a") as stream: + stream.write(json.dumps(record) + "\n") +if "--output-format" in sys.argv and sys.argv[sys.argv.index("--output-format") + 1] == "json": + context = cwd / "context" + if (context / "rubric.md").is_file(): + assert (context / "task.md").is_file() + assert (context / "deliverables/final.md").is_file() + text = os.environ.get("FAKE_JUDGE", '{"n_passed": 2, "n_criteria": 3, "notes": "one missing"}') + elif (context / "candidates").is_dir(): + chosen = sorted((context / "candidates").glob("*.md"))[0].stem + text = "Complete change.\n```choice\n" + chosen + "\n```" + elif (context / "adoption.json").is_file(): + text = "NO_ENTRY" + else: + harness = context / "harness/CLAUDE.md" + harness.write_text(harness.read_text() + "Run the project tests.\n") + text = "Use the project's tests." + print(json.dumps({"type": "result", "subtype": "success", "result": text, + "total_cost_usd": 0.05})) + sys.exit(0) +if Path(sys.argv[0]).name == "codex" and (cwd / "context/rubric.md").is_file(): + print(json.dumps({"type": "item.completed", "item": { + "id": "judge", "type": "agent_message", "text": os.environ["FAKE_JUDGE"]}})) + sys.exit(0) +assert not (cwd / ".harness2").exists(), "project state leaked" +assert not (cwd / "check").exists(), "checks leaked" +assert not (cwd / "rubric.md").exists(), "rubric leaked" +(cwd / "code.txt").write_text("after\n") +for name in ("CLAUDE.md", "AGENTS.md"): + if (cwd / name).is_file(): + (cwd / name).write_text("agent changed instructions\n") +(cwd / ".harness2").mkdir() +(cwd / ".harness2/leak").write_text("private state") +(cwd / "output").mkdir(exist_ok=True) +(cwd / "output/report.txt").write_text("report\n") +if Path(sys.argv[0]).name == "codex": + print(json.dumps({"type": "item.completed", "item": { + "id": "1", "type": "agent_message", "text": "Finished the change."}}), flush=True) + print(json.dumps({"type": "turn.completed", "usage": { + "input_tokens": 10, "cached_input_tokens": 0, "output_tokens": 5}}), flush=True) +else: + print(json.dumps({"type": "system", "subtype": "init", "model": "fake"}), flush=True) + print(json.dumps({"type": "assistant", "message": {"content": [ + {"type": "tool_use", "name": "Write", "input": {"file_path": "code.txt"}}, + {"type": "text", "text": "Finished the change."}]}}), flush=True) + print(json.dumps({"type": "result", "subtype": "success", "result": "Finished the change.", + "total_cost_usd": 0.125}), flush=True) +time.sleep(float(os.environ.get("FAKE_SLEEP", "0"))) +sys.exit(int(os.environ.get("FAKE_RC", "0"))) +""" + + +class SuiteTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name).resolve() + self.project = self.root / "project" + self.project.mkdir() + (self.project / ".harness2/tasks").mkdir(parents=True) + self.tasks_dir = self.project / ".harness2/tasks" + (self.project / "code.txt").write_text("before\n") + (self.project / "CLAUDE.md").write_text("Read the code first.\n") + (self.project / "AGENTS.md").write_text("Read the code first.\n") + subprocess.run(["git", "init", "-q", str(self.project)], check=True) + binary = self.root / "bin" + binary.mkdir() + for name in ("claude", "codex"): + path = binary / name + path.write_text(f"#!{sys.executable}\n" + FAKE_CLI) + path.chmod(0o755) + self.record = self.root / "records.jsonl" + env = {k: v for k, v in os.environ.items() if not k.startswith(("HARNESS2_", "FAKE_"))} + env.update( + PATH=f"{binary}{os.pathsep}{env['PATH']}", + FAKE_RECORD=str(self.record), + XDG_CONFIG_HOME=str(self.root / "config"), + ANTHROPIC_API_KEY="fake-key", + OPENAI_API_KEY="fake-key", + ) + environment = patch.dict(os.environ, env, clear=True) + environment.start() + self.addCleanup(environment.stop) + self.addCleanup(events.start, None) + events.start(None) + self.task("one", domain="coding", split="train") + workspace.commit_all(self.project, "initial") + self.ctx = self.context() + self.adapter = suite.make_adapter(self.ctx) + + def context(self, **overrides): + values = {"executor.model": "anthropic/fake", "improver.model": "anthropic/fake"} + values.update(overrides) + settings = load( + self.project, overrides={key: (value, "test") for key, value in values.items()} + ) + return registry.SuiteContext(settings, self.project, self.project / ".harness2/runs/test") + + def task(self, task_id, *, files=False, **values): + task_dir = suite.new_task( + self.tasks_dir, + task_id, + files=files, + domain=values.pop("domain", "default"), + check=values.pop("check", None), + ) + (task_dir / "task.md").write_text("Change code.txt as requested.\n") + with (task_dir / "task.toml").open("a") as stream: + for key, value in values.items(): + stream.write(f"{key} = {json.dumps(value)}\n") + return task_dir + + def records(self): + return [json.loads(line) for line in self.record.read_text().splitlines()] + + def test_module_task_protocol_splits_and_prompt_profiles(self): + self.assertIs(registry.load("tasks"), suite) + self.assertIsInstance(self.adapter, BenchmarkAdapter) + self.assertEqual(self.adapter.tasks(), ["one"]) + self.assertEqual(self.adapter.tasks("train"), ["one"]) + self.assertEqual(self.adapter.tasks("test"), []) + self.assertEqual(self.adapter.domain_of("one"), "coding") + self.assertIsNone(self.adapter.task_files("one")) + self.assertEqual(self.adapter.task_prompt("one"), "Change code.txt as requested.\n") + self.assertIs(self.adapter.spec.prompts, suite.PROJECT_PROMPTS) + self.assertEqual(self.adapter.prompts_for("one"), suite.PROJECT_PROMPTS) + self.adapter.validate_tree({}) + files = self.task("docs", files=True, deliverable="message") + self.assertEqual(self.adapter.task_files("docs"), files / "files") + (self.project / "steer.txt").write_text("Use local conventions.") + ctx = self.context(**{"improver.steer": "@steer.txt", "suite.work": "Maintain the project"}) + prompts = suite.make_adapter(ctx).prompts_for("docs") + self.assertEqual(prompts.work, "Maintain the project") + self.assertEqual(prompts.deliverable_word, "answer") + self.assertEqual(prompts.domain_guidance, "Use local conventions.") + self.assertEqual(prompts.yardstick, suite.FILES_PROMPTS.yardstick) + for overrides in ({"suite.work": "rubric"}, {"improver.steer": "score"}): + with self.assertRaisesRegex(ConfigError, "improver.steer / suite.work:"): + suite.make_adapter(self.context(**overrides)).prompts_for("one") + with self.assertRaisesRegex(ConfigError, "cannot read improver.steer"): + suite.make_adapter(self.context(**{"improver.steer": "@missing"})).prompts_for("one") + with self.assertRaises(ConfigError): + self.adapter.tasks("missing") + + def test_claude_execute_capture_idempotence_and_isolation(self): + self.assertFalse(self.adapter.delivered(self.root / "missing")) + run_dir = self.adapter.execute("one", None, "nested/run") + self.assertEqual(run_dir.name, "nested__run") + self.assertTrue(runmark.is_complete(run_dir, harness_dir=self.adapter.base_harness())) + self.assertTrue(self.adapter.delivered(run_dir)) + self.assertEqual(self.adapter.rollout_cost(run_dir), 0.125) + artifacts = self.adapter.deliverable_files(run_dir) + patch_text = artifacts["deliverable.patch"] + self.assertIn("+after", patch_text) + self.assertNotIn("CLAUDE.md", patch_text) + self.assertNotIn(".harness2", patch_text) + self.assertEqual( + self.adapter.deliverable_text(run_dir), patch_text + "\n\nFinished the change." + ) + self.assertIn("[ACTION] Write:", self.adapter.trajectory(run_dir, obs_trunc=10000)) + meta = json.loads((run_dir / "meta.json").read_text()) + self.assertTrue(meta["ok"]) + self.assertEqual(meta["exit_code"], 0) + first = self.records()[0] + self.assertEqual(first["prompt"], self.adapter.task_prompt("one")) + self.assertEqual(first["claude_home"], str(run_dir / "claude-home")) + self.assertFalse(Path(first["cwd"]).exists()) + self.assertEqual(self.adapter.execute("one", None, "nested/run"), run_dir) + self.assertEqual(len(self.records()), 1) + self.assertEqual((self.project / "code.txt").read_text(), "before\n") + (run_dir / "deliverable.patch").write_text("") + self.assertFalse(self.adapter.delivered(run_dir)) + + def test_changed_candidate_reexecutes_and_drops_stale_judge(self): + candidate = self.root / "candidate" + surface = self.adapter.substrate.surface() + tree.write_tree(surface, {"CLAUDE.md": "First candidate\n"}, candidate) + run_dir = self.adapter.execute("one", candidate, "candidate") + (run_dir / "judge.json").write_text('{"n_passed": 0}') + (candidate / "CLAUDE.md").write_text("Second candidate\n") + self.adapter.execute("one", candidate, "candidate") + self.assertEqual(len(self.records()), 2) + self.assertFalse((run_dir / "judge.json").exists()) + + def test_codex_stream_and_provider_home(self): + ctx = self.context(**{"executor.name": "codex", "executor.model": "openai/fake"}) + adapter = suite.make_adapter(ctx) + run_dir = adapter.execute("one", None, "codex") + self.assertTrue(adapter.delivered(run_dir)) + self.assertEqual(adapter.rollout_cost(run_dir), 0) + self.assertNotIn("AGENTS.md", adapter.deliverable_text(run_dir)) + self.assertIn("Finished the change.", adapter.deliverable_text(run_dir)) + self.assertIn("tokens in=10", adapter.trajectory(run_dir, obs_trunc=10000)) + record = self.records()[0] + self.assertEqual(record["codex_home"], str(run_dir / "codex-home")) + self.assertIn('model = "fake"', record["config"]) + self.assertEqual(record["argv"][-1], "-") + self.assertFalse(adapter.spec.allow_subagents) + + def test_agent_failure_and_timeout_are_completed(self): + for run_id, env, kwargs, expected in ( + ("failed", {"FAKE_RC": "7"}, {}, 7), + ("timeout", {"FAKE_SLEEP": "2"}, {"timeout": 0.1}, -1), + ): + with self.subTest(run_id=run_id), patch.dict(os.environ, env): + run_dir = self.adapter.execute("one", None, run_id, **kwargs) + meta = json.loads((run_dir / "meta.json").read_text()) + self.assertFalse(meta["ok"]) + self.assertEqual(meta["exit_code"], expected) + self.assertEqual(meta["timed_out"], run_id == "timeout") + self.assertTrue( + runmark.is_complete(run_dir, harness_dir=self.adapter.base_harness()) + ) + count = len(self.records()) + self.adapter.execute("one", None, run_id, **kwargs) + self.assertEqual(len(self.records()), count) + + def test_task_timeout_takes_precedence(self): + self.task("bounded", timeout=1) + with patch.dict(os.environ, {"FAKE_SLEEP": "0.02"}): + run_dir = self.adapter.execute("bounded", None, "bounded", timeout=0.001) + self.assertTrue(json.loads((run_dir / "meta.json").read_text())["ok"]) + + def test_files_command_executor_and_binary_deliverables(self): + task_dir = self.task("files", files=True, check="test -s output/report.txt") + (task_dir / "files/source.txt").write_text("supplied source\n") + script = self.root / "command.py" + script.write_text( + "import pathlib, sys\n" + "ws, prompt, output = map(pathlib.Path, sys.argv[1:])\n" + "assert ws == pathlib.Path.cwd()\n" + "assert 'Save your deliverables in the output/ directory.' in prompt.read_text()\n" + "(output / 'report.txt').write_text((ws / 'source.txt').read_text())\n" + "(output / 'binary.dat').write_bytes(b'\\xff\\x00')\n" + "print('Saved report.')\n" + ) + command = f"{shlex.quote(sys.executable)} {shlex.quote(str(script))} {{workspace}} {{prompt_file}} {{output}}" + adapter = suite.make_adapter( + self.context( + **{ + "executor.name": "command", + "executor.command": command, + "executor.editable": ["instructions.txt"], + } + ) + ) + (self.project / "instructions.txt").write_text("Use the source.\n") + run_dir = adapter.execute("files", None, "files") + self.assertTrue(adapter.delivered(run_dir)) + self.assertEqual(adapter.rollout_cost(run_dir), 0) + self.assertEqual( + adapter.deliverable_files(run_dir), + { + "binary.dat": "(binary file, 2 bytes)", + "report.txt": "supplied source\n", + }, + ) + self.assertIn("supplied source", adapter.deliverable_text(run_dir)) + self.assertIn("Saved report.", adapter.trajectory(run_dir, obs_trunc=10000)) + self.assertTrue(adapter.score(run_dir, "files")["all_pass"]) + self.assertTrue(adapter.has_judge("files")) + shutil.rmtree(run_dir / "output") + self.assertFalse(adapter.delivered(run_dir)) + + def test_message_deliverable(self): + self.task("message", deliverable="message") + run_dir = self.adapter.execute("message", None, "message") + self.assertTrue(self.adapter.delivered(run_dir)) + self.assertEqual(self.adapter.deliverable_text(run_dir), "Finished the change.") + self.assertEqual( + self.adapter.deliverable_files(run_dir), {"final.md": "Finished the change."} + ) + self.assertFalse(self.adapter.has_judge("message")) + self.assertIsNone(self.adapter.score(run_dir, "message")) + self.assertFalse((run_dir / "judge.json").exists()) + + def test_check_judge_fresh_patch_json_exit_code_cache_and_failure(self): + task_dir = self.tasks_dir / "one" + (task_dir / "check").mkdir() + checker = task_dir / "check/probe.py" + checker.write_text( + "import os, pathlib\n" + "assert pathlib.Path('code.txt').read_text() == 'after\\n'\n" + "assert pathlib.Path('CLAUDE.md').read_text() == 'Read the code first.\\n'\n" + "assert pathlib.Path(os.environ['HARNESS2_RUN_DIR']).is_dir()\n" + "assert pathlib.Path(os.environ['HARNESS2_TASK_DIR']).is_dir()\n" + "print('detail')\n" + 'print(\'{"n_passed": 2, "n_criteria": 3}\')\n' + ) + with (task_dir / "task.toml").open("a") as stream: + stream.write(f"check = {json.dumps(shlex.quote(sys.executable) + ' check/probe.py')}\n") + self.assertTrue(self.adapter.has_judge("one")) + run_dir = self.adapter.execute("one", None, "judge") + result = self.adapter.score(run_dir, "one") + self.assertEqual((result["n_passed"], result["n_criteria"], result["exit_code"]), (2, 3, 0)) + self.assertFalse(result["all_pass"]) + with patch("harness2.suite.workspace.project_workspace") as create_workspace: + self.assertEqual(self.adapter.score(run_dir, "one"), result) + create_workspace.assert_not_called() + checker.write_text("raise RuntimeError('changed checker')") + changed = self.adapter.score(run_dir, "one") + self.assertEqual(changed["n_passed"], 0) + self.assertNotEqual(changed["identity"]["check_sha256"], result["identity"]["check_sha256"]) + for command, expected in ( + ("exit 0", 1), + ("exit 3", 0), + ('echo \'{"n_passed": true, "n_criteria": 1}\'; exit 4', 0), + ): + (task_dir / "task.toml").write_text(f"check = {json.dumps(command)}\n") + result = self.adapter.score(run_dir, "one") + self.assertEqual((result["n_passed"], result["n_criteria"]), (expected, 1)) + self.assertEqual(result["identity"]["check"], command) + (run_dir / "judge.json").unlink() + (run_dir / "deliverable.patch").write_text("not a patch") + self.assertIsNone(self.adapter.score(run_dir, "one")) + self.assertFalse((run_dir / "judge.json").exists()) + + def test_check_timeout_is_a_failed_judgment(self): + task_dir = self.task("slow", timeout=1) + command = f"exec {shlex.quote(sys.executable)} -c " + shlex.quote( + "import time; print('started', flush=True); time.sleep(2)" + ) + with (task_dir / "task.toml").open("a") as stream: + stream.write(f"check = {json.dumps(command)}\n") + run_dir = self.adapter.execute("slow", None, "slow") + result = self.adapter.score(run_dir, "slow") + self.assertEqual( + (result["n_passed"], result["n_criteria"], result["exit_code"]), (0, 1, -1) + ) + self.assertIn("started", result["output_tail"]) + + def test_check_cache_hashes_contents_names_and_empty_directories(self): + task_dir = self.task("check-identity", check="test -f check/ready") + check_dir = task_dir / "check" + check_dir.mkdir() + ready = check_dir / "ready" + ready.write_bytes(b"first\0content") + run_dir = self.adapter.execute("check-identity", None, "check-identity") + first = self.adapter.score(run_dir, "check-identity") + self.assertEqual(json.loads((run_dir / "judge.json").read_text()), first) + ready.write_bytes(b"second\0content") + second = self.adapter.score(run_dir, "check-identity") + self.assertNotEqual(first["identity"], second["identity"]) + ready.rename(check_dir / "renamed") + renamed = self.adapter.score(run_dir, "check-identity") + self.assertFalse(renamed["all_pass"]) + self.assertNotEqual(second["identity"], renamed["identity"]) + (check_dir / "empty").mkdir() + empty = self.adapter.score(run_dir, "check-identity") + self.assertNotEqual(renamed["identity"], empty["identity"]) + (check_dir / "compiled.pyc").write_bytes(b"compiled check") + compiled = self.adapter.score(run_dir, "check-identity") + self.assertNotEqual(empty["identity"], compiled["identity"]) + for legacy in ('{"n_passed": 1, "n_criteria": 1}', "invalid json", "[]"): + (run_dir / "judge.json").write_text(legacy) + self.assertEqual(self.adapter.score(run_dir, "check-identity"), compiled) + + def test_rubric_cache_regrades_after_model_channel_or_rubric_changes(self): + task_dir = self.tasks_dir / "one" + rubric = task_dir / "rubric.md" + rubric.write_text("Three criteria.\n") + run_dir = self.adapter.execute("one", None, "rubric-identity") + first = self.adapter.score(run_dir, "one") + self.assertEqual(first["identity"]["channel"], "claude") + self.assertEqual(first["identity"]["model"], "anthropic/fake") + calls = len(self.records()) + output = '{"n_passed": 3, "n_criteria": 3}' + with patch.dict(os.environ, {"FAKE_JUDGE": output}): + self.assertEqual(self.adapter.score(run_dir, "one"), first) + self.assertEqual(len(self.records()), calls) + changed_model = suite.make_adapter(self.context(**{"judge.model": "anthropic/other"})) + second = changed_model.score(run_dir, "one") + self.assertEqual(second["identity"]["model"], "anthropic/other") + self.assertTrue(second["all_pass"]) + self.assertEqual(len(self.records()), calls + 1) + self.assertEqual( + second["identity"]["rubric_sha256"], first["identity"]["rubric_sha256"] + ) + with patch.dict(os.environ, {"FAKE_JUDGE": '{"n_passed": 0, "n_criteria": 3}'}): + rubric.write_text("Three changed criteria.\n") + third = changed_model.score(run_dir, "one") + self.assertEqual(third["n_passed"], 0) + self.assertNotEqual( + third["identity"]["rubric_sha256"], second["identity"]["rubric_sha256"] + ) + changed_channel = suite.make_adapter( + self.context( + **{ + "judge.channel": "codex", + "judge.model": "openai/fake", + } + ) + ) + fourth = changed_channel.score(run_dir, "one") + self.assertEqual(fourth["identity"]["channel"], "codex") + self.assertEqual(fourth["identity"]["model"], "openai/fake") + self.assertEqual(fourth["n_passed"], 0) + self.assertEqual(len(self.records()), calls + 3) + + def test_rubric_judge_json_fence_bare_and_unparseable(self): + task_dir = self.tasks_dir / "one" + (task_dir / "rubric.md").write_text("Three criteria.\n") + self.assertTrue(self.adapter.has_judge("one")) + run_dir = self.adapter.execute("one", None, "rubric") + for text, expected in ( + ('{"n_passed": 2, "n_criteria": 3, "notes": "one missing"}', 2), + ( + 'Reasoning\n```json\n{"n_passed": 0, "n_criteria": 3}\n```\n' + '```json\n{"n_passed": 3, "n_criteria": 3}\n```', + 3, + ), + ('Reasoning\n{"n_passed": 1, "n_criteria": 3}', 1), + ('{"n_passed": 4, "n_criteria": 3}', None), + ("not json", None), + ): + with self.subTest(text=text), patch.dict(os.environ, {"FAKE_JUDGE": text}): + (run_dir / "judge.json").unlink(missing_ok=True) + result = self.adapter.score(run_dir, "one") + if expected is None: + self.assertIsNone(result) + self.assertFalse((run_dir / "judge.json").exists()) + else: + self.assertEqual(result["n_passed"], expected) + + def test_base_harness_is_stored_warns_once_and_survives_relocation(self): + base = self.adapter.base_harness() + original = tree.read_tree(self.adapter.substrate.surface(), base) + (self.project / "CLAUDE.md").write_text("changed harness\n") + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.assertEqual(self.adapter.base_harness(), base) + self.adapter.base_harness() + self.assertEqual(len(caught), 1) + self.assertEqual(str(caught[0].message), suite.BASE_DRIFT_WARNING.format(path=base)) + self.assertEqual(tree.read_tree(self.adapter.substrate.surface(), base), original) + self.adapter.execute("one", None, "stored-base") + self.assertEqual(self.records()[-1]["harness"]["CLAUDE.md"], original["CLAUDE.md"]) + other = suite.make_adapter(self.ctx) + with self.assertWarns(UserWarning): + self.assertEqual(other.base_harness(), base) + new_root = self.project / ".harness2/runs/moved" + shutil.move(self.ctx.run_root, new_root) + moved = suite.make_adapter(replace(self.ctx, run_root=new_root)) + with self.assertWarns(UserWarning): + self.assertEqual(moved.base_harness(), new_root / "base_harness") + (self.project / "CLAUDE.md").unlink() + empty = suite.make_adapter(replace(self.ctx, run_root=self.root / "empty")) + empty.base_harness() + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + empty.base_harness() + self.assertEqual(caught, []) + + def test_concurrent_base_harness_has_one_complete_snapshot(self): + with ThreadPoolExecutor(max_workers=4) as pool: + paths = list(pool.map(lambda _: self.adapter.base_harness(), range(8))) + self.assertEqual(len(set(paths)), 1) + self.assertEqual((paths[0] / "CLAUDE.md").read_text(), "Read the code first.\n") + + def test_task_validation_new_import_jsonl_csv_and_force(self): + self.assertEqual(suite.list_tasks(self.root / "missing"), []) + with self.assertRaisesRegex(ConfigError, "already exists"): + suite.new_task(self.tasks_dir, "one", files=False, domain="x", check=None) + for task_id in ("../escape", "Upper", "", "/absolute"): + with self.assertRaises(ConfigError): + suite.new_task(self.tasks_dir, task_id, files=False, domain="x", check=None) + source = self.root / "tasks.jsonl" + source.write_text( + "\n".join( + json.dumps(row) + for row in ( + { + "id": "two", + "prompt": "Second request", + "domain": "docs", + "split": "test", + "workspace": "files", + "deliverable": "files", + "timeout": 10, + }, + {"id": "three", "prompt": "Third request", "check": "exit 0"}, + ) + ) + ) + self.assertEqual(suite.import_tasks(self.tasks_dir, source, force=False), ["two", "three"]) + self.assertEqual((self.tasks_dir / "two/task.md").read_text(), "Second request") + self.assertEqual(self.adapter.tasks("test"), ["two"]) + self.assertEqual(self.adapter.task_files("two"), self.tasks_dir / "two/files") + with self.assertRaisesRegex(ConfigError, "already exists"): + suite.import_tasks(self.tasks_dir, source, force=False) + self.assertEqual(suite.import_tasks(self.tasks_dir, source, force=True), ["two", "three"]) + source = self.root / "tasks.csv" + with source.open("w", newline="") as stream: + writer = csv.DictWriter( + stream, fieldnames=["id", "prompt", "timeout", "split", "check"] + ) + writer.writeheader() + writer.writerow({"id": "four", "prompt": "First line\nSecond line", "timeout": 12}) + self.assertEqual(suite.import_tasks(self.tasks_dir, source, force=False), ["four"]) + self.assertEqual((self.tasks_dir / "four/task.md").read_text(), "First line\nSecond line") + self.assertEqual(suite._read_task(self.tasks_dir, "four")["timeout"], 12) + for values in ( + {"unknown": 1}, + {"split": "all"}, + {"workspace": "bad"}, + {"deliverable": "bad"}, + {"timeout": 0}, + {"timeout": True}, + {"domain": 1}, + ): + with self.subTest(values=values): + (self.tasks_dir / "four/task.toml").write_text( + "".join(f"{k} = {json.dumps(v)}\n" for k, v in values.items()) + ) + with self.assertRaises(ConfigError): + suite.list_tasks(self.tasks_dir) + (self.tasks_dir / "four/task.toml").write_text("bad toml") + with self.assertRaises(ConfigError): + suite.list_tasks(self.tasks_dir) + source = self.root / "bad.jsonl" + for content in ( + '{"id": "bad"}', + '{"id": "../escape", "prompt": "x"}', + "[1]", + "not json", + '{"id":"dup", "prompt":"x"}\n{"id":"dup", "prompt":"y"}', + ): + source.write_text(content) + with self.assertRaises(ConfigError): + suite.import_tasks(self.tasks_dir, source, force=False) + self.assertFalse((self.tasks_dir / "dup").exists()) + + def test_force_import_rejects_symlinks_and_malformed_csv(self): + outside = self.root / "outside.md" + outside.write_text("keep this") + (self.tasks_dir / "one/task.md").unlink() + (self.tasks_dir / "one/task.md").symlink_to(outside) + source = self.root / "force.jsonl" + source.write_text('{"id": "one", "prompt": "replace"}') + with self.assertRaisesRegex(ConfigError, "symlink"): + suite.import_tasks(self.tasks_dir, source, force=True) + self.assertEqual(outside.read_text(), "keep this") + source = self.root / "missing-column.csv" + source.write_text("id,prompt,timeout\nnew,request\n") + with self.assertRaisesRegex(ConfigError, "timeout must be a positive integer"): + suite.import_tasks(self.tasks_dir, source, force=False) + + def test_preflight_reports_all_invalid_tasks_and_config(self): + self.assertEqual(suite.check(self.ctx), []) + self.task("template") + (self.tasks_dir / "template/task.md").write_text(suite.TASK_TEMPLATE) + self.task("invalid") + (self.tasks_dir / "invalid/task.toml").write_text('unknown = "key"\n') + problems = suite.check( + self.context(**{"executor.bin": "nonexistent-executor", "judge.model": "bad"}) + ) + self.assertTrue(any("binary not found" in p for p in problems)) + self.assertTrue(any("judge: model" in p for p in problems)) + self.assertTrue( + any("task template: task.md still holds the template text" == p for p in problems) + ) + self.assertTrue(any("unknown task.toml key 'unknown'" in p for p in problems)) + human = self.context(**{"selector.channel": "human"}) + self.assertEqual(suite.make_adapter(human).spec.channels["selector"].kind, "human") + with self.assertRaisesRegex(ConfigError, "only allowed for selector"): + suite.make_adapter(self.context(**{"improver.channel": "human"})) + with self.assertRaisesRegex(ConfigError, "tasks suite runs"): + suite.make_adapter(self.context(**{"executor.name": "opencode"})) + with self.assertRaisesRegex(ConfigError, "non-empty editable"): + suite.make_adapter(self.context(**{"executor.name": "command"})) + non_git = self.root / "not-git" + non_git.mkdir() + problems = suite.check(replace(self.ctx, project=non_git)) + self.assertTrue(any("needs a git repository" in p for p in problems)) + + def test_full_two_task_schedule_k1_k2_and_resume(self): + self.task("two", domain="coding") + for k in (1, 2): + with self.subTest(k=k): + run_root = self.project / f".harness2/runs/k{k}" + adapter = suite.make_adapter(replace(self.ctx, run_root=run_root)) + proposer = Proposer(adapter.spec.channels["proposer"], audit_dir=run_root / "audit") + events.start(run_root / "events.jsonl") + kwargs = dict( + adapter=adapter, + prop=proposer, + runs_root=run_root, + library_root=run_root / "library", + modes=("agg", "seq"), + ) + result = schedule.run_domain("coding", adapter.tasks(), k, **kwargs) + self.assertEqual((result["done"], result["failed"]), (2, 0), result) + summaries = list(run_root.glob("tasks-*/**/summary.json")) + self.assertEqual(len(summaries), 2) + for path in summaries: + summary = json.loads(path.read_text()) + for mode in ("agg", "seq"): + ledger = summary["modes"][mode]["ledger"] + self.assertEqual((ledger["R"], ledger["P"]), (4 * k, 3 * k)) + count = len(self.records()) + resumed = schedule.run_domain("coding", adapter.tasks(), k, **kwargs) + self.assertEqual( + [row["status"] for row in resumed["tasks"]], ["skipped", "skipped"] + ) + self.assertEqual(len(self.records()), count) + records = [ + json.loads(line) + for line in (run_root / "events.jsonl").read_text().splitlines() + ] + rollouts = [row for row in records if row["kind"] == "rollout"] + self.assertTrue(rollouts) + self.assertTrue(all(row["cost_usd"] == 0.125 for row in rollouts)) + self.assertTrue(any(row["kind"] == "session" for row in records)) diff --git a/tests/test_ui.py b/tests/test_ui.py new file mode 100644 index 00000000..021f6190 --- /dev/null +++ b/tests/test_ui.py @@ -0,0 +1,210 @@ +"""The interactive commands: init, status, apply and the human selector.""" + +import argparse +import io +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from harness2.cli import apply, init, status +from harness2.core.spec import ChannelSpec + + +class _Ctx: + def __init__(self, project: Path, json: bool = False): + self.args = argparse.Namespace(json=json) + self.project = project + self.settings = type("S", (), {"path": lambda _self, key: project})() + self.logged: list[str] = [] + + def log(self, message): + self.logged.append(message) + + def out(self, data, text): + print(text(data)) + + +class UiTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name) + + def test_human_selector_returns_the_typed_choice(self): + from harness2.cli.human import HumanSelector + + context = self.root / "context" + (context / "candidates").mkdir(parents=True) + (context / "task.md").write_text("Fix the bug.\n") + for anon in ("A", "B"): + (context / "candidates" / f"{anon}.md").write_text(f"candidate {anon}\n") + selector = HumanSelector(ChannelSpec("human", "")) + with ( + patch.object(sys.stdin, "isatty", return_value=True), + patch("builtins.input", side_effect=["C", "B"]), + patch("sys.stderr"), + ): + reply = selector.call("mission", context, "select.agg2") + self.assertIn("```choice\nB\n```", reply.text) + + def test_apply_keeps_project_edits_made_since_the_run_started(self): + base = {"CLAUDE.md": "base\n", "a.md": "a\n", "b.md": "b\n"} + adopted = {"CLAUDE.md": "base\nrun rule\n", "a.md": "a run\n", "b.md": "b\n"} + current = {"CLAUDE.md": "base\nmy rule\n", "a.md": "a\n", "b.md": "b mine\n"} + written = {} + + def perform(settings, adapter, tree, *, base, branch): + written.update(tree) + return {"written": sorted(tree), "deleted": []} + + args = argparse.Namespace( + name=None, + task=None, + mode="parallel", + interactive=False, + yes=True, + branch=None, + dry_run=False, + force=False, + ) + with ( + patch.object(apply.A, "open_run", return_value=(None, self.root / "run")), + patch.object(apply.A, "adopted_tree", return_value=("t", adopted)), + patch.object(apply.A, "base_tree", return_value=base), + patch.object(apply.A, "project_tree", return_value=current), + patch.object(apply.A, "dirty_paths", return_value=[]), + patch.object(apply.A, "perform", side_effect=perform), + patch("sys.stdout"), + ): + self.assertEqual(apply.run(args, _Ctx(self.root)), 0) + # CLAUDE.md changed on both sides: skipped. a.md changed by the run only: applied. + # b.md changed by the project only: kept. + self.assertEqual( + written, {"CLAUDE.md": "base\nmy rule\n", "a.md": "a run\n", "b.md": "b mine\n"} + ) + + def test_apply_json_reports_the_plan_and_refuses_to_ask(self): + base, adopted = {"CLAUDE.md": "a\n"}, {"CLAUDE.md": "a\nb\n"} + args = argparse.Namespace( + name=None, task=None, mode="parallel", interactive=False, yes=False, + branch=None, dry_run=False, force=False, + ) # fmt: skip + patches = ( + patch.object(apply.A, "open_run", return_value=(None, self.root / "run")), + patch.object(apply.A, "harness_for", return_value=("consolidated", adopted)), + patch.object(apply.A, "base_tree", return_value=base), + patch.object(apply.A, "project_tree", return_value=dict(base)), + patch.object(apply.A, "dirty_paths", return_value=[]), + ) + for item in patches: + item.start() + self.addCleanup(item.stop) + out = io.StringIO() + with patch("sys.stdout", out): + self.assertEqual(apply.run(args, _Ctx(self.root, json=True)), 1) + self.assertIn("--yes or --dry-run", json.loads(out.getvalue())["error"]) + args.dry_run, out = True, io.StringIO() + with patch("sys.stdout", out), patch("sys.stderr"): + self.assertEqual(apply.run(args, _Ctx(self.root, json=True)), 0) + plan = json.loads(out.getvalue()) + self.assertEqual(plan["changes"][0]["path"], "CLAUDE.md") + self.assertFalse(plan["changes"][0]["applied"]) + + def test_login_status_uses_the_harness2_codex_home(self): + from harness2.cli import login + + homes = [] + ctx = _Ctx(self.root) + ctx.settings = type("S", (), {"provider": lambda _self, name: None})() + with ( + patch.object(login.providers, "codex_login_home", return_value=self.root / "h"), + patch.object(login.providers, "codex_logged_in", side_effect=homes.append), + patch.object(login, "_ok", return_value=True), + patch.object(login.shutil, "which", return_value="/bin/claude"), + patch("sys.stdout", io.StringIO()) as out, + ): + args = argparse.Namespace(service=None, device=False) + self.assertEqual(login.run(args, ctx), 0) + self.assertEqual(homes, [self.root / "h"]) + self.assertIn("claude-login", out.getvalue()) + self.assertIn("harness2 login codex", out.getvalue()) + + def test_guide_mentions_every_agent_contract(self): + from harness2.cli import guide + + for needle in ("--json", "--detach", "--max-calls", "harness2 stop", "apply", "130"): + self.assertIn(needle, guide.GUIDE) + + def test_status_reads_tasks_and_spend_from_events(self): + run = self.root / "run" + run.mkdir() + (run / "run.json").write_text( + json.dumps( + { + "name": "r", + "suite": "tasks", + "executor": "claude", + "settings": {}, + "schedule": {"k": 2, "modes": "parallel"}, + "created": "2026-01-01T00:00:00Z", + "stream": [["d", "t1"], ["d", "t2"], ["d", "t3"]], + } + ) + ) + events = [ + {"kind": "run_start", "t": "2026-01-01T00:00:01Z"}, + {"kind": "task_start", "task": "t1", "t": "2026-01-01T00:00:02Z"}, + {"kind": "rollout", "task": "t1", "cost_usd": 0.5, "t": "2026-01-01T00:00:03Z"}, + {"kind": "session", "cost_usd": 0.25, "cached": False, "t": "2026-01-01T00:00:04Z"}, + {"kind": "selection", "task": "t1", "mode": "agg", "chosen": "agg2"}, + {"kind": "task_done", "task": "t1", "status": "ok"}, + {"kind": "task_start", "task": "t2"}, + {"kind": "run_end", "status": "stopped", "reason": "max_calls"}, + ] + (run / "events.jsonl").write_text("\n".join(map(json.dumps, events)) + "\n{partial") + with patch.object( + status.C, "load_run", side_effect=lambda d: json.loads((d / "run.json").read_text()) + ): + snap = status.snapshot(run) + states = {row["task"]: row["state"] for row in snap["tasks"]} + self.assertEqual(states, {"t1": "done", "t2": "stopped", "t3": "pending"}) + self.assertEqual((snap["spent_usd"], snap["calls"]), (0.75, 2)) + self.assertEqual(snap["tasks"][0]["picked"], {"parallel": "agg2"}) + text = status.render(snap) + self.assertIn("1 done · 1 stopped · 1 pending", text) + self.assertIn("run ended: stopped (max_calls) · run the same command to resume", text) + + def test_init_writes_a_loadable_config_once(self): + bin_dir = self.root / "bin" + bin_dir.mkdir() + fake = bin_dir / "claude" + fake.write_text("#!/bin/sh\necho 'fake claude 1.0'\n") + fake.chmod(0o755) + project = self.root / "project" + project.mkdir() + args = argparse.Namespace( + project=str(project), yes=True, force=False, executor=None, auth=None + ) + env = { + "PATH": f"{bin_dir}{os.pathsep}{os.defpath}", + "XDG_CONFIG_HOME": str(self.root / "config"), + } + with patch.dict(os.environ, env, clear=True), patch("sys.stdout"): + self.assertEqual(init.run(args, None), 0) + from harness2 import settings + + loaded = settings.load(project) + self.assertEqual(loaded.get("executor.name"), "claude") + self.assertEqual(loaded.get("improver.model"), "claude-login/claude-opus-5-5") + self.assertTrue((project / ".harness2/tasks").is_dir()) + self.assertIn("runs/", (project / ".harness2/.gitignore").read_text()) + with patch("sys.stderr"): + self.assertEqual(init.run(args, None), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_workflow.py b/tests/test_workflow.py index 2b7ebeb1..52b11143 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -20,11 +20,11 @@ from pathlib import Path from unittest.mock import Mock, patch -from harness2 import config from harness2.core import context, judge, recursion, schedule, tree from harness2.core.library import Library from harness2.core.proposer import Proposer, ProposerError, Reply -from harness2.core.spec import resolve_channel_model +from harness2.core.runmark import resolve_path +from harness2.core.spec import ChannelSpec from harness2.demo import BASE_SCRIPT, DemoAdapter, DemoProposer @@ -48,13 +48,8 @@ def run_stream(self, k=2, proposer=None): ) def test_recursion_budget_and_resume(self): - with patch.dict(config.PROPOSER_MODEL_BY_CHANNEL, {"opencode": "", "claude_cli": ""}): - unresolved = resolve_channel_model(None, None) - self.assertEqual(unresolved.model, "") - with self.assertRaises(ValueError): - Proposer(unresolved) - with self.assertRaises(ValueError): - resolve_channel_model("opencode", "") + with self.assertRaises(ValueError): + Proposer(ChannelSpec("opencode", "")) for k in (1, 2): with self.subTest(k=k): result = self.run_stream(k) @@ -74,7 +69,11 @@ def test_recursion_budget_and_resume(self): selection = summary["selections"][mode] expected = "Total: 85.15\n" if task_id == "ledger-1" else "Total: 20.25\n" self.assertEqual( - self.adapter.deliverable_text(Path(selection["chosen_run_dir"])), + self.adapter.deliverable_text( + resolve_path( + selection["chosen_run_dir"], self.adapter.spec.runs_root + ) + ), expected, ) executions, calls = self.adapter.execution_count, len(self.proposer._calls) @@ -101,7 +100,9 @@ def test_recursion_budget_and_resume(self): repaired = self.run_stream() self.assertEqual([row["status"] for row in repaired["tasks"]], ["ok", "skipped"]) chosen = json.loads(summary_path.read_text())["selections"]["agg"] - (Path(chosen["chosen_run_dir"]) / "answer.txt").unlink() + ( + resolve_path(chosen["chosen_run_dir"], self.adapter.spec.runs_root) / "answer.txt" + ).unlink() repaired = self.run_stream() self.assertEqual([row["status"] for row in repaired["tasks"]], ["ok", "skipped"]) @@ -138,7 +139,6 @@ def run_opencode(command, **kwargs): proposer = Proposer(self.proposer.spec) with ( - patch("harness2.core.proposer._opencode_dir", return_value=self.root), patch("harness2.core.proposer.subprocess.run", side_effect=run_opencode), ): for index in range(2): diff --git a/tests/test_workspace.py b/tests/test_workspace.py new file mode 100644 index 00000000..a423a4dc --- /dev/null +++ b/tests/test_workspace.py @@ -0,0 +1,94 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline workspace isolation, binary patches, and git cleanup.""" + +from __future__ import annotations + +import subprocess +import tempfile +import unittest +from pathlib import Path + +from harness2 import workspace +from harness2.settings import ConfigError + + +class WorkspaceTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name).resolve() + self.project = self.root / "project" + self.project.mkdir() + subprocess.run(["git", "init", "-q", str(self.project)], check=True) + (self.project / "code.txt").write_text("before\n") + (self.project / "CLAUDE.md").write_text("instructions\n") + workspace.commit_all(self.project, "initial") + + def test_project_diff_apply_and_cleanup(self): + with workspace.project_workspace(self.project, "HEAD") as ws: + self.assertFalse(ws.is_relative_to(self.project)) + workspace.commit_all(ws, "empty candidate") + base = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ws, text=True).strip() + (ws / "code.txt").write_text("after\n") + (ws / "new.bin").write_bytes(b"\0\xffbinary") + (ws / "CLAUDE.md").write_text("private changes\n") + (ws / ".harness2").mkdir() + (ws / ".harness2/secret").write_text("secret") + patch = workspace.diff_since(ws, base, ["CLAUDE.md", ".harness2"]) + self.assertIn("GIT binary patch", patch) + self.assertNotIn("CLAUDE.md", patch) + self.assertNotIn(".harness2", patch) + self.assertFalse(ws.exists()) + with workspace.project_workspace(self.project, "HEAD") as judge: + workspace.apply_patch(judge, patch) + workspace.apply_patch(judge, "") + self.assertEqual((judge / "code.txt").read_text(), "after\n") + self.assertEqual((judge / "new.bin").read_bytes(), b"\0\xffbinary") + self.assertEqual((judge / "CLAUDE.md").read_text(), "instructions\n") + with self.assertRaises(subprocess.CalledProcessError): + workspace.apply_patch(judge, "broken patch") + self.assertEqual((self.project / "code.txt").read_text(), "before\n") + listing = subprocess.check_output( + ["git", "worktree", "list", "--porcelain"], cwd=self.project, text=True + ) + self.assertEqual(listing.count("worktree "), 1) + + def test_glob_exclusions_and_cleanup_on_error(self): + with self.assertRaisesRegex(RuntimeError, "test"): + with workspace.project_workspace(self.project, "HEAD") as ws: + (ws / "CLAUDE.md").write_text("new instructions") + (ws / "code.txt").write_text("after") + patch = workspace.diff_since(ws, "HEAD", [":(glob)*.md"]) + self.assertNotIn("CLAUDE.md", patch) + self.assertIn("code.txt", patch) + raise RuntimeError("test") + self.assertFalse(ws.exists()) + for project, ref in ((self.project, "missing-ref"), (self.root, "HEAD")): + with self.assertRaises(ConfigError): + with workspace.project_workspace(project, ref): + self.fail("invalid project/ref accepted") + + def test_files_copy_output_and_cleanup(self): + src = self.root / "files" + src.mkdir() + (src / "data.csv").write_text("amount\n1\n") + with workspace.files_workspace(src) as ws: + self.assertTrue((ws / "output").is_dir()) + (ws / "data.csv").write_text("changed") + self.assertFalse(ws.exists()) + self.assertEqual((src / "data.csv").read_text(), "amount\n1\n") + with workspace.files_workspace(None) as ws: + self.assertEqual([p.name for p in ws.iterdir()], ["output"])