From 7d0ec1d23b240f3c0ebacf20873fa50426ded22e Mon Sep 17 00:00:00 2001 From: Bae Seongmin <83870598+3ae3ae@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:19:07 +0900 Subject: [PATCH 01/21] feat(prompts): add Claude Code and command harness surfaces (#1) Describe the editable surface of the new Claude Code executor (CLAUDE.md, .claude/skills, .claude/scripts, sub_agents installed as .claude/agents, .claude/hooks.json merged into .claude/settings.json) and of the generic command executor, whose surface is a list of path patterns. Codex harness scripts move to .agents/scripts so they cannot collide with a project's own scripts/ directory. The shared INV template now takes the script and skill levers per surface instead of hardcoding scripts/ and trigger lines. Co-authored-by: Claude Opus 5.5 --- harness2/core/prompts.py | 262 ++++++++++++++++++++++++++++++++++----- 1 file changed, 234 insertions(+), 28 deletions(-) diff --git a/harness2/core/prompts.py b/harness2/core/prompts.py index 9ccacf23..b38b5221 100644 --- a/harness2/core/prompts.py +++ b/harness2/core/prompts.py @@ -22,7 +22,7 @@ from __future__ import annotations import re -from dataclasses import dataclass +from dataclasses import dataclass, replace from .spec import BenchPrompts from .tree import SurfaceSpec @@ -204,7 +204,7 @@ def _tidy(text: str) -> str: | `.agents/skills//SKILL.md` | a reusable workflow | name+description advertised; the BODY is read ON DEMAND | | `.agents/skills//scripts/*` | a helper carried with its skill | run via bash | | `sub_agents//AGENT.md` | a delegated agent role | spawned in its OWN context, returns text | -| `scripts/.py` | a deterministic helper | run on demand via bash | +| `.agents/scripts/.py` | a deterministic helper | run on demand via bash | | `.codex/hooks.json` | lifecycle hooks | fire INSIDE the tool loop, on every matching tool call | The three that are easy to get wrong: @@ -229,7 +229,8 @@ def _tidy(text: str) -> str: "reason" sends that reason back to the model; `PreToolUse` can deny a call the same way. The only component that acts in the tool loop rather than in the prompt -- the only way to enforce something the model cannot skip. Prose can be stopped reading; a hook cannot. -Keep commands fast and non-interactive; put real logic in a `scripts/` helper the hook calls. +Keep commands fast and non-interactive; put real logic in a `.agents/scripts/` helper the hook +calls. Change only what a trajectory shows needs changing -- but WHERE you change something, the FORM matters as much as the content, and the cheap form is not the obvious one. @@ -352,10 +353,9 @@ def _lens(bp: BenchPrompts) -> str: - TOOL ERROR: distinguish bad input from timeout, missing capability, permission, or unread output. Give one bounded fallback and stop condition; never prescribe blind repetition. -- MECHANICAL REPETITION: use `scripts/.py` for repeated deterministic extraction, +- MECHANICAL REPETITION: use __SCRIPT_LEVER__ for repeated deterministic extraction, conversion or checking, with clear inputs, outputs and failure behavior. -- TRIGGERED WORKFLOW: use an on-demand skill for a narrow multi-step procedure, name its trigger - in always-on text, and carry any helper it requires. +- TRIGGERED WORKFLOW: __SKILL_LEVER__ - IGNORED INVARIANT: only repeated evidence that clear prose is skipped can justify a defensive __LOOP_LEVER__. It must not block valid completion. - ORDINARY SEQUENCING: prefer concise always-on guidance. Do not add code for a one-off mistake @@ -476,7 +476,7 @@ def _surface_part(surface_md: str, start: str, end: str | None = None) -> str: | `.agents/skills//SKILL.md` | c6 reusable workflow | name+description advertised; the BODY is read ON DEMAND | | `.agents/skills//scripts/*` | helper carried with its skill | run via Bash | | `sub_agents//AGENT.md` | c5 delegated agent role | spawned in its OWN context, returns text | -| `scripts/.py` | c7 deterministic helper | run ON DEMAND via Bash as `python scripts/.py` | +| `.agents/scripts/.py` | c7 deterministic helper | run ON DEMAND via Bash as `python .agents/scripts/.py` | | `.codex/hooks.json` | c8 lifecycle hooks | fire INSIDE the tool loop, on every matching tool call | ## The two on-demand levers that are easy to get wrong @@ -494,8 +494,8 @@ def _surface_part(surface_md: str, start: str, end: str | None = None) -> str: the event as JSON on stdin; a `PostToolUse` command printing a JSON "decision" of "block" with a "reason" sends that reason back to the model. Unlike everything else here it acts in the tool loop rather than in the prompt, which is the only way to enforce something the model cannot -skip. Keep commands fast and non-interactive; put real logic in a `scripts/` helper the hook -calls. +skip. Keep commands fast and non-interactive; put real logic in a `.agents/scripts/` helper +the hook calls. Both levers are rare in the harnesses adopted so far. Read that as a gap in what has been tried, not as a verdict -- but still add one only where a trajectory shows the failure it would fix. @@ -508,7 +508,7 @@ def _surface_part(surface_md: str, start: str, end: str | None = None) -> str: one is a harness that does not exist. - A skill's literal directory name must appear in `AGENTS.md` as a trigger, or the skill is never opened. Scripts run via Bash. -- `scripts/*.py` filenames must be snake_case and must compile under py_compile. +- `.agents/scripts/*.py` filenames must be snake_case and must compile under py_compile. - Never write `harness.yaml` or other runtime-owned files: they are not on the editable surface.""" @@ -516,7 +516,7 @@ def _surface_part(surface_md: str, start: str, end: str | None = None) -> str: _TTS_INV = """\ # YOUR DIMENSION: INVARIANT -- how the agent works, independent of the subject -You own the machinery: tool guidance, `scripts/*.py`, `sub_agents/*/AGENT.md`, the tool-loop +You own the machinery: tool guidance, `__SCRIPT_DIR__/*.py`, `sub_agents/*/AGENT.md`, the tool-loop lever, and SHORT imperative always-on rules that hold for essentially EVERY task in this {domain} (output discipline, verification order, format mechanics, budget and finalization). @@ -526,7 +526,7 @@ def _surface_part(surface_md: str, start: str, end: str | None = None) -> str: Pick the cheapest mechanism that fixes what the trajectory shows: - the model fumbled a format or a library repeatedly -> a tool-guidance note, or a small - deterministic `scripts/*.py` helper it can call instead of improvising + deterministic `__SCRIPT_DIR__/*.py` helper it can call instead of improvising - a required step is skipped under time pressure -> ONE always-on rule, stated in an imperative line (every token here competes with the {deliverable} -- if you add, cut) - a bounded sub-task drowned the context and returned @@ -604,6 +604,132 @@ def _surface_part(surface_md: str, start: str, end: str | None = None) -> str: context for nothing.""" +_SURFACE_CLAUDE_HEAD = """\ +# WHAT YOU CAN CHANGE + +The harness is a tree of components. Each has a different runtime cost and a different reach: + +| path | what it is | when it is in context | +|---|---|---| +| `CLAUDE.md` | role, procedure, rules, durable facts | ALWAYS ON -- loaded whole into every session; every token competes with the agent's own work | +| `.claude/skills//SKILL.md` | a reusable workflow | name+description always listed; the BODY is read ON DEMAND when the agent invokes the skill | +| `.claude/skills//scripts/*` | a helper carried with its skill | run via Bash | +| `sub_agents//AGENT.md` | a delegated subagent, installed as `.claude/agents/.md` | runs in its OWN context, returns text | +| `.claude/scripts/.py` | a deterministic helper | run on demand via Bash | +| `.claude/hooks.json` | lifecycle hooks, installed into `.claude/settings.json` | fire INSIDE the tool loop, on every matching event | + +The three that are easy to get wrong: + +`.claude/skills/` -- YAML frontmatter with a `description:` is REQUIRED; a `name:` line is +optional and must match the directory. Only the name and description are shown up front, and +the agent decides from the description alone whether to open the skill -- so the description IS +the trigger: name the situation the skill handles, precisely and narrowly. If a skill calls a +helper, the helper must travel with it. + +`sub_agents/` -- frontmatter needs a `description:` saying when to delegate; an optional +`tools:` line (comma-separated tool names) narrows what it may use. It sees only what its own +file and the delegation message say, never the main conversation, and returns text. Worth it +for exactly one shape of problem: a bounded sub-task whose INPUT is large and whose ANSWER is +small. A bad trade when the output is as big as the input. + +`.claude/hooks.json` -- ONE JSON file: a top-level "hooks" object maps an event -- +`PreToolUse`, `PostToolUse`, `Stop`, `SessionStart` -- to matcher entries (a "matcher" such as +"Bash" or "Edit|Write", plus a "hooks" list of "command"-type entries). Each hook is a shell +command receiving the event as JSON on stdin (`tool_name`, `tool_input`; PostToolUse adds +`tool_response`). A command that exits with status 2 blocks the step and its stderr is shown to +the model: a `PreToolUse` hook refuses the call, a `PostToolUse` hook reports a problem with +what just happened, a `Stop` hook sends the agent back to work. The only component that acts in +the tool loop rather than in the prompt -- the only way to enforce something the model cannot +skip. Prose can be stopped reading; a hook cannot. Keep commands fast and non-interactive; put +real logic in a `.claude/scripts/` helper the hook calls.""" + + +_SURFACE_CLAUDE_PREFER = """\ +So prefer, in this order: + - a SKILL for anything with a recognisable trigger -- a procedure for one kind of change or + document, a recurring situation, a tool the work keeps needing. This is the default form + for new instruction. + - a SCRIPT for a deterministic operation the runs hand-rolled and got wrong, or did slowly. + - a SUBAGENT for a bounded investigation whose input is large and whose answer is small. + - a HOOK when a recurring, mechanical problem in the tool loop is better handled there than + in prose. It acts inside the loop, so it cannot be ignored AND it costs nothing in always-on + budget -- which makes it the cheap way to carry knowledge that prose would have to repeat. + The recurring shapes: running a fast check after an edit and reporting what broke; blocking + a known-bad command with a reason that teaches the fix; and refusing to stop while a + required step is still missing. + - ALWAYS-ON TEXT last, and only for a short rule that genuinely applies to every run. If you + add always-on text, cut something else to pay for it. + +EVERY SCRIPT YOU NAME MUST EXIST. A `.claude/scripts/.py` invoked from `CLAUDE.md`, a +skill or a hook must ship in the same proposal, and a skill's own helper goes at +`.claude/skills//scripts/.py`. A helper placed directly in the skill +directory is OFF THE EDITABLE SURFACE and silently dropped, leaving the skill invoking a file +that does not exist. Never write `.claude/settings.json` or `.claude/agents/` directly: the +runner builds them from `.claude/hooks.json` and `sub_agents/`.""" + + +_TTS_SURFACE_HEAD_CLAUDE = """\ +# EDITABLE HARNESS SURFACE + +You may write ONLY these paths. Anything else is off the editable surface: it is dropped before +your work is used, however good it is. + +| path | component | when it is in context | +|---|---|---| +| `CLAUDE.md` | c1/c3/c4/c6 persona, rules, durable facts, tool prose | ALWAYS ON -- loaded whole into every session | +| `.claude/skills//SKILL.md` | c6 reusable workflow | name+description listed; the BODY is read ON DEMAND | +| `.claude/skills//scripts/*` | helper carried with its skill | run via Bash | +| `sub_agents//AGENT.md` | c5 delegated subagent | run ON DEMAND in its OWN context, returns text | +| `.claude/scripts/.py` | c7 deterministic helper | run ON DEMAND via Bash as `python3 .claude/scripts/.py` | +| `.claude/hooks.json` | c8 lifecycle hooks | fire INSIDE the tool loop, on every matching event | + +## The two on-demand levers that are easy to get wrong + +`sub_agents//AGENT.md` -- frontmatter needs a `description:` saying when to delegate. It +runs in a SEPARATE context: it sees only what its own file and the delegation message say, +never the main conversation, and it returns text. Worth it for exactly one shape of problem -- +a bounded sub-task whose inputs are large and whose ANSWER is small. A bad trade when the +output is as big as the input. + +`.claude/hooks.json` -- ONE JSON file: a top-level "hooks" object maps an event (`PreToolUse`, +`PostToolUse`, `Stop`, `SessionStart`) to matcher entries -- a "matcher" such as "Bash" plus a +"hooks" list of "command"-type entries. Each hook is a shell command receiving the event as +JSON on stdin; one that exits with status 2 blocks the step and shows its stderr to the model. +Unlike everything else here it acts in the tool loop rather than in the prompt, which is the +only way to enforce something the model cannot skip. Keep commands fast and non-interactive; +put real logic in a `.claude/scripts/` helper the hook calls. + +Both levers are rare in the harnesses adopted so far. Read that as a gap in what has been tried, +not as a verdict -- but still add one only where a trajectory shows the failure it would fix. + +Hard rules, each one learned from a previous failure: +- The single most common failure of this harness family is producing NO deliverable because the + model exhausted its budget on reasoning. Always-on prose makes that WORSE. Prefer an + on-demand skill over new always-on prose; if you add always-on text, cut something else -- + `CLAUDE.md` is capped. +- A skill's description is its trigger: the agent opens a skill only when the description + matches the situation in front of it. Scripts run via Bash. +- `.claude/scripts/*.py` filenames must be snake_case and must compile under py_compile.""" + + +_SURFACE_COMMAND_HEAD = """\ +# WHAT YOU CAN CHANGE + +The harness is the set of files in `context/harness/`: whatever the agent being tuned reads at +run time -- prompt text, configuration, helper code. These path patterns are the whole editable +surface: + +__EDITABLE__ + +A file you create counts only if its path matches one of those patterns; anything else is +dropped before your work is used, however good it is. Where other parts of these instructions +name skills, scripts, subagents or hooks, those levers exist here only as files the patterns +allow. + +Change only what a trajectory shows needs changing. Text the agent reads on every run is paid +for on every run, competing with the agent's own work: if you add some, cut something else.""" + + @dataclass(frozen=True) class _SurfaceText: """The per-substrate prompt fragments, resolved once at module load.""" @@ -612,6 +738,20 @@ class _SurfaceText: inv: str dom: str tts_surface: str + script_dir: str = "scripts" + + +def _levers(template: str, **values: str) -> str: + """Fill the `__NAME__` slots of a shared INV/DOM template for one surface.""" + for name, value in values.items(): + template = template.replace(f"__{name.upper()}__", value) + return template + + +_SKILL_TRIGGER_LEVER = ( + "use an on-demand skill for a narrow multi-step procedure, name its trigger\n" + " in always-on text, and carry any helper it requires." +) def _build_surface_texts() -> dict[str, _SurfaceText]: @@ -629,34 +769,91 @@ def _build_surface_texts() -> dict[str, _SurfaceText]: _surface_part(_SURFACE_CODEX, "# HOW TO MAKE A CHANGE"), ] ) + shape_and_how = _surface_part(_SURFACE_CODEX, "THE DELIVERABLE'S SHAPE") + claude_md = "\n\n".join( + [ + _SURFACE_CLAUDE_HEAD, + _surface_part(_SURFACE_CODEX, "Change only what a trajectory", "So prefer, in this"), + _SURFACE_CLAUDE_PREFER, + shape_and_how, + ] + ) + claude_tts = "\n\n".join( + [ + _TTS_SURFACE_HEAD_CLAUDE, + _surface_part(claude_md, "EVERY SCRIPT YOU NAME", "THE DELIVERABLE'S SHAPE"), + _surface_part(claude_md, "# HOW TO MAKE A CHANGE"), + ] + ) + command_md = "\n\n".join([_SURFACE_COMMAND_HEAD, shape_and_how]) return { "opencode": _SurfaceText( surface_md=_SURFACE_OPENCODE, - inv=_INV_TEMPLATE.replace( - "__LOOP_LEVER__", - "`plugin/.ts` hook -- for example, preventing exit before a " + inv=_levers( + _INV_TEMPLATE, + script_lever="`scripts/.py`", + skill_lever=_SKILL_TRIGGER_LEVER, + loop_lever="`plugin/.ts` hook -- for example, preventing exit before a " "{deliverable} exists", ), - dom=_DOM_TEMPLATE.replace( - "__TRIGGER_SENTENCE__", - "A skill exists only once its name appears in the always-on text: write that " - "trigger line in the same edit.", + dom=_levers( + _DOM_TEMPLATE, + trigger_sentence="A skill exists only once its name appears in the always-on " + "text: write that trigger line in the same edit.", ), tts_surface=oc_tts, ), "codex": _SurfaceText( surface_md=_SURFACE_CODEX, - inv=_INV_TEMPLATE.replace( - "__LOOP_LEVER__", - "hook in `.codex/hooks.json` -- for example, blocking a finish before a " - "{deliverable} exists, with a corrective reason", + inv=_levers( + _INV_TEMPLATE, + script_lever="`.agents/scripts/.py`", + skill_lever=_SKILL_TRIGGER_LEVER, + loop_lever="hook in `.codex/hooks.json` -- for example, blocking a finish before " + "a {deliverable} exists, with a corrective reason", ), - dom=_DOM_TEMPLATE.replace( - "__TRIGGER_SENTENCE__", - "A skill exists only once its name appears in `AGENTS.md` as a trigger: write " - "that trigger line in the same edit.", + dom=_levers( + _DOM_TEMPLATE, + trigger_sentence="A skill exists only once its name appears in `AGENTS.md` as a " + "trigger: write that trigger line in the same edit.", ), tts_surface=cx_tts, + script_dir=".agents/scripts", + ), + "claude": _SurfaceText( + surface_md=claude_md, + inv=_levers( + _INV_TEMPLATE, + script_lever="`.claude/scripts/.py`", + skill_lever="use an on-demand skill for a narrow multi-step procedure, write its\n" + " description as its trigger, and carry any helper it requires.", + loop_lever="hook in `.claude/hooks.json` -- for example, a `Stop` hook that " + "refuses to finish before a {deliverable} exists, with a corrective reason", + ), + dom=_levers( + _DOM_TEMPLATE, + trigger_sentence="The agent finds a skill by its description alone: write the " + "description as the situation the skill handles.", + ), + tts_surface=claude_tts, + script_dir=".claude/scripts", + ), + "command": _SurfaceText( + surface_md=command_md, + inv=_levers( + _INV_TEMPLATE, + script_lever="a helper file, where the editable patterns allow one,", + skill_lever="keep a narrow multi-step procedure in a file the agent opens\n" + " only when it applies, if the patterns allow one, and say in always-on text " + "when to open it.", + loop_lever="check in whatever runtime configuration the editable patterns include", + ), + dom=_levers( + _DOM_TEMPLATE, + trigger_sentence="Knowledge in a file the agent opens on demand is read only " + "when always-on text says when to open it: write that line in the same edit.", + ), + tts_surface=command_md, ), } @@ -669,12 +866,21 @@ def _st(surface: SurfaceSpec | str) -> _SurfaceText: got = _SURFACE_TEXTS.get(name) if got is None: raise ValueError(f"no prompt surface for {name!r}; known: {sorted(_SURFACE_TEXTS)}") + globs = getattr(surface, "editable_globs", ()) + if name == "command" and globs: + listing = "\n".join(f"- `{g}`" for g in globs) + got = replace( + got, + surface_md=got.surface_md.replace("__EDITABLE__", listing), + tts_surface=got.tts_surface.replace("__EDITABLE__", listing), + ) return got def _tts_propose(bp: BenchPrompts, dim: str, st: _SurfaceText, *, k: int, index: int) -> str: """propose(), in the tts pack's words.""" - block = (_TTS_INV if dim == "INV" else _TTS_DOM).format( + template = (_TTS_INV if dim == "INV" else _TTS_DOM).replace("__SCRIPT_DIR__", st.script_dir) + block = template.format( deliverable=bp.deliverable_word, domain=bp.domain_word, levers=(bp.inv_levers if dim == "INV" else bp.dom_levers).strip(), From 60252249b6273c19441088395ed6e260b1c4e554 Mon Sep 17 00:00:00 2001 From: Bae Seongmin <83870598+3ae3ae@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:27:54 +0900 Subject: [PATCH 02/21] feat: add layered settings, provider profiles and a suite registry (#2) settings.py resolves configuration from defaults, the global ~/.config/harness2/config.toml, the project's .harness2/config.toml, a named profile, HARNESS2_
__ environment variables and CLI overrides, and records where every value came from. It also finds the project root, loads .harness2/.env, and edits TOML files in place without dropping comments. providers.py turns a provider profile (anthropic, openai, vertex, bedrock) into the environment or config each agent CLI expects, so no code path assumes Vertex AI any more. Secrets are only ever referenced through api_key_env. registry.py finds task suites: the built-in "tasks" suite plus any installed through the harness2.suites entry-point group. Co-authored-by: Claude Opus 5.5 --- harness2/providers.py | 168 +++++++++++++++ harness2/registry.py | 49 +++++ harness2/settings.py | 437 ++++++++++++++++++++++++++++++++++++++++ tests/test_providers.py | 326 ++++++++++++++++++++++++++++++ tests/test_registry.py | 84 ++++++++ tests/test_settings.py | 427 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 1491 insertions(+) create mode 100644 harness2/providers.py create mode 100644 harness2/registry.py create mode 100644 harness2/settings.py create mode 100644 tests/test_providers.py create mode 100644 tests/test_registry.py create mode 100644 tests/test_settings.py diff --git a/harness2/providers.py b/harness2/providers.py new file mode 100644 index 00000000..9901ef27 --- /dev/null +++ b/harness2/providers.py @@ -0,0 +1,168 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Translate provider profiles into each model client's configuration.""" + +from __future__ import annotations + +import json +import os +from dataclasses import dataclass +from urllib.parse import urlsplit + +from .settings import ConfigError + +PROVIDER_TYPES = ("anthropic", "openai", "vertex", "bedrock") +STRIP_ENV = ( + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_BASE_URL", + "ANTHROPIC_MODEL", + "ANTHROPIC_BEDROCK_BASE_URL", + "ANTHROPIC_VERTEX_PROJECT_ID", + "CLAUDE_CODE_USE_BEDROCK", + "CLAUDE_CODE_USE_VERTEX", + "CLOUD_ML_REGION", + "OPENAI_BASE_URL", +) + + +@dataclass(frozen=True) +class Provider: + """A named endpoint with credentials referenced by environment variable.""" + + name: str + type: str + base_url: str = "" + api_key_env: str = "" + project: str = "" + region: str = "" + + +def split_model(ref: str) -> tuple[str, str]: + provider, separator, model = ref.partition("/") + if not separator or not provider or not model: + raise ConfigError( + f"model {ref!r} must be written provider/model, e.g. anthropic/claude-sonnet-5-5" + ) + return provider, model + + +def claude_env(p: Provider) -> dict[str, str]: + if p.type == "anthropic": + env = {} + if p.api_key_env: + env["ANTHROPIC_API_KEY"] = os.environ.get(p.api_key_env, "") + if p.base_url: + env["ANTHROPIC_BASE_URL"] = p.base_url + return env + if p.type == "vertex": + return { + "CLAUDE_CODE_USE_VERTEX": "1", + "ANTHROPIC_VERTEX_PROJECT_ID": p.project, + "CLOUD_ML_REGION": p.region or "global", + } + if p.type == "bedrock": + return {"CLAUDE_CODE_USE_BEDROCK": "1", **({"AWS_REGION": p.region} if p.region else {})} + if p.type == "openai": + raise ConfigError("Claude Code needs an Anthropic-compatible provider.") + raise ConfigError(f"unknown provider type {p.type!r}; use one of {PROVIDER_TYPES}") + + +def codex_config(p: Provider, model: str, effort: str) -> str: + if p.type != "openai": + raise ConfigError( + "Codex needs an OpenAI Responses-compatible provider; serve one with " + "`harness2 bridge --model /` and point an openai-type provider at it." + ) + + def quote(value): + return json.dumps(value, ensure_ascii=False).replace("\x7f", "\\u007f") + + # Codex refuses to redefine its built-in provider ids (e.g. "openai"), so namespace ours. + pid = f"harness2-{p.name}" + return ( + f"model = {quote(model)}\n" + f"model_provider = {quote(pid)}\n" + f"model_reasoning_effort = {quote(effort)}\n\n" + f"[model_providers.{quote(pid)}]\n" + f"name = {quote(p.name)}\n" + f"base_url = {quote(p.base_url)}\n" + f"env_key = {quote(p.api_key_env)}\n" + 'wire_api = "responses"\n' + ) + + +def opencode_config(p: Provider, model: str) -> tuple[dict, dict[str, str]]: + if p.type == "vertex": + region = p.region or "global" + return {}, { + "GOOGLE_VERTEX_PROJECT": p.project, + "GOOGLE_VERTEX_LOCATION": region, + "VERTEXAI_LOCATION": region, + } + if p.type == "bedrock": + raise ConfigError("Bedrock is not supported on the opencode channel.") + if p.type not in ("openai", "anthropic"): + raise ConfigError(f"unknown provider type {p.type!r}; use one of {PROVIDER_TYPES}") + options = {} + if p.api_key_env or p.type == "openai": + options["apiKey"] = "{env:" + p.api_key_env + "}" + if p.base_url or p.type == "openai": + options["baseURL"] = p.base_url + npm = "@ai-sdk/openai-compatible" if p.type == "openai" else "@ai-sdk/anthropic" + return { + "provider": { + p.name: { + "npm": npm, + "name": p.name, + "options": options, + "models": {model: {}}, + } + } + }, {} + + +def litellm_params(p: Provider, model: str) -> dict: + if p.type in ("anthropic", "openai"): + params = {"model": f"{p.type}/{model}", "api_key": f"os.environ/{p.api_key_env}"} + if p.base_url or p.type == "openai": + params["api_base"] = p.base_url + return params + if p.type == "vertex": + return { + "model": f"vertex_ai/{model}", + "vertex_project": p.project, + "vertex_location": p.region or "global", + } + if p.type == "bedrock": + return {"model": f"bedrock/{model}", "aws_region_name": p.region} + raise ConfigError(f"unknown provider type {p.type!r}; use one of {PROVIDER_TYPES}") + + +def check(p: Provider) -> list[str]: + problems = [] + if p.api_key_env and not os.environ.get(p.api_key_env): + problems.append(f"{p.name}: set {p.api_key_env}") + if p.type == "vertex" and not p.project: + problems.append(f"{p.name}: set project") + if p.base_url: + try: + url = urlsplit(p.base_url) + valid = url.scheme in ("http", "https") and bool(url.netloc) + except ValueError: + valid = False + if not valid: + problems.append(f"{p.name}: base_url must be an HTTP(S) URL") + return problems diff --git a/harness2/registry.py b/harness2/registry.py new file mode 100644 index 00000000..62f8abd8 --- /dev/null +++ b/harness2/registry.py @@ -0,0 +1,49 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Discover suite modules without importing them until selected.""" + +from __future__ import annotations + +import importlib +import importlib.metadata +from dataclasses import dataclass +from pathlib import Path +from types import ModuleType + +from .settings import ConfigError, Settings + + +@dataclass(frozen=True) +class SuiteContext: + """The project and run settings supplied to a suite's adapter factory.""" + + settings: Settings + project: Path + run_root: Path + split_file: Path | None = None + + +def names() -> list[str]: + installed = importlib.metadata.entry_points(group="harness2.suites") + return ["tasks", *sorted({entry.name for entry in installed} - {"tasks"})] + + +def load(name: str) -> ModuleType: + if name == "tasks": + return importlib.import_module("harness2.suite") + for entry in importlib.metadata.entry_points(group="harness2.suites"): + if entry.name == name: + return entry.load() + raise ConfigError(f"unknown suite {name!r}; available suites: {', '.join(names())}") diff --git a/harness2/settings.py b/harness2/settings.py new file mode 100644 index 00000000..5a1fcb1a --- /dev/null +++ b/harness2/settings.py @@ -0,0 +1,437 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Layered project settings, provenance, and TOML edits.""" + +from __future__ import annotations + +import json +import os +import re +import tomllib +from collections.abc import Mapping +from copy import deepcopy +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from . import providers + +DEFAULTS = { + "suite": {"name": "tasks", "work": ""}, + "paths": {"harness": ".", "tasks": ".harness2/tasks", "runs": ".harness2/runs"}, + "executor": { + "name": "claude", + "model": "", + "bin": "", + "effort": "high", + "timeout": 1800, + "command": "", + "editable": [], + }, + "improver": { + "channel": "claude", + "model": "", + "bin": "", + "effort": "high", + "timeout": 3600, + "steps": 120, + "steer": "", + }, + "selector": {}, + "reflector": {}, + "judge": {}, + "run": { + "k": 1, + "modes": "parallel", + "seed": None, + "workers": 1, + "phase_width": 2, + "proposer_concurrency": 2, + "transport_attempts": 2, + "obs_trunc": 50000, + "budget_usd": 0.0, + "max_calls": 0, + }, + "providers": { + "anthropic": {"type": "anthropic", "base_url": "", "api_key_env": "ANTHROPIC_API_KEY"}, + "openai": { + "type": "openai", + "base_url": "https://api.openai.com/v1", + "api_key_env": "OPENAI_API_KEY", + }, + }, +} +_MISSING = object() + + +class ConfigError(RuntimeError): + """Invalid configuration; each message line carries the remediation.""" + + +def find_project(start: Path) -> Path | None: + """Find the nearest ancestor, including start, with a .harness2 directory.""" + start = start.resolve() + return next((p for p in (start, *start.parents) if (p / ".harness2").is_dir()), None) + + +def global_path() -> Path: + return ( + Path(os.environ.get("XDG_CONFIG_HOME") or Path.home() / ".config") / "harness2/config.toml" + ) + + +def project_path(project: Path) -> Path: + return project / ".harness2/config.toml" + + +def cache_dir() -> Path: + return Path(os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache") / "harness2" + + +def parse_value(raw: str) -> Any: + """Parse a TOML literal, falling back to an unquoted string.""" + try: + return tomllib.loads("v = " + raw)["v"] + except (tomllib.TOMLDecodeError, KeyError): + return raw + + +def _merge(data: dict, values: Mapping, origins: dict[str, str], origin: str, prefix="") -> None: + for key, value in values.items(): + dotted = f"{prefix}.{key}" if prefix else key + if isinstance(value, dict): + if not isinstance(data.get(key), dict): + data[key] = {} + _merge(data[key], value, origins, origin, dotted) + else: + data[key] = deepcopy(value) + origins[dotted] = origin + + +def _put(data: dict, key: str, value: Any, origins: dict[str, str], origin: str) -> None: + parts = key.split(".") + if not all(parts): + raise ConfigError(f"invalid setting key {key!r}; use a dotted key") + nested = value + for part in reversed(parts): + nested = {part: nested} + _merge(data, nested, origins, origin) + + +def _validate(data: dict) -> None: + from . import providers # providers imports ConfigError from here + + problems = [] + + def secrets(value): + if isinstance(value, dict): + if "api_key" in value: + problems.append( + "store secrets in environment variables and reference them with api_key_env" + ) + for child in value.values(): + secrets(child) + elif isinstance(value, list): + for child in value: + secrets(child) + + secrets(data) + for name, value in data.items(): + if name not in DEFAULTS and name != "profiles": + problems.append( + f"unknown section {name!r}; use one of {', '.join(DEFAULTS)} or profiles" + ) + elif not isinstance(value, dict): + problems.append(f"{name} must be a table") + run = data.get("run", {}) + if isinstance(run, dict): + for key in ("k", "workers", "phase_width", "proposer_concurrency", "transport_attempts"): + value = run.get(key) + if type(value) is not int or value <= 0: + problems.append(f"run.{key} must be a positive integer") + for key in ("budget_usd", "max_calls"): + value = run.get(key) + if type(value) not in (int, float) or not value >= 0: + problems.append(f"run.{key} must be a non-negative number") + configured = data.get("providers", {}) + if isinstance(configured, dict): + for name, value in configured.items(): + if not isinstance(value, dict): + problems.append(f"providers.{name} must be a table") + elif value.get("type") not in providers.PROVIDER_TYPES: + problems.append( + f"providers.{name}.type must be one of {', '.join(providers.PROVIDER_TYPES)}" + ) + if problems: + raise ConfigError("\n".join(dict.fromkeys(problems))) + + +@dataclass +class Settings: + """Resolved settings and the source of each stored value.""" + + project: Path | None + _data: dict = field(repr=False) + _origins: dict[str, str] = field(repr=False) + + def get(self, key: str, default=_MISSING) -> Any: + value = self._data + for part in key.split("."): + if not isinstance(value, dict) or part not in value: + if default is _MISSING: + raise KeyError(key) + return default + value = value[part] + return deepcopy(value) + + def origin(self, key: str) -> str: + self.get(key) + return self._origins[key] + + def flat(self) -> dict[str, Any]: + values = {} + + def visit(data, prefix=""): + for key, value in data.items(): + dotted = f"{prefix}.{key}" if prefix else key + if isinstance(value, dict): + visit(value, dotted) + else: + values[dotted] = deepcopy(value) + + visit(self._data) + return values + + def section(self, name: str) -> dict: + return self.get(name) + + def role(self, name: str) -> dict: + if name not in ("improver", "selector", "reflector", "judge"): + raise ConfigError(f"unknown role {name!r}; use improver, selector, reflector or judge") + values = self.section("improver") + if name == "reflector": + selector = self.role("selector") + if selector["channel"] != "human": + values.update(selector) + if name != "improver": + values.update(self.section(name)) + keys = ("channel", "model", "bin", "effort", "timeout", "steps") + if name == "improver": + keys += ("steer",) + return {key: values[key] for key in keys} + + def provider(self, name: str) -> providers.Provider: + from . import providers + + configured = self.section("providers") + if name not in configured: + raise ConfigError(f"unknown provider {name!r}; configure providers.{name}") + fields = ("type", "base_url", "api_key_env", "project", "region") + return providers.Provider( + name, **{k: v for k, v in configured[name].items() if k in fields} + ) + + def path(self, key: str) -> Path: + path = Path(self.get(key)).expanduser() + return ((self.project or Path.cwd()) / path).resolve() + + +def load( + project: Path | None, + *, + profile: str | None = None, + overrides: Mapping[str, tuple[Any, str]] = {}, +) -> Settings: + """Apply defaults, files, a profile, environment, and explicit overrides in order.""" + if project is not None: + project = project.resolve() + load_dotenv(project / ".harness2/.env") + data, origins = {}, {} + _merge(data, DEFAULTS, origins, "default") + paths = [global_path()] + if project is not None: + paths.append(project_path(project)) + for path in paths: + try: + content = path.read_text(encoding="utf-8") + except FileNotFoundError: + continue + except (OSError, UnicodeError) as e: + raise ConfigError(f"cannot read {path}: {e}") from e + try: + values = tomllib.loads(content) + except tomllib.TOMLDecodeError as e: + raise ConfigError(f"invalid TOML in {path}: {e}") from e + _merge(data, values, origins, str(path)) + profile = profile if profile is not None else os.environ.get("HARNESS2_PROFILE") + if profile: + profiles = data.get("profiles", {}) + if not isinstance(profiles, dict) or profile not in profiles: + raise ConfigError(f"unknown profile {profile!r}; configure profiles.{profile}") + if not isinstance(profiles[profile], dict): + raise ConfigError(f"profiles.{profile} must be a table") + _merge(data, deepcopy(profiles[profile]), origins, f"profile {profile}") + for name, raw in os.environ.items(): + if name.startswith("HARNESS2_") and "__" in name: + key = name.removeprefix("HARNESS2_").lower().replace("__", ".") + _put(data, key, parse_value(raw), origins, f"env {name}") + for key, (value, origin) in overrides.items(): + _put(data, key, value, origins, origin) + _validate(data) + return Settings(project, data, origins) + + +def _literal(value: Any) -> str: + if isinstance(value, str): + return json.dumps(value, ensure_ascii=False).replace("\x7f", "\\u007f") + if isinstance(value, bool): + return str(value).lower() + if type(value) in (int, float): + return str(value) + if isinstance(value, list) and all(not isinstance(v, list) for v in value): + return "[" + ", ".join(_literal(v) for v in value) + "]" + raise ConfigError("config values must be strings, booleans, numbers or lists of scalars") + + +def _toml_key(key: str) -> str: + return key if re.fullmatch(r"[A-Za-z0-9_-]+", key) else _literal(key) + + +def _comments(text: str) -> list[str]: + comments = [] + quote = "" + i = 0 + while i < len(text): + if quote: + if text[i] == "\\" and quote.startswith('"'): + i += 2 + continue + if text.startswith(quote, i): + i += len(quote) + if len(quote) == 3: + while i < len(text) and text[i] == quote[0]: + i += 1 + quote = "" + continue + elif text[i] in "\"'": + quote = text[i] * (3 if text.startswith(text[i] * 3, i) else 1) + i += len(quote) + continue + elif text[i] == "#": + end = text.find("\n", i) + end = len(text) if end == -1 else end + comments.append(text[i:end].rstrip("\r")) + i = end + i += 1 + return comments + + +def set_value(path: Path, key: str, value: Any) -> None: + """Replace or append a TOML assignment while retaining unrelated lines and comments.""" + parts = key.split(".") + if not all(parts): + raise ConfigError(f"invalid setting key {key!r}; use a dotted key") + literal = _literal(value) + try: + with path.open(encoding="utf-8", newline="") as source: + text = source.read() + except FileNotFoundError: + text = "" + except (OSError, UnicodeError) as e: + raise ConfigError(f"cannot read {path}: {e}") from e + try: + tomllib.loads(text) + except tomllib.TOMLDecodeError as e: + raise ConfigError(f"invalid TOML in {path}: {e}") from e + lines = text.splitlines(keepends=True) + newline = "\r\n" if "\r\n" in text else "\n" + table, leaf = tuple(parts[:-1]), parts[-1] + current = () + insert = 0 if not table else None + start = 0 + while start < len(lines): + end = start + 1 + while True: + try: + parsed = tomllib.loads("".join(lines[start:end])) + break + except tomllib.TOMLDecodeError: + end += 1 + if end > len(lines): + raise ConfigError(f"cannot edit {key!r} in {path}") from None + if lines[start].lstrip().startswith("["): + names = [] + node = parsed + while isinstance(node, dict) and len(node) == 1: + name, node = next(iter(node.items())) + names.append(name) + current = tuple(names) if isinstance(node, dict) else None + if current == table: + insert = end + if leaf in parsed and not lines[start].lstrip().startswith("["): + comments = _comments("".join(lines[start:end])) + indent = lines[start][: len(lines[start]) - len(lines[start].lstrip())] + replacement = f"{indent}{_toml_key(leaf)} = {literal}" + if comments: + replacement += " " + comments.pop() + replacement += newline + lines[start:end] = [*(c + newline for c in comments), replacement] + break + start = end + else: + assignment = f"{_toml_key(leaf)} = {literal}{newline}" + if insert is None: + header = ".".join(_toml_key(part) for part in table) + lines.append(f"{newline}[{header}]{newline}{assignment}") + else: + if insert and not lines[insert - 1].endswith("\n"): + assignment = newline + assignment + lines.insert(insert, assignment) + result = "".join(lines) + try: + tomllib.loads(result) + except tomllib.TOMLDecodeError as e: + raise ConfigError(f"cannot set {key!r} in {path}: {e}") from e + try: + encoded = result.encode("utf-8") + except UnicodeError as e: + raise ConfigError("config strings must contain valid Unicode") from e + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(encoded) + + +def load_dotenv(path: Path) -> None: + """Load KEY=VALUE lines with setdefault, using the original config module's parser.""" + try: + text = Path(path).read_text(encoding="utf-8") + except OSError: + return + for raw in text.splitlines(): + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + if line.startswith("export "): + line = line[len("export ") :].lstrip() + key, _, value = line.partition("=") + key = key.strip() + if not key or not all(c.isalnum() or c == "_" for c in key): + continue + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'): + value = value[1:-1] + elif " #" in value: + value = value.split(" #", 1)[0].rstrip() + os.environ.setdefault(key, value) diff --git a/tests/test_providers.py b/tests/test_providers.py new file mode 100644 index 00000000..96c5efc7 --- /dev/null +++ b/tests/test_providers.py @@ -0,0 +1,326 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Provider translations without network or model calls.""" + +from __future__ import annotations + +import os +import tomllib +import unittest +from dataclasses import FrozenInstanceError +from unittest.mock import patch + +from harness2 import providers +from harness2.providers import Provider +from harness2.settings import ConfigError + + +class ProviderTests(unittest.TestCase): + def test_model_split(self): + self.assertEqual(providers.split_model("anthropic/claude-x"), ("anthropic", "claude-x")) + self.assertEqual(providers.split_model("local/team/model"), ("local", "team/model")) + for ref in ("bare", "", "/model", "provider/"): + with self.subTest(ref=ref), self.assertRaises(ConfigError) as caught: + providers.split_model(ref) + self.assertEqual( + str(caught.exception), + (f"model {ref!r} must be written provider/model, e.g. anthropic/claude-sonnet-5-5"), + ) + + def test_provider_is_frozen_and_strip_env_is_exact(self): + provider = Provider("custom", "anthropic") + with self.assertRaises(FrozenInstanceError): + provider.region = "changed" + self.assertEqual(providers.PROVIDER_TYPES, ("anthropic", "openai", "vertex", "bedrock")) + self.assertEqual( + providers.STRIP_ENV, + ( + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_BASE_URL", + "ANTHROPIC_MODEL", + "ANTHROPIC_BEDROCK_BASE_URL", + "ANTHROPIC_VERTEX_PROJECT_ID", + "CLAUDE_CODE_USE_BEDROCK", + "CLAUDE_CODE_USE_VERTEX", + "CLOUD_ML_REGION", + "OPENAI_BASE_URL", + ), + ) + + def test_claude_anthropic_env_and_optional_values(self): + with patch.dict(os.environ, {"CUSTOM_KEY": "secret"}, clear=True): + self.assertEqual( + providers.claude_env( + Provider( + "custom", + "anthropic", + "https://example.com", + "CUSTOM_KEY", + ) + ), + {"ANTHROPIC_API_KEY": "secret", "ANTHROPIC_BASE_URL": "https://example.com"}, + ) + self.assertEqual(providers.claude_env(Provider("a", "anthropic")), {}) + self.assertEqual( + providers.claude_env(Provider("a", "anthropic", api_key_env="MISSING")), + { + "ANTHROPIC_API_KEY": "", + }, + ) + + def test_claude_vertex_and_bedrock_env(self): + self.assertEqual( + providers.claude_env(Provider("v", "vertex", project="project")), + { + "CLAUDE_CODE_USE_VERTEX": "1", + "ANTHROPIC_VERTEX_PROJECT_ID": "project", + "CLOUD_ML_REGION": "global", + }, + ) + self.assertEqual( + providers.claude_env( + Provider( + "v", + "vertex", + project="project", + region="us-east1", + ) + )["CLOUD_ML_REGION"], + "us-east1", + ) + self.assertEqual( + providers.claude_env(Provider("b", "bedrock")), + { + "CLAUDE_CODE_USE_BEDROCK": "1", + }, + ) + self.assertEqual( + providers.claude_env(Provider("b", "bedrock", region="us-east-1")), + { + "CLAUDE_CODE_USE_BEDROCK": "1", + "AWS_REGION": "us-east-1", + }, + ) + with self.assertRaisesRegex( + ConfigError, "Claude Code needs an Anthropic-compatible provider" + ): + providers.claude_env(Provider("o", "openai")) + + def test_codex_config_is_valid_toml_and_escapes_values(self): + provider = Provider('local."name 😀', "openai", "http://localhost:8000/v1", "LOCAL_KEY") + parsed = tomllib.loads(providers.codex_config(provider, 'model"\nnext 😀\x7f', "high")) + self.assertEqual( + parsed, + { + "model": 'model"\nnext 😀\x7f', + "model_provider": f"harness2-{provider.name}", + "model_reasoning_effort": "high", + "model_providers": { + f"harness2-{provider.name}": { + "name": provider.name, + "base_url": provider.base_url, + "env_key": "LOCAL_KEY", + "wire_api": "responses", + } + }, + }, + ) + for kind in ("anthropic", "vertex", "bedrock"): + with self.subTest(kind=kind), self.assertRaises(ConfigError) as caught: + providers.codex_config(Provider("p", kind), "model", "high") + self.assertEqual( + str(caught.exception), + ( + "Codex needs an OpenAI Responses-compatible provider; serve one with " + "`harness2 bridge --model /` and point an openai-type " + "provider at it." + ), + ) + + def test_opencode_openai_and_anthropic(self): + for kind, npm in ( + ("openai", "@ai-sdk/openai-compatible"), + ("anthropic", "@ai-sdk/anthropic"), + ): + with self.subTest(kind=kind): + provider = Provider("local", kind, "http://localhost:8000", "MY_KEY") + self.assertEqual( + providers.opencode_config(provider, "model"), + ( + { + "provider": { + "local": { + "npm": npm, + "name": "local", + "options": { + "baseURL": "http://localhost:8000", + "apiKey": "{env:MY_KEY}", + }, + "models": {"model": {}}, + } + } + }, + {}, + ), + ) + self.assertEqual( + providers.opencode_config(Provider("a", "anthropic"), "model")[0], + { + "provider": { + "a": { + "npm": "@ai-sdk/anthropic", + "name": "a", + "options": {}, + "models": {"model": {}}, + } + }, + }, + ) + self.assertEqual( + providers.opencode_config(Provider("o", "openai"), "model")[0]["provider"]["o"][ + "options" + ], + {"baseURL": "", "apiKey": "{env:}"}, + ) + + def test_opencode_vertex_and_bedrock(self): + for region in ("", "europe-west1"): + with self.subTest(region=region): + self.assertEqual( + providers.opencode_config( + Provider( + "v", + "vertex", + project="project", + region=region, + ), + "model", + ), + ( + {}, + { + "GOOGLE_VERTEX_PROJECT": "project", + "GOOGLE_VERTEX_LOCATION": region or "global", + "VERTEXAI_LOCATION": region or "global", + }, + ), + ) + with self.assertRaisesRegex( + ConfigError, "Bedrock is not supported on the opencode channel" + ): + providers.opencode_config(Provider("b", "bedrock"), "model") + + def test_litellm_params_for_each_type(self): + for kind in ("anthropic", "openai"): + with self.subTest(kind=kind): + self.assertEqual( + providers.litellm_params( + Provider( + "local", + kind, + "https://example.com", + "CUSTOM_KEY", + ), + "model", + ), + { + "model": f"{kind}/model", + "api_key": "os.environ/CUSTOM_KEY", + "api_base": "https://example.com", + }, + ) + self.assertEqual( + providers.litellm_params(Provider("local", kind), "model"), + { + "model": f"{kind}/model", + "api_key": "os.environ/", + **({"api_base": ""} if kind == "openai" else {}), + }, + ) + self.assertEqual( + providers.litellm_params( + Provider( + "v", + "vertex", + project="project", + region="us-east1", + ), + "model", + ), + { + "model": "vertex_ai/model", + "vertex_project": "project", + "vertex_location": "us-east1", + }, + ) + self.assertEqual( + providers.litellm_params(Provider("v", "vertex"), "model")["vertex_location"], "global" + ) + self.assertEqual( + providers.litellm_params(Provider("b", "bedrock", region="us-west-2"), "model"), + { + "model": "bedrock/model", + "aws_region_name": "us-west-2", + }, + ) + + def test_unknown_provider_types_raise_config_error(self): + for function, args in ( + (providers.claude_env, ()), + (providers.codex_config, ("model", "high")), + (providers.opencode_config, ("model",)), + (providers.litellm_params, ("model",)), + ): + with self.subTest(function=function.__name__), self.assertRaises(ConfigError): + function(Provider("p", "unknown"), *args) + + def test_doctor_checks_missing_env_project_and_invalid_urls(self): + with patch.dict(os.environ, {}, clear=True): + provider = Provider("v", "vertex", "ftp://example.com", "CUSTOM_KEY") + self.assertEqual( + providers.check(provider), + [ + "v: set CUSTOM_KEY", + "v: set project", + "v: base_url must be an HTTP(S) URL", + ], + ) + os.environ["CUSTOM_KEY"] = "" + self.assertEqual( + providers.check(Provider("a", "anthropic", api_key_env="CUSTOM_KEY")), + [ + "a: set CUSTOM_KEY", + ], + ) + os.environ["CUSTOM_KEY"] = "present" + self.assertEqual( + providers.check(Provider("a", "anthropic", api_key_env="CUSTOM_KEY")), [] + ) + for url in ("https://example.com/v1", "http://localhost:8080", ""): + with self.subTest(url=url): + self.assertEqual(providers.check(Provider("o", "openai", url)), []) + for url in ("file:///tmp/endpoint", "localhost:8080", "https://", "http://[bad"): + with self.subTest(url=url): + self.assertEqual( + providers.check(Provider("o", "openai", url)), + [ + "o: base_url must be an HTTP(S) URL", + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_registry.py b/tests/test_registry.py new file mode 100644 index 00000000..71dfd1c0 --- /dev/null +++ b/tests/test_registry.py @@ -0,0 +1,84 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Suite discovery with isolated, fake entry points.""" + +from __future__ import annotations + +import unittest +from dataclasses import FrozenInstanceError +from importlib.metadata import EntryPoint +from pathlib import Path +from types import ModuleType +from unittest.mock import patch + +from harness2 import registry +from harness2.settings import ConfigError + + +class RegistryTests(unittest.TestCase): + def test_builtin_is_listed_first_and_imported_lazily(self): + entries = [ + EntryPoint("zeta", "fake.zeta", "harness2.suites"), + EntryPoint("alpha", "fake.alpha", "harness2.suites"), + EntryPoint("tasks", "fake.tasks", "harness2.suites"), + ] + module = ModuleType("harness2.suite") + with ( + patch("importlib.metadata.entry_points", return_value=entries) as discover, + patch("importlib.import_module", return_value=module) as import_module, + ): + self.assertEqual(registry.names(), ["tasks", "alpha", "zeta"]) + import_module.assert_not_called() + self.assertIs(registry.load("tasks"), module) + import_module.assert_called_once_with("harness2.suite") + discover.assert_called_once_with(group="harness2.suites") + + def test_plugin_module_is_loaded_through_entry_point(self): + module = ModuleType("fake_suite") + module.make_adapter = lambda ctx: ctx.project + entry = EntryPoint("custom", "fake_suite", "harness2.suites") + with ( + patch("importlib.metadata.entry_points", return_value=[entry]) as discover, + patch.dict("sys.modules", {"fake_suite": module}), + ): + self.assertEqual(registry.names(), ["tasks", "custom"]) + self.assertIs(registry.load("custom"), module) + discover.assert_called_with(group="harness2.suites") + context = registry.SuiteContext(None, Path("/project"), Path("/project/runs/name")) + self.assertEqual(module.make_adapter(context), Path("/project")) + + def test_unknown_suite_lists_available_names(self): + entry = EntryPoint("example", "fake.example", "harness2.suites") + with patch("importlib.metadata.entry_points", return_value=[entry]): + with self.assertRaisesRegex( + ConfigError, "unknown suite 'missing'; available suites: tasks, example" + ): + registry.load("missing") + with patch("importlib.metadata.entry_points", return_value=[]): + self.assertEqual(registry.names(), ["tasks"]) + + def test_suite_context_is_frozen_with_optional_split_file(self): + context = registry.SuiteContext(None, Path("/project"), Path("/runs")) + self.assertIsNone(context.split_file) + with self.assertRaises(FrozenInstanceError): + context.project = Path("/changed") + split = Path("/split.json") + self.assertEqual( + registry.SuiteContext(None, Path("/p"), Path("/r"), split).split_file, split + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_settings.py b/tests/test_settings.py new file mode 100644 index 00000000..25b3fbc8 --- /dev/null +++ b/tests/test_settings.py @@ -0,0 +1,427 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline tests for layered settings and edits to project TOML files.""" + +from __future__ import annotations + +import os +import tempfile +import tomllib +import unittest +from pathlib import Path +from unittest.mock import patch + +from harness2 import providers, settings + + +class SettingsTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.project = self.root / "project" + (self.project / ".harness2").mkdir(parents=True) + self.global_file = self.root / "config/harness2/config.toml" + self.project_file = settings.project_path(self.project) + environment = patch.dict( + os.environ, {"XDG_CONFIG_HOME": str(self.root / "config")}, clear=True + ) + environment.start() + self.addCleanup(environment.stop) + + def write(self, path, text): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + def test_defaults_access_and_copy_isolation(self): + resolved = settings.load(self.project) + for section, values in settings.DEFAULTS.items(): + self.assertEqual(resolved.section(section), values) + self.assertEqual(resolved.project, self.project) + self.assertEqual(resolved.origin("run.k"), "default") + self.assertIsNone(resolved.get("run.seed")) + self.assertEqual(resolved.get("run.absent", 4), 4) + for key in ("absent", "run.absent", "run.k.absent"): + with self.subTest(key=key), self.assertRaises(KeyError): + resolved.get(key) + with self.assertRaises(KeyError): + resolved.origin(key) + self.assertEqual(resolved.flat()["executor.editable"], []) + self.assertNotIn("selector", resolved.flat()) + resolved.section("executor")["editable"].append("one.py") + resolved.flat()["executor.editable"].append("two.py") + self.assertEqual(resolved.get("executor.editable"), []) + self.assertEqual(settings.load(None).section("executor"), settings.DEFAULTS["executor"]) + + def test_precedence_and_origins_across_all_six_layers(self): + self.write(self.global_file, "[run]\nk = 2\nworkers = 2\n") + self.write( + self.project_file, + "[run]\nk = 3\nphase_width = 3\n[profiles.fast.run]\nk = 4\n", + ) + layers = [ + (settings.load(None), 2, str(self.global_file)), + (settings.load(self.project), 3, str(self.project_file)), + (settings.load(self.project, profile="fast"), 4, "profile fast"), + ] + os.environ["HARNESS2_RUN__K"] = "5" + layers.append((settings.load(self.project, profile="fast"), 5, "env HARNESS2_RUN__K")) + layers.append( + ( + settings.load( + self.project, + profile="fast", + overrides={"run.k": (6, "flag --k")}, + ), + 6, + "flag --k", + ) + ) + for resolved, value, origin in layers: + with self.subTest(origin=origin): + self.assertEqual(resolved.get("run.k"), value) + self.assertEqual(resolved.origin("run.k"), origin) + resolved = layers[-1][0] + self.assertEqual(resolved.origin("run.workers"), str(self.global_file)) + self.assertEqual(resolved.origin("run.phase_width"), str(self.project_file)) + self.assertEqual(resolved.origin("run.obs_trunc"), "default") + + def test_profiles_merge_files_and_explicit_profile_wins(self): + self.write( + self.global_file, + '[profiles.fast.improver]\nmodel = "anthropic/global"\neffort = "low"\n', + ) + self.write( + self.project_file, + '[profiles.fast.improver]\neffort = "medium"\n[profiles.slow.run]\nk = 7\n', + ) + os.environ["HARNESS2_PROFILE"] = "fast" + resolved = settings.load(self.project) + self.assertEqual(resolved.role("improver")["model"], "anthropic/global") + self.assertEqual(resolved.role("improver")["effort"], "medium") + self.assertEqual(resolved.origin("improver.model"), "profile fast") + self.assertEqual(settings.load(self.project, profile="slow").get("run.k"), 7) + with self.assertRaisesRegex(settings.ConfigError, "unknown profile 'missing'"): + settings.load(self.project, profile="missing") + self.write(self.project_file, '[profiles]\nbad = "invalid"\n') + with self.assertRaisesRegex(settings.ConfigError, "profiles.bad must be a table"): + settings.load(self.project, profile="bad") + + def test_env_values_and_dotted_overrides(self): + os.environ.update( + { + "HARNESS2_EXECUTOR__MODEL": "anthropic/model/submodel", + "HARNESS2_EXECUTOR__EDITABLE": '["a.py", "b.py"]', + "HARNESS2_PROVIDERS__LOCAL__TYPE": "openai", + "HARNESS2_PROVIDERS__LOCAL__BASE_URL": "http://localhost:8080/v1", + "HARNESS2_RUN__K": "3", + "HARNESS2_RUN__PLUGIN_BOOL": "true", + "HARNESS2_OLD_OPTION": "ignored", + } + ) + resolved = settings.load(self.project, overrides={"run.plugin.nested": ({"x": 1}, "-c")}) + self.assertEqual(resolved.get("executor.model"), "anthropic/model/submodel") + self.assertEqual(resolved.get("executor.editable"), ["a.py", "b.py"]) + self.assertEqual(resolved.get("run.k"), 3) + self.assertIs(resolved.get("run.plugin_bool"), True) + self.assertEqual(resolved.get("run.plugin.nested.x"), 1) + self.assertEqual(resolved.origin("run.plugin.nested.x"), "-c") + self.assertEqual(resolved.provider("local").base_url, "http://localhost:8080/v1") + self.assertNotIn("old_option", resolved.flat()) + + def test_parse_value(self): + for raw, value in ( + ("8", 8), + ("0.25", 0.25), + ("false", False), + ('"hello"', "hello"), + ('["one", 2, true]', ["one", 2, True]), + ("{x = 1}", {"x": 1}), + ("provider/model", "provider/model"), + ("", ""), + ("[broken", "[broken"), + ): + with self.subTest(raw=raw): + self.assertEqual(settings.parse_value(raw), value) + + def test_role_inheritance_and_human_selector(self): + resolved = settings.load( + self.project, + overrides={ + "improver.model": ("anthropic/root", "-c"), + "improver.steer": ("keep it small", "-c"), + "selector.effort": ("low", "-c"), + "selector.model": ("anthropic/selector", "-c"), + "reflector.steps": (4, "-c"), + "judge.timeout": (20, "-c"), + "judge.plugin_key": ("ignored by role", "-c"), + }, + ) + root = resolved.role("improver") + self.assertEqual(root["steer"], "keep it small") + self.assertEqual( + resolved.role("selector"), + { + key: value + for key, value in {**root, "effort": "low", "model": "anthropic/selector"}.items() + if key != "steer" + }, + ) + self.assertEqual(resolved.role("reflector"), {**resolved.role("selector"), "steps": 4}) + self.assertEqual( + resolved.role("judge"), + {key: value for key, value in {**root, "timeout": 20}.items() if key != "steer"}, + ) + human = settings.load( + self.project, + overrides={ + "selector.channel": ("human", "-c"), + "selector.model": ("unused", "-c"), + "reflector.timeout": (10, "-c"), + }, + ) + self.assertEqual(human.role("selector")["channel"], "human") + self.assertEqual(human.role("reflector")["channel"], "claude") + self.assertEqual(human.role("reflector")["model"], "") + self.assertEqual(human.role("reflector")["timeout"], 10) + with self.assertRaisesRegex(settings.ConfigError, "unknown role"): + resolved.role("executor") + + def test_provider_lookup_and_unknown_plugin_keys(self): + resolved = settings.load( + None, + overrides={ + "providers.local": ( + { + "type": "vertex", + "project": "my-project", + "region": "us-east1", + "plugin": 9, + }, + "-c", + ), + }, + ) + self.assertEqual( + resolved.provider("local"), + providers.Provider( + "local", + "vertex", + project="my-project", + region="us-east1", + ), + ) + with self.assertRaisesRegex(settings.ConfigError, "unknown provider 'absent'"): + resolved.provider("absent") + + def test_validation_aggregates_problems_and_rejects_secrets_anywhere(self): + with self.assertRaises(settings.ConfigError) as caught: + settings.load( + self.project, + overrides={ + "unexpected.key": (1, "-c"), + "executor.api_key": ("never-print-this", "-c"), + "run.k": (0, "-c"), + "run.workers": (True, "-c"), + "run.phase_width": (-1, "-c"), + "run.proposer_concurrency": (1.5, "-c"), + "run.transport_attempts": ("two", "-c"), + "run.budget_usd": (-0.1, "-c"), + "run.max_calls": (-1, "-c"), + "providers.local.type": ("invalid", "-c"), + }, + ) + message = str(caught.exception) + self.assertEqual(len(message.splitlines()), 10) + self.assertIn( + "store secrets in environment variables and reference them with api_key_env", message + ) + self.assertNotIn("never-print-this", message) + self.assertIn("unknown section 'unexpected'", message) + for name in ("k", "workers", "phase_width", "proposer_concurrency", "transport_attempts"): + self.assertIn(f"run.{name} must be a positive integer", message) + self.assertIn("run.budget_usd must be a non-negative number", message) + self.assertIn("run.max_calls must be a non-negative number", message) + self.assertIn("providers.local.type must be one of", message) + for key, value in ( + ("profiles.unused.executor.api_key", "hidden"), + ("executor.plugin", [{"api_key": "hidden"}]), + ): + with ( + self.subTest(key=key), + self.assertRaisesRegex(settings.ConfigError, "store secrets"), + ): + settings.load(None, overrides={key: (value, "-c")}) + + def test_malformed_sections_and_non_numeric_limits(self): + for key, value, message in ( + ("run", "invalid", "run must be a table"), + ("profiles", 3, "profiles must be a table"), + ("providers.local", 3, "providers.local must be a table"), + ("run.max_calls", "many", "run.max_calls must be a non-negative number"), + ("run.budget_usd", float("nan"), "run.budget_usd must be a non-negative number"), + ("run.budget_usd", False, "run.budget_usd must be a non-negative number"), + ): + with self.subTest(key=key), self.assertRaisesRegex(settings.ConfigError, message): + settings.load(None, overrides={key: (value, "-c")}) + with self.assertRaisesRegex(settings.ConfigError, "invalid setting key"): + settings.load(None, overrides={"run..k": (1, "-c")}) + + def test_invalid_toml_and_read_error(self): + self.write(self.project_file, "[broken") + with self.assertRaisesRegex(settings.ConfigError, "invalid TOML in"): + settings.load(self.project) + self.project_file.unlink() + self.project_file.mkdir() + with self.assertRaisesRegex(settings.ConfigError, "cannot read"): + settings.load(self.project) + + def test_locations_find_project_and_path_resolution(self): + self.assertEqual(settings.global_path(), self.global_file) + os.environ["XDG_CACHE_HOME"] = str(self.root / "cache") + self.assertEqual(settings.cache_dir(), self.root / "cache/harness2") + os.environ.pop("XDG_CACHE_HOME") + os.environ.pop("XDG_CONFIG_HOME") + with patch.object(Path, "home", return_value=self.root): + self.assertEqual(settings.global_path(), self.root / ".config/harness2/config.toml") + self.assertEqual(settings.cache_dir(), self.root / ".cache/harness2") + os.environ["XDG_CONFIG_HOME"] = "" + self.assertEqual(settings.global_path(), self.root / ".config/harness2/config.toml") + child = self.project / "src/deep" + child.mkdir(parents=True) + self.assertEqual(settings.find_project(child), self.project) + self.assertEqual(settings.find_project(self.project), self.project) + (child / ".harness2").mkdir() + self.assertEqual(settings.find_project(child), child) + (self.root / ".harness2").touch() + self.assertIsNone(settings.find_project(self.root)) + resolved = settings.load(self.project) + self.assertEqual(resolved.path("paths.tasks"), self.project / ".harness2/tasks") + absolute = settings.load( + self.project, overrides={"paths.runs": (str(self.root / "runs"), "-c")} + ) + self.assertEqual(absolute.path("paths.runs"), self.root / "runs") + self.assertEqual(settings.load(None).path("paths.harness"), Path.cwd().resolve()) + + def test_dotenv_parser_and_load_order(self): + dotenv = self.project / ".harness2/.env" + self.write( + dotenv, + ( + "# comment\n\nexport SHELL_VALUE = ignored\n" + "DOUBLE = \"two # literal\"\nSINGLE = 'one'\nPLAIN = value # note\n" + "EMPTY=\nINVALID-KEY=no\nno assignment\n = invalid\n" + "HARNESS2_RUN__K=4\nHARNESS2_PROFILE=fromenv\n" + ), + ) + self.write(self.project_file, "[profiles.fromenv.run]\nworkers = 2\n") + os.environ["SHELL_VALUE"] = "shell" + resolved = settings.load(self.project) + self.assertEqual(resolved.get("run.k"), 4) + self.assertEqual(resolved.get("run.workers"), 2) + self.assertEqual(resolved.origin("run.k"), "env HARNESS2_RUN__K") + self.assertEqual(os.environ["SHELL_VALUE"], "shell") + self.assertEqual(os.environ["DOUBLE"], "two # literal") + self.assertEqual(os.environ["SINGLE"], "one") + self.assertEqual(os.environ["PLAIN"], "value") + self.assertEqual(os.environ["EMPTY"], "") + self.assertNotIn("INVALID-KEY", os.environ) + settings.load_dotenv(self.root / "missing") + settings.load_dotenv(self.root) + + def test_set_value_preserves_comments_and_other_sections(self): + original = '# heading\n[run] # tuning\nk = 1 # count\n\n[executor]\nmodel = "original"\n' + self.write(self.project_file, original) + settings.set_value(self.project_file, "run.k", 3) + self.assertEqual(self.project_file.read_text(), original.replace("k = 1", "k = 3")) + settings.set_value(self.project_file, "run.workers", 2) + settings.set_value(self.project_file, "providers.local.type", "openai") + parsed = tomllib.loads(self.project_file.read_text()) + self.assertEqual(parsed["run"], {"k": 3, "workers": 2}) + self.assertEqual(parsed["providers"]["local"]["type"], "openai") + self.assertEqual(parsed["executor"], {"model": "original"}) + self.assertIn("# heading\n[run] # tuning\nk = 3 # count\n", self.project_file.read_text()) + + def test_set_value_round_trips_literals_and_creates_file(self): + for name, value in { + "string": 'quote " and \\ and\nnewline 😀\x7f', + "bool": True, + "int": 2, + "float": 0.25, + "array": ["one # literal", False, 2, 0.5], + "empty": [], + }.items(): + with self.subTest(name=name): + path = self.root / "new/nested/config.toml" + settings.set_value(path, "custom." + name, value) + parsed = tomllib.loads(path.read_text()) + self.assertEqual(parsed["custom"][name], value) + settings.set_value(path, "custom.a spaced key", "quoted key") + self.assertEqual(tomllib.loads(path.read_text())["custom"]["a spaced key"], "quoted key") + + def test_set_value_multiline_values_root_keys_and_line_endings(self): + original = ( + '# top\r\nroot = 1\r\n["executor"]\r\neditable = [\r\n' + ' "one#file", # first\r\n "two", # second\r\n] # files\r\n' + 'command = """first\r\n# inside string\r\nlast""" # command\r\n' + "[other]\r\nkeep = 'same'\r\n" + ) + self.project_file.write_bytes(original.encode()) + settings.set_value(self.project_file, "executor.editable", ["new"]) + settings.set_value(self.project_file, "executor.command", 'quoted " # literal') + settings.set_value(self.project_file, "added", False) + result = self.project_file.read_bytes().decode() + self.assertNotIn("one#file", result) + for comment in ("# top", "# first", "# second", "# files", "# command"): + self.assertIn(comment, result) + self.assertNotIn("# inside string", result) + self.assertIn("[other]\r\nkeep = 'same'\r\n", result) + self.assertNotIn("\n", result.replace("\r\n", "")) + parsed = tomllib.loads(result) + self.assertIs(parsed["added"], False) + self.assertEqual(parsed["executor"]["editable"], ["new"]) + self.assertEqual(parsed["executor"]["command"], 'quoted " # literal') + + def test_set_value_no_final_newline_and_invalid_input_do_not_destroy_file(self): + self.write(self.project_file, "[run]\nk = 1") + settings.set_value(self.project_file, "run.workers", 2) + self.assertEqual(tomllib.loads(self.project_file.read_text())["run"]["workers"], 2) + before = self.project_file.read_text() + for value in (None, {}, [[1]], [None], "\ud800"): + with self.subTest(value=value), self.assertRaises(settings.ConfigError): + settings.set_value(self.project_file, "run.value", value) + self.assertEqual(self.project_file.read_text(), before) + with self.assertRaisesRegex(settings.ConfigError, "invalid setting key"): + settings.set_value(self.project_file, "run..k", 2) + self.write(self.project_file, "[invalid") + with self.assertRaisesRegex(settings.ConfigError, "invalid TOML"): + settings.set_value(self.project_file, "run.k", 2) + self.assertEqual(self.project_file.read_text(), "[invalid") + + def test_set_value_multiline_quotes_and_escaped_hashes(self): + for literal in ( + "'literal # hash'", + '"escaped \\" # hash"', + "'''multiline # hash\nends with quote''''", + '"""multiline # hash\nends with quote""""', + ): + with self.subTest(literal=literal): + self.write(self.project_file, f"[suite]\nwork = {literal} # retain\n") + settings.set_value(self.project_file, "suite.work", "new") + self.assertEqual(self.project_file.read_text(), '[suite]\nwork = "new" # retain\n') + + +if __name__ == "__main__": + unittest.main() From fdb6574478279f949a3c66122674be900dfdddc1 Mon Sep 17 00:00:00 2001 From: Bae Seongmin <83870598+3ae3ae@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:34:56 +0900 Subject: [PATCH 03/21] refactor: move the three benchmarks into a harness2-benchmarks plugin (#3) JobBench, LAB and WorkBuddy-Bench now live in their own distribution under plugins/harness2-benchmarks/ (package harness2_benchmarks) and register through the harness2.suites entry-point group as the jb, lab, wb-office, wb-web and wb-code suites. The core package keeps only the method, the executors and the CLI. Moved with history: the benchmark adapters and renderers, the installer, preflight and setup, the vendored runners and patches (third_party/), the dependency metadata, the benchmark scripts and the benchmark docs. Downloaded benchmark dependencies now go to the user cache (~/.cache/harness2/benchmarks, or HARNESS2_BENCH_DEPS) instead of the repository, and each run's root comes from the suite context. The plugin keeps its env-driven configuration (HARNESS2_SOLVER_MODEL and friends) so the paper setup keeps working; runner assets ship as package data, so a wheel install works outside a checkout. Co-authored-by: Claude Opus 5.5 --- harness2/cli/_common.py | 20 +- harness2/cli/check.py | 5 +- harness2/cli/run_sweep.py | 4 +- plugins/harness2-benchmarks/LICENSE | 202 +++++++++ .../harness2-benchmarks/docs}/BENCHMARKS.md | 50 +-- .../harness2-benchmarks/docs}/EXPERIMENTS.md | 38 +- .../harness2-benchmarks/docs}/SETUP.md | 60 +-- .../harness2_benchmarks}/__init__.py | 0 .../harness2_benchmarks/config.py | 211 +++++++++ .../dependencies_meta}/ASSETS.json | 18 +- .../dependencies_meta}/PINNED.md | 8 +- .../dependencies_meta}/README.md | 4 +- .../harness2_benchmarks}/install.py | 8 +- .../jb_opencode_runner.py | 6 +- .../harness2_benchmarks}/jobbench.py | 52 ++- .../harness2_benchmarks}/lab.py | 43 +- .../harness2_benchmarks}/preflight.py | 40 +- .../harness2_benchmarks}/setup.py | 4 +- .../harness2_benchmarks/suites/__init__.py | 17 + .../harness2_benchmarks/suites/jb.py | 38 ++ .../harness2_benchmarks/suites/lab.py | 38 ++ .../harness2_benchmarks/suites/wb_code.py | 38 ++ .../harness2_benchmarks/suites/wb_office.py | 40 ++ .../harness2_benchmarks/suites/wb_web.py | 38 ++ .../harness2_benchmarks}/tasks.py | 8 +- .../third_party}/README.md | 8 +- .../third_party}/jb_runner/LICENSE | 0 .../third_party}/jb_runner/PROVENANCE.md | 6 +- .../jb_runner/codex_base_harness/AGENTS.md | 0 .../harness_baseline_oc/LongTermMEMORY.md | 0 .../harness_baseline_oc/guardrails.md | 0 .../harness_baseline_oc/harness.yaml | 0 .../harness_baseline_oc/plugin/README.md | 0 .../harness_baseline_oc/scripts/README.md | 0 .../harness_baseline_oc/sub_agents/README.md | 0 .../harness_baseline_oc/systemprompt.md | 0 .../tool_descriptions/tool_guidance.md | 0 .../run_benchmark_codex_cli_adapt.sh | 6 +- .../third_party}/lab_runner/LICENSE | 0 .../third_party}/lab_runner/PROVENANCE.md | 6 +- .../lab_runner/adapt_runs_cx/run_codex_lab.py | 1 + .../lab_runner/opencode/README.md | 0 .../lab_runner/opencode/__init__.py | 0 .../lab_runner/opencode/render.py | 1 - .../lab_runner/opencode/render_tree.py | 0 .../lab_runner/opencode/run_opencode.py | 0 .../lab_runner/opencode/scripts/read_doc.py | 0 .../lab_runner/opencode/session.py | 0 .../lab_runner/tts_lab/__init__.py | 0 .../third_party}/lab_runner/tts_lab/config.py | 1 + .../third_party}/licenses/harvey-labs.LICENSE | 0 .../licenses/job-bench-eval.LICENSE | 0 .../third_party}/licenses/opencode.LICENSE | 0 .../licenses/workbuddy-bench.LICENSE | 0 .../harvey-labs-judge-vertex-anthropic.diff | 0 .../patches/job-bench-eval-judge.diff | 0 .../third_party}/patches/opencode-compat.diff | 0 .../patches/workbuddy-bench-runtime.diff | 0 .../third_party}/wb_runner/LICENSE | 0 .../third_party}/wb_runner/PROVENANCE.md | 6 +- .../wb_runner/agents/codex_agent.py | 2 +- .../wb_runner/agents/codex_harness_agent.py | 0 .../third_party}/wb_runner/agents/oc_agent.py | 0 .../wb_runner/agents/oc_harness_agent.py | 0 .../harnesses/codex-harness/_defaults.yaml | 2 +- .../codex-harness/versions/0.144.6.yaml | 0 .../harnesses/oc-harness/_defaults.yaml | 0 .../oc-harness/versions/1.14.18.yaml | 0 .../configs/harnesses/opencode/CONFIG.md | 0 .../configs/harnesses/opencode/_defaults.yaml | 0 .../harnesses/opencode/docker/Dockerfile | 0 .../opencode/presets/settings/v1_14.json | 0 .../harnesses/opencode/versions/1.14.18.yaml | 0 .../harness2_benchmarks}/wb_render.py | 2 +- .../harness2_benchmarks}/workbuddy.py | 50 ++- plugins/harness2-benchmarks/pyproject.toml | 44 ++ .../scripts}/check_provenance.py | 9 +- .../scripts}/check_setup.py | 7 +- .../scripts}/run_jobbench.sh | 23 +- .../harness2-benchmarks/scripts}/run_lab.sh | 23 +- .../scripts}/run_workbuddy.sh | 23 +- .../scripts}/setup_benchmark.py | 4 +- .../scripts}/split_tasks.py | 2 +- .../harness2-benchmarks/tests/test_plugin.py | 425 ++++++++++++++++++ pyproject.toml | 3 +- 85 files changed, 1404 insertions(+), 240 deletions(-) create mode 100644 plugins/harness2-benchmarks/LICENSE rename {docs => plugins/harness2-benchmarks/docs}/BENCHMARKS.md (78%) rename {docs => plugins/harness2-benchmarks/docs}/EXPERIMENTS.md (74%) rename {docs => plugins/harness2-benchmarks/docs}/SETUP.md (68%) rename {harness2/benchmarks => plugins/harness2-benchmarks/harness2_benchmarks}/__init__.py (100%) create mode 100644 plugins/harness2-benchmarks/harness2_benchmarks/config.py rename {dependencies => plugins/harness2-benchmarks/harness2_benchmarks/dependencies_meta}/ASSETS.json (82%) rename {dependencies => plugins/harness2-benchmarks/harness2_benchmarks/dependencies_meta}/PINNED.md (84%) rename {dependencies => plugins/harness2-benchmarks/harness2_benchmarks/dependencies_meta}/README.md (90%) rename {harness2 => plugins/harness2-benchmarks/harness2_benchmarks}/install.py (98%) rename {harness2/benchmarks => plugins/harness2-benchmarks/harness2_benchmarks}/jb_opencode_runner.py (98%) rename {harness2/benchmarks => plugins/harness2-benchmarks/harness2_benchmarks}/jobbench.py (97%) rename {harness2/benchmarks => plugins/harness2-benchmarks/harness2_benchmarks}/lab.py (97%) rename {harness2 => plugins/harness2-benchmarks/harness2_benchmarks}/preflight.py (79%) rename {harness2/cli => plugins/harness2-benchmarks/harness2_benchmarks}/setup.py (97%) create mode 100644 plugins/harness2-benchmarks/harness2_benchmarks/suites/__init__.py create mode 100644 plugins/harness2-benchmarks/harness2_benchmarks/suites/jb.py create mode 100644 plugins/harness2-benchmarks/harness2_benchmarks/suites/lab.py create mode 100644 plugins/harness2-benchmarks/harness2_benchmarks/suites/wb_code.py create mode 100644 plugins/harness2-benchmarks/harness2_benchmarks/suites/wb_office.py create mode 100644 plugins/harness2-benchmarks/harness2_benchmarks/suites/wb_web.py rename {harness2/benchmarks => plugins/harness2-benchmarks/harness2_benchmarks}/tasks.py (93%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/README.md (71%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/jb_runner/LICENSE (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/jb_runner/PROVENANCE.md (85%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/jb_runner/codex_base_harness/AGENTS.md (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/jb_runner/harness_baseline_oc/LongTermMEMORY.md (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/jb_runner/harness_baseline_oc/guardrails.md (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/jb_runner/harness_baseline_oc/harness.yaml (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/jb_runner/harness_baseline_oc/plugin/README.md (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/jb_runner/harness_baseline_oc/scripts/README.md (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/jb_runner/harness_baseline_oc/sub_agents/README.md (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/jb_runner/harness_baseline_oc/systemprompt.md (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/jb_runner/harness_baseline_oc/tool_descriptions/tool_guidance.md (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/jb_runner/run_benchmark_codex_cli_adapt.sh (98%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/lab_runner/LICENSE (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/lab_runner/PROVENANCE.md (66%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/lab_runner/adapt_runs_cx/run_codex_lab.py (99%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/lab_runner/opencode/README.md (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/lab_runner/opencode/__init__.py (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/lab_runner/opencode/render.py (99%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/lab_runner/opencode/render_tree.py (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/lab_runner/opencode/run_opencode.py (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/lab_runner/opencode/scripts/read_doc.py (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/lab_runner/opencode/session.py (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/lab_runner/tts_lab/__init__.py (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/lab_runner/tts_lab/config.py (85%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/licenses/harvey-labs.LICENSE (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/licenses/job-bench-eval.LICENSE (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/licenses/opencode.LICENSE (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/licenses/workbuddy-bench.LICENSE (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/patches/harvey-labs-judge-vertex-anthropic.diff (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/patches/job-bench-eval-judge.diff (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/patches/opencode-compat.diff (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/patches/workbuddy-bench-runtime.diff (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/wb_runner/LICENSE (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/wb_runner/PROVENANCE.md (68%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/wb_runner/agents/codex_agent.py (99%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/wb_runner/agents/codex_harness_agent.py (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/wb_runner/agents/oc_agent.py (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/wb_runner/agents/oc_harness_agent.py (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/wb_runner/configs/harnesses/codex-harness/_defaults.yaml (77%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/wb_runner/configs/harnesses/codex-harness/versions/0.144.6.yaml (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/wb_runner/configs/harnesses/oc-harness/_defaults.yaml (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/wb_runner/configs/harnesses/oc-harness/versions/1.14.18.yaml (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/wb_runner/configs/harnesses/opencode/CONFIG.md (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/wb_runner/configs/harnesses/opencode/_defaults.yaml (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/wb_runner/configs/harnesses/opencode/docker/Dockerfile (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/wb_runner/configs/harnesses/opencode/presets/settings/v1_14.json (100%) rename {third_party => plugins/harness2-benchmarks/harness2_benchmarks/third_party}/wb_runner/configs/harnesses/opencode/versions/1.14.18.yaml (100%) rename {harness2/benchmarks => plugins/harness2-benchmarks/harness2_benchmarks}/wb_render.py (99%) rename {harness2/benchmarks => plugins/harness2-benchmarks/harness2_benchmarks}/workbuddy.py (96%) create mode 100644 plugins/harness2-benchmarks/pyproject.toml rename {scripts => plugins/harness2-benchmarks/scripts}/check_provenance.py (82%) rename {scripts => plugins/harness2-benchmarks/scripts}/check_setup.py (79%) rename {scripts => plugins/harness2-benchmarks/scripts}/run_jobbench.sh (59%) rename {scripts => plugins/harness2-benchmarks/scripts}/run_lab.sh (59%) rename {scripts => plugins/harness2-benchmarks/scripts}/run_workbuddy.sh (60%) rename {scripts => plugins/harness2-benchmarks/scripts}/setup_benchmark.py (86%) rename {scripts => plugins/harness2-benchmarks/scripts}/split_tasks.py (98%) create mode 100644 plugins/harness2-benchmarks/tests/test_plugin.py diff --git a/harness2/cli/_common.py b/harness2/cli/_common.py index 06d41307..ea88dbc9 100644 --- a/harness2/cli/_common.py +++ b/harness2/cli/_common.py @@ -23,6 +23,7 @@ import argparse from pathlib import Path +from .. import config from ..core import library as LIB from ..core import schedule as SCHED from ..core.spec import BenchmarkAdapter @@ -53,18 +54,25 @@ def make_adapter( if bench == "wb": if subset not in WB_SUBSETS: raise ValueError(f"--bench wb needs --subset from {WB_SUBSETS}, got {subset!r} ") - from ..benchmarks.workbuddy import WbAdapter + from harness2_benchmarks.workbuddy import WbAdapter - return WbAdapter(subset, substrate=substrate, split_file=split_file) + return WbAdapter( + subset, + substrate=substrate, + split_file=split_file, + runs_root=config.RUNS / "wb" / subset, + ) if subset: raise ValueError(f"--subset applies to --bench wb only, got {subset!r} for {bench}") if bench == "jb": - from ..benchmarks.jobbench import JobBenchAdapter + from harness2_benchmarks.jobbench import JobBenchAdapter - return JobBenchAdapter(substrate=substrate, split_file=split_file) - from ..benchmarks.lab import LabAdapter + return JobBenchAdapter( + substrate=substrate, split_file=split_file, runs_root=config.RUNS / "jb" + ) + from harness2_benchmarks.lab import LabAdapter - return LabAdapter(substrate=substrate, split_file=split_file) + return LabAdapter(substrate=substrate, split_file=split_file, runs_root=config.RUNS / "lab") def runs_root_for(adapter: BenchmarkAdapter) -> Path: diff --git a/harness2/cli/check.py b/harness2/cli/check.py index 638eaab5..ca79e5d2 100644 --- a/harness2/cli/check.py +++ b/harness2/cli/check.py @@ -18,8 +18,9 @@ import argparse +from harness2_benchmarks.preflight import check + from harness2.cli._common import make_adapter -from harness2.preflight import check def main(argv=None): @@ -36,7 +37,7 @@ def main(argv=None): for error in errors: print(f"FAIL: {error}") if errors: - print(f"Setup incomplete: {len(errors)} issue(s). See docs/BENCHMARKS.md.") + print(f"Setup incomplete: {len(errors)} issue(s). See plugins/harness2-benchmarks/docs/BENCHMARKS.md.") return 1 print( f"Ready: {adapter.spec.name}/{args.substrate}; {len(adapter.tasks())} tasks. " diff --git a/harness2/cli/run_sweep.py b/harness2/cli/run_sweep.py index 0876f4e6..70387e3f 100644 --- a/harness2/cli/run_sweep.py +++ b/harness2/cli/run_sweep.py @@ -52,7 +52,7 @@ def _build_parser() -> argparse.ArgumentParser: C.add_run_identity_args(p.add_argument_group("run identity (enters the namespace)")) g = p.add_argument_group("run control") g.add_argument("--split", choices=("all", "train", "test"), default="all") - g.add_argument("--split-file", type=Path, help="optional partition from scripts/split_tasks.py") + g.add_argument("--split-file", type=Path, help="optional partition from plugins/harness2-benchmarks/scripts/split_tasks.py") g.add_argument("--workers", type=int, default=1, help="domains to run concurrently") g.add_argument( "--only-domains", default="", help="comma list restricting the sweep to these domains" @@ -140,7 +140,7 @@ def _main(args: argparse.Namespace) -> int: adapter = C.make_adapter(args.bench, args.subset, args.substrate, args.split_file) _stream_for(adapter, args) - from ..preflight import require_ready + from harness2_benchmarks.preflight import require_ready require_ready(adapter, judge=True) diff --git a/plugins/harness2-benchmarks/LICENSE b/plugins/harness2-benchmarks/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/plugins/harness2-benchmarks/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/docs/BENCHMARKS.md b/plugins/harness2-benchmarks/docs/BENCHMARKS.md similarity index 78% rename from docs/BENCHMARKS.md rename to plugins/harness2-benchmarks/docs/BENCHMARKS.md index 18ca85f0..84845558 100644 --- a/docs/BENCHMARKS.md +++ b/plugins/harness2-benchmarks/docs/BENCHMARKS.md @@ -2,7 +2,7 @@ Complete the [installation and credentials](SETUP.md) first. Each benchmark installs separately; install the ones you intend to run. Every command -below takes `--substrate cx` to use the +below takes `-c executor.name=codex` to use the [Codex executor](SETUP.md#codex-executor) instead of OpenCode. Choose the recursion mode with `--mode parallel` or `--mode sequential` and the task-level recursion budget with `--k`. Keep both identical across the run, judge, and @@ -16,17 +16,17 @@ groups tasks by practice area and removes criteria from the material passed to the improver. Needs Pandoc and Poppler. ```bash -harness2-setup lab -harness2-check --bench lab -harness2 --bench lab --k 1 --mode parallel -harness2-judge --bench lab --k 1 --mode parallel -harness2-report --bench lab --k 1 --mode parallel +harness2 setup lab +harness2 -c executor.name=opencode -c suite.name=lab doctor +harness2 -c executor.name=opencode run --suite lab --k 1 --mode parallel +harness2 judge +harness2 report ``` Sequential recursion runs the same commands with `--mode sequential`. Tasks arrive with the pinned checkout at -`dependencies/harvey-labs/tasks///task.json`, and the adapter +`~/.cache/harness2/benchmarks/harvey-labs/tasks///task.json`, and the adapter discovers all installed tasks; no separate task list is required. Setup installs the LAB dependency environment, the runner overlays, and the included Vertex judge patch. @@ -34,9 +34,9 @@ judge patch. The judge reads outputs only after selection. Reports use Partial (fraction of criteria passed) and All-pass (all criteria passed). A failed judge remains unscored and writes its output to `judge.log` in the run directory; a Vertex -`429 RESOURCE_EXHAUSTED` there means quota, so rerun `harness2-judge` later. +`429 RESOURCE_EXHAUSTED` there means quota, so rerun `harness2 judge` later. Changing the judge model changes the measurement. Outputs are under -`runs/lab/`. +`.harness2/runs//`. ## JobBench @@ -46,18 +46,18 @@ Professional tasks, with the library grouped by the dataset's occupation directories. ```bash -harness2-setup jb -harness2-check --bench jb -harness2 --bench jb --k 1 --mode parallel -harness2-judge --bench jb --k 1 --mode parallel -harness2-report --bench jb --k 1 --mode parallel +harness2 setup jb +harness2 -c executor.name=opencode -c suite.name=jb doctor +harness2 -c executor.name=opencode run --suite jb --k 1 --mode parallel +harness2 judge +harness2 report ``` Sequential recursion runs the same commands with `--mode sequential`. Setup downloads only the main dataset from Hugging Face — public and ungated, so no account, token, or terms acceptance is required — records its revision, and -places tasks at `dependencies/job-bench-eval/dataset/main/`. It also installs the +places tasks at `~/.cache/harness2/benchmarks/job-bench-eval/dataset/main/`. It also installs the document extraction dependencies and a judge patch that handles reasoning effort and empty completions. Rubrics stay outside the improver's evidence. @@ -65,7 +65,7 @@ With a Gemini `HARNESS2_JUDGE_MODEL`, the judge uses the project's Vertex OpenAI-compatible endpoint and Application Default Credentials. For a separate judge provider, set `HARNESS2_JUDGE_BASE_URL`, `HARNESS2_JUDGE_API_KEY`, and the model ID that endpoint accepts. The reported score uses the upstream rubric -weights; unscored tasks are reported separately. Results are under `runs/jb/`. +weights; unscored tasks are reported separately. Results are under `.harness2/runs//`. Note: JobBench data, including `RUBRICS.json`, is public on Hugging Face, so an executor with web access may retrieve it. Block those hosts during runs as @@ -84,22 +84,22 @@ retain that license. ```bash docker info -harness2-setup wb --subset office -harness2-check --bench wb --subset office -harness2 --bench wb --subset office --k 1 --mode parallel -harness2-judge --bench wb --subset office --k 1 --mode parallel -harness2-report --bench wb --subset office --k 1 --mode parallel +harness2 setup wb-office +harness2 -c executor.name=opencode -c suite.name=wb-office doctor +harness2 -c executor.name=opencode run --suite wb-office --k 1 --mode parallel +harness2 judge +harness2 report ``` Sequential recursion runs the same commands with `--mode sequential`. -`harness2-setup wb` without `--subset` installs all three subsets in one run; +`python plugins/harness2-benchmarks/scripts/setup_benchmark.py wb` without `--subset` installs all three subsets in one run; pass `--subset` to install only the one you intend to run. Every other command takes one subset at a time. Setup installs the pinned Harbor-based runtime, applies the included adapter, mount, and credential/transport patches, copies the harness runners, creates -solver and judge routes from `.env`, downloads the requested subset with the +solver and judge routes from `.harness2/.env`, downloads the requested subset with the upstream fetch script, and builds the OpenCode harness image. The public Vertex setup uses a `google-vertex/gemini...` executor and a bare `gemini...` judge ID; no model YAML or account-specific proxy configuration needs to be copied from @@ -115,10 +115,10 @@ cannot be verified is not unpacked. A verified download prints `wb-bench--v1.0.tar.gz: OK`. Tasks are discovered from `task.toml` and `instruction.md` under -`dependencies/workbuddy-bench/datasets/wb-bench--v1.0/tasks/`. Libraries use +`~/.cache/harness2/benchmarks/workbuddy-bench/datasets/wb-bench--v1.0/tasks/`. Libraries use `metadata.role` on code and `metadata.category` on office and web. Each rollout uses a fresh benchmark container. The adapter reads the verifier's `score.json`; the improver never receives verifier output. Missing verifier results remain -unscored. Runs live under `runs/wb//`. +unscored. Runs live under `.harness2/runs//`. For Codex, start the bridge with `harness2-bridge --host 0.0.0.0` so that the containers can reach it. diff --git a/docs/EXPERIMENTS.md b/plugins/harness2-benchmarks/docs/EXPERIMENTS.md similarity index 74% rename from docs/EXPERIMENTS.md rename to plugins/harness2-benchmarks/docs/EXPERIMENTS.md index bd52e150..c37229ff 100644 --- a/docs/EXPERIMENTS.md +++ b/plugins/harness2-benchmarks/docs/EXPERIMENTS.md @@ -24,18 +24,18 @@ separated by benchmark, executor, mode, budget, and seed. | `--k K --mode parallel` | Explore `K` candidates per scope and produce `K` compositions | | `--k K --mode sequential` | Refine the harness through `K` successive steps | | `--workers N` | Run `N` domains concurrently; tasks within each domain stay ordered | -| `--substrate cx` | Use Codex instead of the default OpenCode executor | +| `-c executor.name=codex` | Use Codex instead of the default OpenCode executor | | `--seed N` | Shuffle the task stream reproducibly; does not control model sampling | Use the same benchmark, executor, subset, `k`, mode, and seed for refinement, judging, and reporting. For example: ```bash -export HARNESS2_RUNS_DIR=runs/lab-sequential-k3 -harness2 --bench lab --k 3 --mode sequential \ +export HARNESS2_PATHS__RUNS=.harness2/runs/lab-sequential-k3 +harness2 -c executor.name=opencode run --suite lab --k 3 --mode sequential \ --only-domains antitrust-competition --max-tasks 1 -harness2-judge --bench lab --k 3 --mode sequential -harness2-report --bench lab --k 3 --mode sequential +harness2 judge +harness2 report ``` Use a separate output root when changing the budget or mode to keep experiments @@ -56,20 +56,20 @@ reporting in sequence. Set `SUBSTRATE`, `K`, `MODE`, and, for WorkBuddy, `SUBSET`; additional arguments are passed to the refinement command: ```bash -bash scripts/run_lab.sh --only-domains antitrust-competition --max-tasks 1 -SUBSTRATE=cx K=3 MODE=sequential bash scripts/run_jobbench.sh --max-tasks 1 -SUBSET=web bash scripts/run_workbuddy.sh --max-tasks 1 +bash plugins/harness2-benchmarks/scripts/run_lab.sh --only-domains antitrust-competition --max-tasks 1 +SUBSTRATE=cx K=3 MODE=sequential bash plugins/harness2-benchmarks/scripts/run_jobbench.sh --max-tasks 1 +SUBSET=web bash plugins/harness2-benchmarks/scripts/run_workbuddy.sh --max-tasks 1 ``` ## Reproducibility When comparing budgets or modes, keep task selection, executor, improver, judge, -and web access fixed. Use a distinct `HARNESS2_RUNS_DIR` when changing models or +and web access fixed. Use a distinct `HARNESS2_PATHS__RUNS` when changing models or task selection; the saved configuration prevents incompatible resumes. JobBench's dataset is resolved at download time rather than pinned to a commit. Setup records the downloaded Hugging Face revision in -`dependencies/job-bench-eval/dataset/main/.harness2-download.json`. Report that +`~/.cache/harness2/benchmarks/job-bench-eval/dataset/main/.harness2-download.json`. Report that revision alongside JobBench results. ### Task partitions @@ -77,8 +77,8 @@ revision alongside JobBench results. To create a reproducible, domain-stratified partition of the installed tasks: ```bash -python scripts/split_tasks.py --bench lab --seed 42 --output splits/lab.json -harness2 --bench lab --split-file splits/lab.json --split test --mode parallel +python plugins/harness2-benchmarks/scripts/split_tasks.py --bench lab --seed 42 --output splits/lab.json +harness2 -c executor.name=opencode run --suite lab --split-file splits/lab.json --split test --mode parallel ``` The default train fraction is 0.5. Generated partitions are local files, not @@ -86,9 +86,9 @@ release assets, and the script refuses to overwrite an existing file. ## Baseline evaluation -`harness2-judge --bases` scores a run namespace's reference executions, writing +`harness2 judge --bases` scores a run namespace's reference executions, writing `base_judged.json` beside `judged.json` for the base-harness comparison. -`harness2-report` covers the refined harness only. +`harness2 report` covers the refined harness only. ## Web access @@ -110,13 +110,13 @@ tool setting. Report the restrictions used alongside results. ## Results and resume -The default output roots are `runs/lab/`, `runs/jb/`, and -`runs/wb//`. They contain task summaries, execution artifacts, adopted +The default output roots are `.harness2/runs//`, `.harness2/runs//`, and +`.harness2/runs//`. They contain task summaries, execution artifacts, adopted harnesses, reflection notes, and graded results. Repeat the same command to resume. Failed evaluations remain unscored rather -than being counted as zero. Use a new `HARNESS2_RUNS_DIR` when changing models or +than being counted as zero. Use a new `HARNESS2_PATHS__RUNS` when changing models or task selection. -Refinement, judging, and reporting use `harness2`, `harness2-judge`, and -`harness2-report`; each supports `--help`. +Refinement, judging, and reporting use `harness2 run`, `harness2 judge`, and +`harness2 report`; each supports `--help`. diff --git a/docs/SETUP.md b/plugins/harness2-benchmarks/docs/SETUP.md similarity index 68% rename from docs/SETUP.md rename to plugins/harness2-benchmarks/docs/SETUP.md index 0d037b8a..13144714 100644 --- a/docs/SETUP.md +++ b/plugins/harness2-benchmarks/docs/SETUP.md @@ -1,6 +1,6 @@ # Setup guide -Start with the [quick start](../README.md#install). This guide covers benchmark +Start with the [quick start](../../../README.md#install). This guide covers benchmark dependencies, model credentials, and the optional Codex executor. For individual benchmarks, see the [benchmark guides](BENCHMARKS.md). @@ -22,19 +22,19 @@ Pandoc and Poppler are needed by LAB. Also install: - [Docker Engine](https://docs.docker.com/engine/install/) for WorkBuddy-Bench, with permission to run `docker info`. -Benchmark setup installs its own pinned Bun runtime under `dependencies/`. +Benchmark setup installs its own pinned Bun runtime under `~/.cache/harness2/benchmarks/`. Keep the main Harness² environment active when running the commands below. ## Models and credentials -Copy `.env.example` to `.env` if you have not already done so. Set your Vertex AI +Copy `.env.example` to `.harness2/.env` if you have not already done so. Set your Vertex AI project and model IDs, then authenticate: ```bash gcloud auth application-default login ``` -Shell environment variables take precedence over `.env`. +Shell environment variables take precedence over `.harness2/.env`. `GOOGLE_CLOUD_PROJECT` identifies your project; `GOOGLE_CLOUD_LOCATION` defaults to `global`. Your account must have access to each configured model. @@ -43,7 +43,7 @@ to `global`. Your account must have access to each configured model. | `HARNESS2_SOLVER_MODEL` | Executor model in OpenCode's `provider/model` format | | `HARNESS2_IMPROVER_MODEL` | Improver model in the same format; defaults to the executor | | `HARNESS2_JUDGE_MODEL` | Bare judge model ID; the adapter supplies the Vertex prefix | -| `HARNESS2_RUNS_DIR` | Output root; defaults to `runs/` | +| `HARNESS2_PATHS__RUNS` | Output root; defaults to `.harness2/runs/` | The provided configuration uses Vertex Gemini executors and judges, and supports either Vertex Claude or Gemini as the improver. LAB also supports the upstream @@ -70,31 +70,31 @@ route: the installation check runs Install only the benchmark you intend to run: ```bash -harness2-setup lab -harness2-check --bench lab +harness2 setup lab +harness2 -c executor.name=opencode -c suite.name=lab doctor ``` -Use `jb` for JobBench or `wb` for WorkBuddy-Bench. See the +Use `jb` for JobBench or `wb-office` for WorkBuddy-Bench. See the [benchmark guides](BENCHMARKS.md) for subset selection and licensing. The runner overlays and compatibility patches are included in -[`third_party/`](../third_party/README.md). Setup clones supported upstream -revisions under `dependencies/`, applies the patches, creates the benchmark +[`third_party/`](../harness2_benchmarks/third_party/README.md). Setup clones supported upstream +revisions under `~/.cache/harness2/benchmarks/`, applies the patches, creates the benchmark environments, and downloads task data. Each benchmark environment lives inside its upstream checkout: -- `dependencies/harvey-labs/.venv` -- `dependencies/job-bench-eval/.venv` -- `dependencies/workbuddy-bench/.venv` +- `~/.cache/harness2/benchmarks/harvey-labs/.venv` +- `~/.cache/harness2/benchmarks/job-bench-eval/.venv` +- `~/.cache/harness2/benchmarks/workbuddy-bench/.venv` Harness² selects these interpreters automatically; do not activate them for the main commands. ### Installation checks -`harness2-setup --dry-run` previews setup without making changes. -`harness2-check` checks installed dependencies, credentials, and configuration +`harness2 setup --dry-run` previews setup without making changes. +`harness2 doctor` checks installed dependencies, credentials, and configuration without calling a model. Refinement runs perform the same checks, including judge configuration, before starting. @@ -112,19 +112,19 @@ LiteLLM server, using the same Application Default Credentials and Install the optional dependencies and start the bridge in a separate session: ```bash -uv pip install -e '.[render,bridge]' +uv pip install -e '.[bridge]' -e 'plugins/harness2-benchmarks[render]' harness2-bridge # for WorkBuddy: harness2-bridge --host 0.0.0.0 ``` -The bridge prints the `HARNESS2_CODEX_*` settings to add to `.env`. Its port and -generated access key are stored under `dependencies/codex/` and reused on -restart. Pass `--substrate cx` to setup, checks, refinement, judging, and +The bridge prints the `HARNESS2_CODEX_*` settings to add to `.harness2/.env`. Its port and +generated access key are stored under `~/.cache/harness2/bridge/` and reused on +restart. Pass `-c executor.name=codex` to setup, checks, refinement, judging, and reporting: ```bash -harness2-setup lab --substrate cx -harness2-check --bench lab --substrate cx -harness2 --bench lab --substrate cx --k 1 --mode parallel \ +harness2 setup lab --substrate cx +harness2 -c executor.name=codex -c suite.name=lab doctor +harness2 -c executor.name=codex run --suite lab --k 1 --mode parallel \ --only-domains antitrust-competition --max-tasks 1 ``` @@ -134,8 +134,8 @@ bridge to all network interfaces; restrict access to the intended clients. To use another Responses API endpoint, set `HARNESS2_CODEX_BASE_URL`, `HARNESS2_CODEX_API_KEY`, and `HARNESS2_CODEX_MODEL` directly. Setup records the -endpoint and model in `dependencies/codex/home/config.toml`; rerun setup after -changing either. `harness2-check` reports stale configuration until then. +endpoint and model in `~/.cache/harness2/benchmarks/codex/home/config.toml`; rerun setup after +changing either. `harness2 doctor` reports stale configuration until then. Codex runs with its sandbox disabled. Review the [executor network restrictions](EXPERIMENTS.md#web-access) before evaluation. @@ -152,14 +152,14 @@ Setup downloads dependencies from the following hosts: | A Docker registry | WorkBuddy container image | | `astral.sh` | CPython 3.12 when the host has no system Python 3.12 | -Use `harness2-setup --dry-run` to review setup commands when configuring +Use `harness2 setup --dry-run` to review setup commands when configuring network access. Experiments also access the model endpoints configured in -`.env`. Setup downloads and executor web access have different requirements; +`.harness2/.env`. Setup downloads and executor web access have different requirements; see [web access during evaluation](EXPERIMENTS.md#web-access). ## Wheel installations -A wheel ships the `harness2` package only. The task partitioner, -`python scripts/split_tasks.py`, requires this source tree. For a wheel -installation, work from a dedicated directory: `.env`, `dependencies/`, and -`runs/` are resolved there. +A wheel ships the `harness2_benchmarks` package only. The task partitioner, +`python plugins/harness2-benchmarks/scripts/split_tasks.py`, requires this source tree. For a wheel +installation, work from a dedicated directory: `.harness2/.env`, `~/.cache/harness2/benchmarks/`, and +`.harness2/runs/` are resolved there. diff --git a/harness2/benchmarks/__init__.py b/plugins/harness2-benchmarks/harness2_benchmarks/__init__.py similarity index 100% rename from harness2/benchmarks/__init__.py rename to plugins/harness2-benchmarks/harness2_benchmarks/__init__.py diff --git a/plugins/harness2-benchmarks/harness2_benchmarks/config.py b/plugins/harness2-benchmarks/harness2_benchmarks/config.py new file mode 100644 index 00000000..2b59f9de --- /dev/null +++ b/plugins/harness2-benchmarks/harness2_benchmarks/config.py @@ -0,0 +1,211 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Deployment settings and dependency versions. Shell values override .env.""" + +from __future__ import annotations + +import os +import sys +from dataclasses import fields +from pathlib import Path + +from harness2.core.spec import ChannelSpec +from harness2.providers import Provider +from harness2.settings import ConfigError, cache_dir +from harness2.settings import load_dotenv as load_dotenv + +PKG = Path(__file__).resolve().parent +THIRD_PARTY = PKG / "third_party" +METHOD_VERSION = 2 +DEPENDENCIES = ( + Path(os.environ.get("HARNESS2_BENCH_DEPS") or cache_dir() / "benchmarks").expanduser().resolve() +) + + +def env(name: str, default=None): + """Env override where a set-but-empty variable is the same as an unset one.""" + return os.environ.get(name) or default + + +def env_int(name: str, default: int) -> int: + v = env(name) + if v is None: + return int(default) + try: + return int(v) + except ValueError as e: + raise ConfigError(f"{name} must be an integer, got {v!r}") from e + + +def require(name: str, *, needed_by: str) -> str: + """The value of `name`, or ConfigError with the remediation line when it is unset/empty.""" + v = env(name) + if not v: + raise ConfigError( + f"{name} is required for {needed_by}; set it in .harness2/.env or the shell (see .env.example)" + ) + return str(v) + + +VERTEX_PROJECT = env("GOOGLE_CLOUD_PROJECT", env("HARNESS2_VERTEX_PROJECT", "")) +VERTEX_REGION = env("GOOGLE_CLOUD_LOCATION", "global") +if VERTEX_PROJECT: + os.environ.setdefault("GOOGLE_CLOUD_PROJECT", VERTEX_PROJECT) +os.environ.setdefault("GOOGLE_CLOUD_LOCATION", VERTEX_REGION) + + +SOLVER_MODEL_OC = env("HARNESS2_SOLVER_MODEL", "") +SOLVER_MODEL_CX = env("HARNESS2_CODEX_MODEL", SOLVER_MODEL_OC.split("/", 1)[-1]) +IMPROVER_MODEL = env("HARNESS2_IMPROVER_MODEL", SOLVER_MODEL_OC) +JUDGE_MODEL = env("HARNESS2_JUDGE_MODEL", "") + +DEPENDENCY_PINS = { + "job-bench-eval": "fa03a17f9ff662f136798a274e10011c3aa00fd0", + "workbuddy-bench": "b516950be5b56eb3be406c2f76ee1c5111dcb57f", + "harvey-labs": "b58a28ae9fde5ee4b810e53b4fd0a1915325a35d", + "opencode": "23fb5e0516c99ac04a1aa46c193efda2e1b9bb24", +} +DEPENDENCY_ORIGINS = { + "job-bench-eval": "https://github.com/Job-Bench/job-bench-eval.git", + "workbuddy-bench": "https://github.com/Tencent/workbuddy-bench.git", + "harvey-labs": "https://github.com/harveyai/harvey-labs.git", + "opencode": "https://github.com/anomalyco/opencode.git", +} + + +def jb_repo() -> Path: + return DEPENDENCIES / "job-bench-eval" + + +def wb_repo() -> Path: + return DEPENDENCIES / "workbuddy-bench" + + +def lab_repo() -> Path: + return DEPENDENCIES / "harvey-labs" + + +OPENCODE_DIR = DEPENDENCIES / "opencode" +BUN = DEPENDENCIES / "tools" / "node_modules" / ".bin" / "bun" +LAB_PYTHON = lab_repo() / ".venv" / "bin" / "python" +LAB_JUDGE_MODEL = ( + "vertex/" + JUDGE_MODEL if JUDGE_MODEL.startswith(("claude", "gemini")) else JUDGE_MODEL +) +LAB_DELIV_CAP = 60000 +JB_JUDGE_PYTHON = jb_repo() / ".venv" / "bin" / "python" +JB_JUDGE_MODEL = ( + "google/" + JUDGE_MODEL + if JUDGE_MODEL.startswith("gemini") and not env("HARNESS2_JUDGE_BASE_URL") + else JUDGE_MODEL +) +_vertex_host = ( + "aiplatform.googleapis.com" + if VERTEX_REGION == "global" + else f"{VERTEX_REGION}-aiplatform.googleapis.com" +) +VERTEX_OPENAI_URL = ( + f"https://{_vertex_host}/v1/projects/{VERTEX_PROJECT}" + f"/locations/{VERTEX_REGION}/endpoints/openapi" +) +JB_JUDGE_API_BASE = env("HARNESS2_JUDGE_BASE_URL", VERTEX_OPENAI_URL if VERTEX_PROJECT else "") +JB_JUDGE_API_KEY = env("HARNESS2_JUDGE_API_KEY", "") +JB_JUDGE_REASONING_EFFORT = "low" +JB_REPORTING_JUDGE_DETAILS = "" +JB_OBS_TRUNC = 50000 + +CODEX_VERSION = "0.144.6" +CODEX_DIST = str(DEPENDENCIES / "codex" / "dist") +CODEX_BASE_URL = env("HARNESS2_CODEX_BASE_URL", "") +CODEX_CONTAINER_URL = env("HARNESS2_CODEX_CONTAINER_URL", CODEX_BASE_URL) +VERTEX_PROXY_KEY = env("HARNESS2_CODEX_API_KEY", "") +CODEX_HOME_DIR = DEPENDENCIES / "codex" / "home" + +WB_SOLVER_MODEL = "harness2/solver" +WB_WEB_JUDGE = "harness2/judge" +WB_RENDER_PY = sys.executable +WB_OC_HARNESS_VER = "1.14.18" +WB_CX_HARNESS_VER = CODEX_VERSION +WB_MAX_JOBS = {"oc": 1, "cx": 1} +WB_MAX_TRIALS = {"oc": 2, "cx": 2} +WB_MODEL_CONNECTION = "local_proxy" + +DEFAULT_PROPOSER_CHANNEL = "opencode" +PROPOSER_MODEL_BY_CHANNEL = {"opencode": IMPROVER_MODEL, "claude_cli": IMPROVER_MODEL} +PHASE_WIDTH = 2 +TRANSPORT_ATTEMPTS = 2 +PROPOSER_CONCURRENCY = 2 + + +def validate_models(*, judge: bool = False) -> None: + require("HARNESS2_SOLVER_MODEL", needed_by="executor rollouts") + if "/" not in SOLVER_MODEL_OC or "/" not in IMPROVER_MODEL: + raise ConfigError("Executor and improver models must use OpenCode's provider/model syntax") + if any(m.startswith("google-vertex") for m in (SOLVER_MODEL_OC, IMPROVER_MODEL)): + if not VERTEX_PROJECT: + raise ConfigError("Set GOOGLE_CLOUD_PROJECT for Vertex AI") + if judge: + require("HARNESS2_JUDGE_MODEL", needed_by="benchmark judging") + + +CHANNELS = ("claude_cli", "opencode") + + +def resolve_channel_model( + kind: str | None, + model: str | None, + *, + effort: str = "high", + steps: int = 120, + timeout: int = 3600, +) -> ChannelSpec: + """The one channel->model mapping; a lone override of either raises.""" + kwargs = {"effort": effort, "steps": steps, "timeout": timeout} + channel_fields = {field.name for field in fields(ChannelSpec)} + # shortcut: W2 adds bin/provider, remove the field guards when it is integrated. + if "bin" in channel_fields and (kind or DEFAULT_PROPOSER_CHANNEL) == "opencode": + kwargs["bin"] = ( + f"{BUN} run --cwd {OPENCODE_DIR}/packages/opencode --conditions=browser src/index.ts" + ) + if "provider" in channel_fields and VERTEX_PROJECT: + kwargs["provider"] = Provider( + "vertex", "vertex", project=VERTEX_PROJECT, region=VERTEX_REGION + ) + use_default = kind is None and model is None + if (kind is None) != (model is None): + raise ValueError( + "channel and model must be overridden TOGETHER (a bare-id/provider-prefixed " + f"mismatch fails silently downstream); got kind={kind!r}, model={model!r}" + ) + if kind is None: + kind = DEFAULT_PROPOSER_CHANNEL + if kind not in CHANNELS: + raise ValueError(f"unknown channel {kind!r}; expected one of {CHANNELS}") + if model is None: + model = PROPOSER_MODEL_BY_CHANNEL[kind] + if not model.strip(): + if use_default: + return ChannelSpec(kind, "", **kwargs) + raise ValueError("model override must not be empty") + if kind == "opencode" and "/" not in model: + raise ValueError( + f"opencode channel needs a provider-prefixed model id " + f"in HARNESS2_IMPROVER_MODEL or HARNESS2_SOLVER_MODEL, got {model!r}" + ) + if kind == "claude_cli" and "/" in model: + raise ValueError( + f"claude_cli channel takes a bare Anthropic id " + f"(e.g. {PROPOSER_MODEL_BY_CHANNEL['claude_cli']!r}), got {model!r}" + ) + return ChannelSpec(kind, model, **kwargs) diff --git a/dependencies/ASSETS.json b/plugins/harness2-benchmarks/harness2_benchmarks/dependencies_meta/ASSETS.json similarity index 82% rename from dependencies/ASSETS.json rename to plugins/harness2-benchmarks/harness2_benchmarks/dependencies_meta/ASSETS.json index 64c838fd..001c9d29 100644 --- a/dependencies/ASSETS.json +++ b/plugins/harness2-benchmarks/harness2_benchmarks/dependencies_meta/ASSETS.json @@ -1,6 +1,6 @@ { "third_party/jb_runner/LICENSE": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4", - "third_party/jb_runner/PROVENANCE.md": "7e16c8f5e43aa272baa5cf39bf245b42cf1f40f885457ac4123c91b7de0b06ef", + "third_party/jb_runner/PROVENANCE.md": "ef40bc6abe5ce4e1367e454cc8a4b18a23be59db43ea42650cce5d0771ba4444", "third_party/jb_runner/codex_base_harness/AGENTS.md": "b5df5264478dfce7d6af52f8443a1797b17967d1bc4a8c4f8c5fe43933516a89", "third_party/jb_runner/harness_baseline_oc/LongTermMEMORY.md": "c78b55245ef1433c017bc3f6f796094e16c640b98543836f6d930972243d28d2", "third_party/jb_runner/harness_baseline_oc/guardrails.md": "ebd0b0f287bcc9bea5efa063d632677b3c0fdf014c7b67bec088c11253605a53", @@ -10,19 +10,19 @@ "third_party/jb_runner/harness_baseline_oc/sub_agents/README.md": "56c5507e02c30c0d611efe2d7426141e9fc626a91599738afc7e003a07d3adb0", "third_party/jb_runner/harness_baseline_oc/systemprompt.md": "f5fa6862d87e4c51b78a4ce4446dace9438b3d201e0350c91ca02d0c83f552e3", "third_party/jb_runner/harness_baseline_oc/tool_descriptions/tool_guidance.md": "5ae4f052905a31e9573abe4971172bf40b4e1f5d265d1be072ed444232371f41", - "third_party/jb_runner/run_benchmark_codex_cli_adapt.sh": "a94c645d3061da354591dde446d11f8b3e6d08b4005f8f587668aae07c0fc100", + "third_party/jb_runner/run_benchmark_codex_cli_adapt.sh": "91ccd802f423dfd1abd0d8f28ac1d962fb3aae896a778532d51e934f8242876f", "third_party/lab_runner/LICENSE": "f92627d2ebe80fc0add3b171b2d7eee5e28a98dd0d0a4a5ee5829314243bb3b9", - "third_party/lab_runner/PROVENANCE.md": "b5b42319000e1522765d2520dffcd9d1058c5472cc49046ec1906ffffa66c53c", - "third_party/lab_runner/adapt_runs_cx/run_codex_lab.py": "567d6a06d8104f853b8d83953af333ea8ef57a9053e0c9788a42692c48196556", + "third_party/lab_runner/PROVENANCE.md": "942fd05e4a4ca804c3650e7681ac788bf9febb2aa57210cc39de0cb9c74344b6", + "third_party/lab_runner/adapt_runs_cx/run_codex_lab.py": "2c9f911adc9d55c968e81ae7cc2ad4585975316ac61ed7634d0453868b3fee7e", "third_party/lab_runner/opencode/README.md": "20f2739eada1a76560c412d39603e040c420d3d36a2bf67a99a814500306f0ae", "third_party/lab_runner/opencode/__init__.py": "df2b298f529bbcba94713aa748b5a04569f38bbb3f96c9219b7fb0d7186066a8", - "third_party/lab_runner/opencode/render.py": "b636a60d039fdfa66d64eb5d5d3c93a99464be2bcbd5b80dc8f91d1f104ea5ee", + "third_party/lab_runner/opencode/render.py": "dbdace5189da9162b916103523fb3e0295239c02561c01360acd4aa72110e749", "third_party/lab_runner/opencode/render_tree.py": "54d64b461e430905b212b46902512aad38d3585fb5921b7a9982eea0413a4b73", "third_party/lab_runner/opencode/run_opencode.py": "7e7178583a6acb2e07bffd1fbef78f1b0095975bc2c61f8b0038fcc528090e32", "third_party/lab_runner/opencode/scripts/read_doc.py": "6b3c8feb3b90b0b2bbbf4cb491c81f1114b67ada04d2c988fac8cb1af68533b5", "third_party/lab_runner/opencode/session.py": "e804e29a20ac4fd71454c85b9e15432cf303739c7f8d7aacc0df19f732bfcb62", "third_party/lab_runner/tts_lab/__init__.py": "4ae8a9112bc00a182c983a35c8c5db40717007cefbc1b0bed89f04bc384a0106", - "third_party/lab_runner/tts_lab/config.py": "f6eb8551c3cb3538de42a2e43cd5cd1aa466370d64dd681e46ec70f1c3dcf965", + "third_party/lab_runner/tts_lab/config.py": "e2071fdad3664ca0f25e36d9c2c9cf1ea7e7398bc913b3921a183c9c5c8aaba7", "third_party/licenses/harvey-labs.LICENSE": "f92627d2ebe80fc0add3b171b2d7eee5e28a98dd0d0a4a5ee5829314243bb3b9", "third_party/licenses/job-bench-eval.LICENSE": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4", "third_party/licenses/opencode.LICENSE": "625f0f619133f89bbbb2abe37369613dfa1885eba1e50d02170deb62bb42cb6b", @@ -32,12 +32,12 @@ "third_party/patches/opencode-compat.diff": "c9a8a0e826d48f3fcfc7d818321d128c28e1f7f7c594954c19453dfca4946dc8", "third_party/patches/workbuddy-bench-runtime.diff": "1d581ee0c23a9a5574ce0f9317d0bda17f6c3a2556898c89b34ccbdbd5b87b3f", "third_party/wb_runner/LICENSE": "b466fc7a63acc5cac4275fc4b0f1e22f332ad188ea600b217076ea3a8e040956", - "third_party/wb_runner/PROVENANCE.md": "9011685c8808fb6ab56d23c823af884e5905caecd00f64323f3d02f22e45524c", - "third_party/wb_runner/agents/codex_agent.py": "dbdf48c19c51d7a0389caf3f5601c1ff967ef534137d829afe8ca8792db40c9d", + "third_party/wb_runner/PROVENANCE.md": "71f11c2fa0858d7f83b9efee09443438c61ba3cc58519a4dab4bf540c200990b", + "third_party/wb_runner/agents/codex_agent.py": "b031dcd1846b028ce97c98b1f5d4d738ad28197f40e54cafdd0a3800fa5ab5a7", "third_party/wb_runner/agents/codex_harness_agent.py": "fc1d1ecfc2de1debabb0ff84ec7a113b05536873ac76f73806835ff873b2f580", "third_party/wb_runner/agents/oc_agent.py": "5375a49b9a9e6d6d6212b47156b36dca17934e54728ddeed6218ac3662c67fff", "third_party/wb_runner/agents/oc_harness_agent.py": "647e31187af1305a5ddc42f7e41740795f825e0c708f63f5d1f29b954625c09c", - "third_party/wb_runner/configs/harnesses/codex-harness/_defaults.yaml": "5dddd5c358eff75ca69f7277ae6ccd58a2e208e19d722b862f04062a3718479e", + "third_party/wb_runner/configs/harnesses/codex-harness/_defaults.yaml": "d0b85da88e06498d37fbc7a8cfe6af5bb42be8c573458e158d1755ca88a28853", "third_party/wb_runner/configs/harnesses/codex-harness/versions/0.144.6.yaml": "f6ebdc457786c3869ceb8a11ee0c4a63db53a080961075c15edff2a7640293bf", "third_party/wb_runner/configs/harnesses/oc-harness/_defaults.yaml": "418843868dc1b01431b339791ae63bb23c88cad3238bf9ab844fbefae8acd71e", "third_party/wb_runner/configs/harnesses/oc-harness/versions/1.14.18.yaml": "fd0515a9d539c011501984b1017f90410cef79045e172c20a9598596f39106a6", diff --git a/dependencies/PINNED.md b/plugins/harness2-benchmarks/harness2_benchmarks/dependencies_meta/PINNED.md similarity index 84% rename from dependencies/PINNED.md rename to plugins/harness2-benchmarks/harness2_benchmarks/dependencies_meta/PINNED.md index bd62a57c..9155f451 100644 --- a/dependencies/PINNED.md +++ b/plugins/harness2-benchmarks/harness2_benchmarks/dependencies_meta/PINNED.md @@ -1,6 +1,6 @@ # Upstream pins and patches -`harness2/config.py` is the authoritative pin table. The setup script fetches +`harness2_benchmarks/config.py` is the authoritative pin table. The setup script fetches these commits directly from the public remotes and refuses to overwrite a checkout at another revision. @@ -26,7 +26,7 @@ checkout at another revision. undefined constant in the upstream Anthropic route support. Patches are checked before applying and skipped if already applied. Derived runner -files belong in `harness2/third_party/{jb,lab,wb}_runner/`, which this copy does not +files belong in `harness2_benchmarks/third_party/{jb,lab,wb}_runner/`, which this copy does not include; setup copies the LAB and WorkBuddy overlays into the upstream checkouts. The upstream dependency locks are used for Python installations. OpenCode uses @@ -35,8 +35,8 @@ JobBench's Hugging Face revision is resolved once for a download and recorded in `dataset/main/.harness2-download.json`. WorkBuddy's dataset fetcher, as patched by setup, verifies each archive against the dataset repository's `SHA256SUMS` and refuses one it cannot verify; see -[the WorkBuddy guide](../docs/BENCHMARKS.md#workbuddy-bench). LAB task files are part of its pinned Git checkout. +[the WorkBuddy guide](../../docs/BENCHMARKS.md#workbuddy-bench). LAB task files are part of its pinned Git checkout. These inputs support the public execution workflow. Historical paper runs may have used different tasks, model deployments, and additional local tooling; -see [experiment notes](../README.md#experiments). +see [experiment notes](../../../../README.md#experiments). diff --git a/dependencies/README.md b/plugins/harness2-benchmarks/harness2_benchmarks/dependencies_meta/README.md similarity index 90% rename from dependencies/README.md rename to plugins/harness2-benchmarks/harness2_benchmarks/dependencies_meta/README.md index 85346f22..d4508617 100644 --- a/dependencies/README.md +++ b/plugins/harness2-benchmarks/harness2_benchmarks/dependencies_meta/README.md @@ -1,8 +1,8 @@ # Third-party code and data Benchmark tools are downloaded into this directory and excluded from version -control. Pins are defined in `harness2/config.py`; setup commands are in the -[benchmark guides](../docs/BENCHMARKS.md). +control. Pins are defined in `harness2_benchmarks/config.py`; setup commands are in the +[benchmark guides](../../docs/BENCHMARKS.md). | Upstream | License | Release integration | |---|---|---| diff --git a/harness2/install.py b/plugins/harness2-benchmarks/harness2_benchmarks/install.py similarity index 98% rename from harness2/install.py rename to plugins/harness2-benchmarks/harness2_benchmarks/install.py index e126bc42..dee43c6e 100644 --- a/harness2/install.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/install.py @@ -54,8 +54,8 @@ def missing_runners() -> str: return "" return ( "benchmark runners and patches are not included in this copy " - f"({len(absent)} of {len(need)} missing under third_party/); " - "see third_party/README.md" + f"({len(absent)} of {len(need)} missing under {config.THIRD_PARTY}/); " + f"see {config.THIRD_PARTY}/README.md" ) @@ -229,7 +229,7 @@ def _share_with_containers(root: Path) -> None: path.chmod(mode | extra) -CODEX_CONFIG_REMEDY = "rerun harness2-setup --substrate cx" +CODEX_CONFIG_REMEDY = "rerun harness2 setup --substrate cx" CODEX_PROVIDER = "harness2" @@ -286,7 +286,7 @@ def download_jobbench(*, dry_run=False): """Install the JobBench task tree from its public Hugging Face dataset repo. The dataset is public, so an executor with web access may retrieve it; - see `docs/BENCHMARKS.md`. + see `plugins/harness2-benchmarks/docs/BENCHMARKS.md`. """ repo = config.jb_repo() dest = repo / "dataset" / "main" diff --git a/harness2/benchmarks/jb_opencode_runner.py b/plugins/harness2-benchmarks/harness2_benchmarks/jb_opencode_runner.py similarity index 98% rename from harness2/benchmarks/jb_opencode_runner.py rename to plugins/harness2-benchmarks/harness2_benchmarks/jb_opencode_runner.py index 1b35190d..37091580 100644 --- a/harness2/benchmarks/jb_opencode_runner.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/jb_opencode_runner.py @@ -26,7 +26,9 @@ from datetime import datetime from pathlib import Path -JOBBENCH = Path(__file__).resolve().parents[1] +from harness2_benchmarks import config + +JOBBENCH = config.jb_repo() TEMP_BASE = Path("/tmp/hevo_ws") TRAJ_DIR_NAME = "model_traj" EVENT_PREFIX = '{"type":' @@ -76,7 +78,7 @@ def _settings() -> Settings: render_dir = Path(env["HARNESS_RENDER_DIR"]) if not (render_dir / ".opencode").is_dir(): raise SystemExit(f"[ERROR] HARNESS_RENDER_DIR has no .opencode/: {render_dir}") - opencode_dir = Path(env.get("OPENCODE_DIR") or JOBBENCH / "opencode") + opencode_dir = Path(env.get("OPENCODE_DIR") or config.OPENCODE_DIR) if not (opencode_dir / "packages" / "opencode").is_dir(): raise SystemExit(f"[ERROR] OpenCode not installed at {opencode_dir}") tasks_base = Path(env.get("TASKS_BASE_DIR") or JOBBENCH / "dataset" / "main") diff --git a/harness2/benchmarks/jobbench.py b/plugins/harness2-benchmarks/harness2_benchmarks/jobbench.py similarity index 97% rename from harness2/benchmarks/jobbench.py rename to plugins/harness2-benchmarks/harness2_benchmarks/jobbench.py index 3d5e1720..ce6015bd 100644 --- a/harness2/benchmarks/jobbench.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/jobbench.py @@ -33,14 +33,17 @@ from datetime import datetime, timezone from pathlib import Path -from .. import config -from ..core import runmark as RM -from ..core import tree as HT -from ..core.spec import BenchPrompts, BenchSpec, resolve_channel_model -from ..install import codex_config_error -from ..substrate.base import Install, RunRequest -from ..substrate.codex import CodexSubstrate -from ..substrate.opencode import OUTPUT_TOKEN_MAX, OpencodeSubstrate +from harness2.core import runmark as RM +from harness2.core import tree as HT +from harness2.core.spec import BenchPrompts, BenchSpec +from harness2.substrate.base import Install, RunRequest +from harness2.substrate.codex import CodexSubstrate +from harness2.substrate.opencode import OUTPUT_TOKEN_MAX, OpencodeSubstrate + +from . import config +from .config import resolve_channel_model +from .install import codex_config_error +from .preflight import codex as codex_preflight from .tasks import discover, select _RUNNER_DIR = config.THIRD_PARTY / "jb_runner" @@ -500,17 +503,17 @@ def render_opencode_tree(staged: Path, render_dir: Path, ctx: dict) -> None: def _channels() -> dict: - """Improver channel, resolved through core.spec.resolve_channel_model.""" + """Improver channel, resolved through config.resolve_channel_model.""" ch = resolve_channel_model(None, None, effort="high") return {"proposer": ch, "selector": ch} -def make_spec(substrate: str) -> BenchSpec: +def make_spec(substrate: str, runs_root: Path) -> BenchSpec: if substrate not in _INSTALL_MODE: raise ValueError(f"JobBench substrate must be 'oc' or 'cx', got {substrate!r}") return BenchSpec( name="jb", - runs_root=config.RUNS / "jb", + runs_root=Path(runs_root), install_mode=_INSTALL_MODE[substrate], prompts=jobbench_prompts(), channels=_channels(), @@ -626,12 +629,13 @@ def __init__( self, substrate: str = "oc", *, + runs_root: Path, timeout: int | None = None, store_root: Path | None = None, split_file: Path | None = None, ): self.split_file = split_file - self.spec = make_spec(substrate) + self.spec = make_spec(substrate, runs_root) launcher = self._launch_opencode if substrate == "oc" else self._launch_codex self.substrate = ( OpencodeSubstrate(launcher=launcher) @@ -639,7 +643,7 @@ def __init__( else CodexSubstrate(launcher=launcher) ) self.timeout = int(timeout or _DEFAULT_TIMEOUT[substrate]) - self.store = Path(store_root) if store_root else config.RUNS / "jb" / "dataset" + self.store = Path(store_root) if store_root else self.spec.runs_root / "dataset" def tasks(self, split: str = "all") -> list[str]: return select(discover("jb"), split, self.split_file, benchmark="jb") @@ -658,8 +662,8 @@ def task_prompt(self, task_id: str) -> str: p = self._dataset_task(task_id) / "task_folder" / "TASK_INSTRUCTIONS.txt" if not p.is_file(): raise FileNotFoundError( - f"task instructions missing: {p} — re-materialise dependencies/job-bench-eval's " - f"dataset/main (recipe in dependencies/PINNED.md)" + f"task instructions missing: {p} — re-materialise {config.jb_repo()}'s " + f"dataset/main (recipe in harness2_benchmarks/dependencies_meta/PINNED.md)" ) return p.read_text(encoding="utf-8", errors="replace") @@ -697,8 +701,8 @@ def _ensure_store_task(self, task_id: str) -> Path: src = self._dataset_task(task_id) / "task_folder" if not src.is_dir(): raise FileNotFoundError( - f"dataset task inputs missing: {src} — re-materialise dependencies/job-bench-eval " - f"(recipe in dependencies/PINNED.md)" + f"dataset task inputs missing: {src} — re-materialise {config.jb_repo()} " + f"(recipe in harness2_benchmarks/dependencies_meta/PINNED.md)" ) dst.parent.mkdir(parents=True, exist_ok=True) with _STORE_SEED_LOCK: @@ -785,7 +789,6 @@ def _launch_common(self, req: RunRequest) -> tuple[str, str, int]: def _launch_opencode(self, req: RunRequest, install: Install) -> list[Path]: task_id, label, t = self._launch_common(req) - self.substrate.preflight() render_dir = Path(tempfile.mkdtemp(prefix=f"hrender_{label}_")) render_opencode_tree(install.staging_dir, render_dir, _render_ctx(self.domain_of(task_id))) env = { @@ -807,7 +810,7 @@ def _launch_opencode(self, req: RunRequest, install: Install) -> list[Path]: def _launch_codex(self, req: RunRequest, install: Install) -> list[Path]: task_id, label, t = self._launch_common(req) - self.substrate.preflight() + codex_preflight() codex_home = config.CODEX_HOME_DIR stale = codex_config_error() if stale is not None: @@ -952,7 +955,7 @@ def base_harness(self) -> Path: if not HT.is_tree(self.substrate.surface(), base): raise FileNotFoundError( f"JobBench base harness missing or not a {self.substrate.surface().name} " - f"tree: {base} — restore third_party/jb_runner/; an absent base would make " + f"tree: {base} — restore {config.THIRD_PARTY / 'jb_runner'}/; an absent base would make " f"every base rollout silently harness-free" ) return base @@ -963,11 +966,12 @@ def validate_tree(self, tree: dict) -> None: def preflight(self) -> None: """Check everything money-adjacent before a rollout is paid for.""" - self.substrate.preflight() + if self.substrate.name == "cx": + codex_preflight() judge_py = config.jb_repo() / "eval" / "judge.py" if not judge_py.is_file(): raise RuntimeError( - f"upstream JobBench judge missing at {judge_py}; run scripts/setup_benchmark.py jb" + f"upstream JobBench judge missing at {judge_py}; run harness2 setup jb" ) if self.substrate.name == "cx": config.require("HARNESS2_CODEX_API_KEY", needed_by="JB codex cells") @@ -978,7 +982,7 @@ def preflight(self) -> None: if stale is not None: raise RuntimeError(stale) py = config.JB_JUDGE_PYTHON - remedy = "python scripts/setup_benchmark.py jb" + remedy = "harness2 setup jb" if not py.is_file(): raise RuntimeError(f"JB judge interpreter missing at {py}; {remedy}") try: @@ -1078,7 +1082,7 @@ def _run_judge(self, task_id: str, label: str, run_dir: Path) -> bool: py = config.JB_JUDGE_PYTHON if not py.is_file(): print( - f" [jb] judge interpreter missing ({py}); run scripts/setup_benchmark.py jb " + f" [jb] judge interpreter missing ({py}); run harness2 setup jb " f"to a python with pandas/openpyxl/python-docx/mammoth/pdfplumber -- judge " f"not run (None, never 0)", flush=True, diff --git a/harness2/benchmarks/lab.py b/plugins/harness2-benchmarks/harness2_benchmarks/lab.py similarity index 97% rename from harness2/benchmarks/lab.py rename to plugins/harness2-benchmarks/harness2_benchmarks/lab.py index 77320b21..600e7df2 100644 --- a/harness2/benchmarks/lab.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/lab.py @@ -31,13 +31,16 @@ from functools import cache from pathlib import Path -from .. import config -from ..core import runmark as RM -from ..core import tree as HT -from ..core.spec import BenchPrompts, BenchSpec, resolve_channel_model -from ..substrate.base import Install, RunRequest -from ..substrate.codex import CODEX_SURFACE, CodexSubstrate -from ..substrate.opencode import OPENCODE_SURFACE, OpencodeSubstrate +from harness2.core import runmark as RM +from harness2.core import tree as HT +from harness2.core.spec import BenchPrompts, BenchSpec +from harness2.substrate.base import Install, RunRequest +from harness2.substrate.codex import CODEX_SURFACE, CodexSubstrate +from harness2.substrate.opencode import OPENCODE_SURFACE, OpencodeSubstrate + +from . import config +from .config import resolve_channel_model +from .preflight import codex as codex_preflight from .tasks import discover, select JUDGE_MODEL = config.LAB_JUDGE_MODEL @@ -424,8 +427,8 @@ def _require_runner(path: Path) -> None: if not Path(path).is_file(): raise RuntimeError( f"upstream LAB runner missing: {path}. The upstream clone does not track it at " - f"the pinned commit; it ships as the tracked overlay third_party/lab_runner/ — " - f"run scripts/setup_benchmark.py to copy it into place." + f"the pinned commit; it ships as the tracked overlay {config.THIRD_PARTY / 'lab_runner'}/ — " + f"run harness2 setup to copy it into place." ) @@ -470,19 +473,19 @@ def _require_runtime_scripts() -> frozenset[str]: f"no skill scripts under {skills} — an empty exemption set is precisely the " f"condition this derivation exists to prevent, and it fails silently as mass " f"fell_back rather than as an error; materialise the LAB checkout with " - f"scripts/setup_benchmark.py lab" + f"harness2 setup lab" ) return names -def lab_spec(substrate_name: str) -> BenchSpec: +def lab_spec(substrate_name: str, runs_root: Path) -> BenchSpec: """The LAB BenchSpec for one substrate cell. A value, not a code path.""" if substrate_name not in ("oc", "cx"): raise ValueError(f"substrate must be 'oc' or 'cx', got {substrate_name!r}") channel = resolve_channel_model(None, None, effort="high") return BenchSpec( name="lab", - runs_root=config.RUNS / "lab", + runs_root=Path(runs_root), install_mode="workspace_config" if substrate_name == "oc" else "codex_root", prompts=LAB_BASE, channels={"proposer": channel, "selector": channel}, @@ -506,6 +509,7 @@ class LabAdapter: def __init__( self, *, + runs_root: Path, substrate: str = "oc", model: str | None = None, timeout: int | None = None, @@ -517,16 +521,16 @@ def __init__( self.split_file = split_file if skill_mode not in ("native", "inline"): raise ValueError(f"skill_mode must be 'native' or 'inline', got {skill_mode!r}") - self.spec = lab_spec(substrate) + self.spec = lab_spec(substrate, runs_root) if substrate == "oc": self.substrate = OpencodeSubstrate(launcher=self._launch_oc) self.model = model or config.SOLVER_MODEL_OC - self._results_root = config.RUNS / "lab" / "results" / "oc" + self._results_root = self.spec.runs_root / "results" / "oc" else: self.substrate = CodexSubstrate(launcher=self._launch_cx) self.model = model or config.SOLVER_MODEL_CX - self._results_root = config.RUNS / "lab" / "results" / "cx" + self._results_root = self.spec.runs_root / "results" / "cx" self.timeout = int(timeout or 3600) self.skill_mode = skill_mode self.solver_effort = solver_effort @@ -712,7 +716,7 @@ def _launch_cx(self, req: RunRequest, install: Install) -> list[Path]: "ADAPT_LABCX_CODEX_BIN": str(Path(config.CODEX_DIST) / "codex"), "ADAPT_LABCX_RESULTS": str(self._results_root), "ADAPT_LABCX_SCRATCH": os.environ.get("ADAPT_LABCX_SCRATCH") - or str(config.RUNS / "lab" / "scratch"), + or str(self.spec.runs_root / "scratch"), "ADAPT_LABCX_BASE_URL": os.environ.get("ADAPT_LABCX_BASE_URL") or config.CODEX_BASE_URL, } @@ -930,10 +934,11 @@ def _check_tool_gate(self, run_dir: Path) -> None: def preflight(self) -> None: """Everything money-adjacent, checked before a rollout is paid for.""" - self.substrate.preflight() + if self.substrate.name == "cx": + codex_preflight() if not config.LAB_PYTHON.is_file(): raise RuntimeError( - f"LAB interpreter missing at {config.LAB_PYTHON}; run python scripts/setup_benchmark.py lab" + f"LAB interpreter missing at {config.LAB_PYTHON}; run harness2 setup lab" ) if self.substrate.name == "oc": _require_runner(config.lab_repo() / "opencode" / "run_opencode.py") @@ -955,7 +960,7 @@ def preflight(self) -> None: raise RuntimeError( f"`import evaluation.scoring` fails under {config.LAB_PYTHON} " f"({r.stderr.strip().splitlines()[-1] if r.stderr.strip() else 'no output'}) " - f"— run python scripts/setup_benchmark.py lab to install its dependency environment" + f"— run harness2 setup lab to install its dependency environment" ) def score(self, run_dir: Path, task_id: str) -> dict | None: diff --git a/harness2/preflight.py b/plugins/harness2-benchmarks/harness2_benchmarks/preflight.py similarity index 79% rename from harness2/preflight.py rename to plugins/harness2-benchmarks/harness2_benchmarks/preflight.py index 93651cde..16e7f7c7 100644 --- a/harness2/preflight.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/preflight.py @@ -18,11 +18,13 @@ import json import shutil +import socket import subprocess +from pathlib import Path +from urllib.parse import urlparse from . import config from .install import PATCHES, missing_runners -from .substrate.opencode import OpencodeSubstrate def check(adapter, *, judge=False) -> list[str]: @@ -31,7 +33,7 @@ def check(adapter, *, judge=False) -> list[str]: return [missing] errors = [] - def probe(label, cmd, *, cwd=None, remedy="run harness2-setup for this benchmark"): + def probe(label, cmd, *, cwd=None, remedy="run harness2 setup "): try: r = subprocess.run( [str(c) for c in cmd], cwd=cwd, capture_output=True, text=True, timeout=45 @@ -46,7 +48,6 @@ def probe(label, cmd, *, cwd=None, remedy="run harness2-setup for this benchmark for validate in ( lambda: config.validate_models(judge=judge), adapter.tasks, - OpencodeSubstrate().preflight, adapter.preflight, ): try: @@ -73,8 +74,15 @@ def probe(label, cmd, *, cwd=None, remedy="run harness2-setup for this benchmark f"{dependency} patch", ["git", "-C", target, "apply", "--reverse", "--check", patch], ) + if not (config.OPENCODE_DIR / "packages/opencode/src/index.ts").is_file(): + errors.append( + f"OpenCode checkout missing at {config.OPENCODE_DIR}; " + f"run harness2 setup {adapter.spec.name}" + ) + if not config.BUN.is_file(): + errors.append(f"bun not found at {config.BUN}; run harness2 setup {adapter.spec.name}") if not (config.OPENCODE_DIR / "node_modules").is_dir(): - errors.append(f"OpenCode dependencies missing; run harness2-setup {bench}") + errors.append(f"OpenCode dependencies missing; run harness2 setup {adapter.spec.name}") if any( m.startswith("google-vertex") for m in (config.SOLVER_MODEL_OC, config.IMPROVER_MODEL) @@ -119,7 +127,7 @@ def probe(label, cmd, *, cwd=None, remedy="run harness2-setup for this benchmark probe( "WorkBuddy document rendering", [config.WB_RENDER_PY, "-c", "import openpyxl, pptx"], - remedy="install 'harness2[render]' in the main environment", + remedy="install 'harness2-benchmarks[render]' in the main environment", ) probe( "WorkBuddy runtime", @@ -158,3 +166,25 @@ def require_ready(adapter, *, judge=False): raise config.ConfigError( "Setup incomplete; nothing has run yet:\n - " + "\n - ".join(errors) ) + + +def codex(*, require_bridge: bool = False) -> None: + """Check the CLI and Responses endpoint, including container configuration.""" + config.require("HARNESS2_CODEX_BASE_URL", needed_by="Codex Responses endpoint") + config.require("HARNESS2_CODEX_API_KEY", needed_by="Codex endpoint authentication") + url = urlparse(config.CODEX_BASE_URL) + if url.scheme not in ("http", "https") or not url.hostname: + raise config.ConfigError("HARNESS2_CODEX_BASE_URL must be an HTTP(S) URL") + binary = Path(config.CODEX_DIST) / "codex" + if not binary.is_file(): + raise RuntimeError( + "Codex is not installed; run harness2 setup --substrate cx" + ) + if require_bridge and not config.CODEX_CONTAINER_URL: + raise RuntimeError( + "Set HARNESS2_CODEX_CONTAINER_URL to a container-accessible Responses endpoint" + ) + with socket.create_connection( + (url.hostname, url.port or (443 if url.scheme == "https" else 80)), timeout=5 + ): + pass diff --git a/harness2/cli/setup.py b/plugins/harness2-benchmarks/harness2_benchmarks/setup.py similarity index 97% rename from harness2/cli/setup.py rename to plugins/harness2-benchmarks/harness2_benchmarks/setup.py index 6c9afc35..716ae3aa 100644 --- a/harness2/cli/setup.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/setup.py @@ -20,7 +20,7 @@ import shutil import subprocess -from harness2 import config, install +from . import config, install def main(argv=None): @@ -111,7 +111,7 @@ def main(argv=None): print( "Setup plan complete." if dry - else "Installation complete. Run harness2-check before an experiment." + else "Installation complete. Run harness2 doctor before an experiment." ) return 0 diff --git a/plugins/harness2-benchmarks/harness2_benchmarks/suites/__init__.py b/plugins/harness2-benchmarks/harness2_benchmarks/suites/__init__.py new file mode 100644 index 00000000..2d3cc526 --- /dev/null +++ b/plugins/harness2-benchmarks/harness2_benchmarks/suites/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Benchmark suite entry points.""" + +from __future__ import annotations diff --git a/plugins/harness2-benchmarks/harness2_benchmarks/suites/jb.py b/plugins/harness2-benchmarks/harness2_benchmarks/suites/jb.py new file mode 100644 index 00000000..fd4c6e43 --- /dev/null +++ b/plugins/harness2-benchmarks/harness2_benchmarks/suites/jb.py @@ -0,0 +1,38 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""jb suite entry point.""" + +from __future__ import annotations + +from .. import preflight +from ..config import ConfigError +from ..jobbench import JobBenchAdapter +from ..setup import main + + +def make_adapter(ctx): + try: + substrate = {"opencode": "oc", "codex": "cx"}[ctx.settings.get("executor.name")] + except KeyError as exc: + raise ConfigError("the jb suite runs on the opencode or codex executor") from exc + return JobBenchAdapter(substrate=substrate, runs_root=ctx.run_root, split_file=ctx.split_file) + + +def check(ctx) -> list[str]: + return preflight.check(make_adapter(ctx), judge=True) + + +def setup(argv: list[str]) -> int: + return main(["jb", *argv]) diff --git a/plugins/harness2-benchmarks/harness2_benchmarks/suites/lab.py b/plugins/harness2-benchmarks/harness2_benchmarks/suites/lab.py new file mode 100644 index 00000000..e5ff4fce --- /dev/null +++ b/plugins/harness2-benchmarks/harness2_benchmarks/suites/lab.py @@ -0,0 +1,38 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""lab suite entry point.""" + +from __future__ import annotations + +from .. import preflight +from ..config import ConfigError +from ..lab import LabAdapter +from ..setup import main + + +def make_adapter(ctx): + try: + substrate = {"opencode": "oc", "codex": "cx"}[ctx.settings.get("executor.name")] + except KeyError as exc: + raise ConfigError("the lab suite runs on the opencode or codex executor") from exc + return LabAdapter(substrate=substrate, runs_root=ctx.run_root, split_file=ctx.split_file) + + +def check(ctx) -> list[str]: + return preflight.check(make_adapter(ctx), judge=True) + + +def setup(argv: list[str]) -> int: + return main(["lab", *argv]) diff --git a/plugins/harness2-benchmarks/harness2_benchmarks/suites/wb_code.py b/plugins/harness2-benchmarks/harness2_benchmarks/suites/wb_code.py new file mode 100644 index 00000000..a8b2b70f --- /dev/null +++ b/plugins/harness2-benchmarks/harness2_benchmarks/suites/wb_code.py @@ -0,0 +1,38 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""wb-code suite entry point.""" + +from __future__ import annotations + +from .. import preflight +from ..config import ConfigError +from ..setup import main +from ..workbuddy import WbAdapter + + +def make_adapter(ctx): + try: + substrate = {"opencode": "oc", "codex": "cx"}[ctx.settings.get("executor.name")] + except KeyError as exc: + raise ConfigError("the wb-code suite runs on the opencode or codex executor") from exc + return WbAdapter("code", substrate=substrate, runs_root=ctx.run_root, split_file=ctx.split_file) + + +def check(ctx) -> list[str]: + return preflight.check(make_adapter(ctx), judge=True) + + +def setup(argv: list[str]) -> int: + return main(["wb", "--subset", "code", *argv]) diff --git a/plugins/harness2-benchmarks/harness2_benchmarks/suites/wb_office.py b/plugins/harness2-benchmarks/harness2_benchmarks/suites/wb_office.py new file mode 100644 index 00000000..b656da92 --- /dev/null +++ b/plugins/harness2-benchmarks/harness2_benchmarks/suites/wb_office.py @@ -0,0 +1,40 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""wb-office suite entry point.""" + +from __future__ import annotations + +from .. import preflight +from ..config import ConfigError +from ..setup import main +from ..workbuddy import WbAdapter + + +def make_adapter(ctx): + try: + substrate = {"opencode": "oc", "codex": "cx"}[ctx.settings.get("executor.name")] + except KeyError as exc: + raise ConfigError("the wb-office suite runs on the opencode or codex executor") from exc + return WbAdapter( + "office", substrate=substrate, runs_root=ctx.run_root, split_file=ctx.split_file + ) + + +def check(ctx) -> list[str]: + return preflight.check(make_adapter(ctx), judge=True) + + +def setup(argv: list[str]) -> int: + return main(["wb", "--subset", "office", *argv]) diff --git a/plugins/harness2-benchmarks/harness2_benchmarks/suites/wb_web.py b/plugins/harness2-benchmarks/harness2_benchmarks/suites/wb_web.py new file mode 100644 index 00000000..cb1baca0 --- /dev/null +++ b/plugins/harness2-benchmarks/harness2_benchmarks/suites/wb_web.py @@ -0,0 +1,38 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""wb-web suite entry point.""" + +from __future__ import annotations + +from .. import preflight +from ..config import ConfigError +from ..setup import main +from ..workbuddy import WbAdapter + + +def make_adapter(ctx): + try: + substrate = {"opencode": "oc", "codex": "cx"}[ctx.settings.get("executor.name")] + except KeyError as exc: + raise ConfigError("the wb-web suite runs on the opencode or codex executor") from exc + return WbAdapter("web", substrate=substrate, runs_root=ctx.run_root, split_file=ctx.split_file) + + +def check(ctx) -> list[str]: + return preflight.check(make_adapter(ctx), judge=True) + + +def setup(argv: list[str]) -> int: + return main(["wb", "--subset", "web", *argv]) diff --git a/harness2/benchmarks/tasks.py b/plugins/harness2-benchmarks/harness2_benchmarks/tasks.py similarity index 93% rename from harness2/benchmarks/tasks.py rename to plugins/harness2-benchmarks/harness2_benchmarks/tasks.py index 6fff81cd..8281ae71 100644 --- a/harness2/benchmarks/tasks.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/tasks.py @@ -20,7 +20,7 @@ import tomllib from pathlib import Path -from .. import config +from . import config def discover(bench: str, subset: str | None = None) -> dict[str, str]: @@ -53,8 +53,8 @@ def discover(bench: str, subset: str | None = None) -> dict[str, str]: raise ValueError(f"Unknown benchmark/subset: {bench}/{subset}") if not tasks: raise FileNotFoundError( - f"No {bench} tasks found at {root}. Run harness2-setup " - f"{bench}" + (f" --subset {subset}" if subset else "") + "." + f"No {bench} tasks found at {root}. Run harness2 setup " + f"{bench}" + (f"-{subset}" if subset else "") + "." ) return tasks @@ -66,7 +66,7 @@ def select(tasks: dict[str, str], split: str, path: Path | None, *, benchmark: s if split != "all": raise ValueError( "--split train/test requires --split-file; create one with " - "python scripts/split_tasks.py, or use --split all" + "python plugins/harness2-benchmarks/scripts/split_tasks.py, or use --split all" ) return sorted(tasks) data = json.loads(Path(path).read_text(encoding="utf-8")) diff --git a/third_party/README.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/README.md similarity index 71% rename from third_party/README.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/README.md index ba0d8279..618821d7 100644 --- a/third_party/README.md +++ b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/README.md @@ -13,9 +13,9 @@ so setup works outside a source checkout. | `licenses/` | License copies covering those patches | Each runner retains its `LICENSE` and `PROVENANCE.md`. Research logic belongs in -`harness2/core/`; benchmark interfaces belong in `harness2/benchmarks/`. -Setup downloads upstream checkouts and data into the invocation's `dependencies/` +`harness2/core/`; benchmark interfaces belong in `harness2_benchmarks/`. +Setup downloads upstream checkouts and data into the invocation's `~/.cache/harness2/benchmarks/` directory. Those downloads are excluded from version control. -The repository's `dependencies/ASSETS.json` records bundled asset hashes. -Run `python scripts/check_provenance.py` after changing these files. +The repository's `harness2_benchmarks/dependencies_meta/ASSETS.json` records bundled asset hashes. +Run `python plugins/harness2-benchmarks/scripts/check_provenance.py` after changing these files. diff --git a/third_party/jb_runner/LICENSE b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/LICENSE similarity index 100% rename from third_party/jb_runner/LICENSE rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/LICENSE diff --git a/third_party/jb_runner/PROVENANCE.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/PROVENANCE.md similarity index 85% rename from third_party/jb_runner/PROVENANCE.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/PROVENANCE.md index 65e16ced..baf98997 100644 --- a/third_party/jb_runner/PROVENANCE.md +++ b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/PROVENANCE.md @@ -1,7 +1,7 @@ # job-bench-eval runner assets The files in this directory derive from the job-bench-eval integration and retain its -[upstream license](LICENSE). See the [pinned revision and patches](../../../dependencies/PINNED.md). +[upstream license](LICENSE). See the [pinned revision and patches](../../dependencies_meta/PINNED.md). What is still vendored here: @@ -23,5 +23,5 @@ with upstream. It has been replaced by the first-party Python module which the adapter launches as a subprocess in its place. That module is release code, not a vendored asset, so it is not listed in `ASSETS.json`. -[ASSETS.json](../../../dependencies/ASSETS.json) records the shipped file checksums. -Verify them with `python scripts/check_provenance.py`. +[ASSETS.json](../../dependencies_meta/ASSETS.json) records the shipped file checksums. +Verify them with `python plugins/harness2-benchmarks/scripts/check_provenance.py`. diff --git a/third_party/jb_runner/codex_base_harness/AGENTS.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/codex_base_harness/AGENTS.md similarity index 100% rename from third_party/jb_runner/codex_base_harness/AGENTS.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/codex_base_harness/AGENTS.md diff --git a/third_party/jb_runner/harness_baseline_oc/LongTermMEMORY.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/LongTermMEMORY.md similarity index 100% rename from third_party/jb_runner/harness_baseline_oc/LongTermMEMORY.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/LongTermMEMORY.md diff --git a/third_party/jb_runner/harness_baseline_oc/guardrails.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/guardrails.md similarity index 100% rename from third_party/jb_runner/harness_baseline_oc/guardrails.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/guardrails.md diff --git a/third_party/jb_runner/harness_baseline_oc/harness.yaml b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/harness.yaml similarity index 100% rename from third_party/jb_runner/harness_baseline_oc/harness.yaml rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/harness.yaml diff --git a/third_party/jb_runner/harness_baseline_oc/plugin/README.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/plugin/README.md similarity index 100% rename from third_party/jb_runner/harness_baseline_oc/plugin/README.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/plugin/README.md diff --git a/third_party/jb_runner/harness_baseline_oc/scripts/README.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/scripts/README.md similarity index 100% rename from third_party/jb_runner/harness_baseline_oc/scripts/README.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/scripts/README.md diff --git a/third_party/jb_runner/harness_baseline_oc/sub_agents/README.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/sub_agents/README.md similarity index 100% rename from third_party/jb_runner/harness_baseline_oc/sub_agents/README.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/sub_agents/README.md diff --git a/third_party/jb_runner/harness_baseline_oc/systemprompt.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/systemprompt.md similarity index 100% rename from third_party/jb_runner/harness_baseline_oc/systemprompt.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/systemprompt.md diff --git a/third_party/jb_runner/harness_baseline_oc/tool_descriptions/tool_guidance.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/tool_descriptions/tool_guidance.md similarity index 100% rename from third_party/jb_runner/harness_baseline_oc/tool_descriptions/tool_guidance.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/tool_descriptions/tool_guidance.md diff --git a/third_party/jb_runner/run_benchmark_codex_cli_adapt.sh b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/run_benchmark_codex_cli_adapt.sh similarity index 98% rename from third_party/jb_runner/run_benchmark_codex_cli_adapt.sh rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/run_benchmark_codex_cli_adapt.sh index 0105da58..0dc6cf8e 100644 --- a/third_party/jb_runner/run_benchmark_codex_cli_adapt.sh +++ b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/run_benchmark_codex_cli_adapt.sh @@ -15,7 +15,7 @@ set -e # Which split of the dataset to run against (main | easy). Defaults to main. SPLIT="${SPLIT:-main}" # Target task root (the directory containing all profession subdirectories) -TASKS_BASE_DIR="${TASKS_BASE_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)/dataset/${SPLIT}}" +TASKS_BASE_DIR="${TASKS_BASE_DIR:-${HARNESS2_BENCH_DEPS:-${XDG_CACHE_HOME:-$HOME/.cache}/harness2/benchmarks}/job-bench-eval/dataset/${SPLIT}}" # Optional run label appended to output directory names to avoid clobbering past runs RUN_LABEL="${RUN_LABEL:-}" # TASK_IDS restricts discovery to space-separated profession/taskN identifiers. @@ -490,8 +490,8 @@ main() { # Check the executable installed by the release setup script. echo "Checking Codex CLI availability..." - "${HARNESS2_CODEX_BIN:?Run scripts/setup_benchmark.py with --substrate cx}" --version 2>&1 || { - echo "[ERROR] Codex CLI not available. Run scripts/setup_benchmark.py with --substrate cx." + "${HARNESS2_CODEX_BIN:?Run harness2 setup jb --substrate cx}" --version 2>&1 || { + echo "[ERROR] Codex CLI not available. Run harness2 setup jb --substrate cx." exit 1 } diff --git a/third_party/lab_runner/LICENSE b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/LICENSE similarity index 100% rename from third_party/lab_runner/LICENSE rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/LICENSE diff --git a/third_party/lab_runner/PROVENANCE.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/PROVENANCE.md similarity index 66% rename from third_party/lab_runner/PROVENANCE.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/PROVENANCE.md index 25c29d8f..c9327194 100644 --- a/third_party/lab_runner/PROVENANCE.md +++ b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/PROVENANCE.md @@ -1,11 +1,11 @@ # harvey-labs runner assets These runners derive from the harvey-labs integration and retain its -[upstream license](LICENSE). See the [pinned revision and patches](../../../dependencies/PINNED.md). +[upstream license](LICENSE). See the [pinned revision and patches](../../dependencies_meta/PINNED.md). The integration adds editable harness installation and configurable models, paths, and endpoints. Local model-provider names, environment variables, and log paths use the `harness2` namespace. Setup installs these runners into the LAB checkout as overlays. -[ASSETS.json](../../../dependencies/ASSETS.json) records the shipped file checksums. -Verify them with `python scripts/check_provenance.py`. +[ASSETS.json](../../dependencies_meta/ASSETS.json) records the shipped file checksums. +Verify them with `python plugins/harness2-benchmarks/scripts/check_provenance.py`. diff --git a/third_party/lab_runner/adapt_runs_cx/run_codex_lab.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/adapt_runs_cx/run_codex_lab.py similarity index 99% rename from third_party/lab_runner/adapt_runs_cx/run_codex_lab.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/adapt_runs_cx/run_codex_lab.py index cb4ed81c..ea08a58d 100644 --- a/third_party/lab_runner/adapt_runs_cx/run_codex_lab.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/adapt_runs_cx/run_codex_lab.py @@ -19,6 +19,7 @@ from pathlib import Path RUNNER_DIR = Path(__file__).resolve().parent +# install.overlays copies this runner into the upstream checkout before execution. REPO = RUNNER_DIR.parents[1] if str(REPO) not in sys.path: sys.path.insert(0, str(REPO)) diff --git a/third_party/lab_runner/opencode/README.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/README.md similarity index 100% rename from third_party/lab_runner/opencode/README.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/README.md diff --git a/third_party/lab_runner/opencode/__init__.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/__init__.py similarity index 100% rename from third_party/lab_runner/opencode/__init__.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/__init__.py diff --git a/third_party/lab_runner/opencode/render.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/render.py similarity index 99% rename from third_party/lab_runner/opencode/render.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/render.py index 747c7469..189f9c81 100644 --- a/third_party/lab_runner/opencode/render.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/render.py @@ -20,7 +20,6 @@ load_skills, ) -BENCH_ROOT = Path(__file__).resolve().parent.parent ADAPTER_SCRIPTS = Path(__file__).resolve().parent / "scripts" AGENT_NAME = "lab" diff --git a/third_party/lab_runner/opencode/render_tree.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/render_tree.py similarity index 100% rename from third_party/lab_runner/opencode/render_tree.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/render_tree.py diff --git a/third_party/lab_runner/opencode/run_opencode.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/run_opencode.py similarity index 100% rename from third_party/lab_runner/opencode/run_opencode.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/run_opencode.py diff --git a/third_party/lab_runner/opencode/scripts/read_doc.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/scripts/read_doc.py similarity index 100% rename from third_party/lab_runner/opencode/scripts/read_doc.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/scripts/read_doc.py diff --git a/third_party/lab_runner/opencode/session.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/session.py similarity index 100% rename from third_party/lab_runner/opencode/session.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/opencode/session.py diff --git a/third_party/lab_runner/tts_lab/__init__.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/tts_lab/__init__.py similarity index 100% rename from third_party/lab_runner/tts_lab/__init__.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/tts_lab/__init__.py diff --git a/third_party/lab_runner/tts_lab/config.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/tts_lab/config.py similarity index 85% rename from third_party/lab_runner/tts_lab/config.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/tts_lab/config.py index 83eeb647..c5ed8313 100644 --- a/third_party/lab_runner/tts_lab/config.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/lab_runner/tts_lab/config.py @@ -3,6 +3,7 @@ import json from pathlib import Path +# install.overlays copies this runner into the upstream checkout before execution. BENCH_ROOT = Path(__file__).resolve().parents[2] TASKS_DIR = BENCH_ROOT / "tasks" diff --git a/third_party/licenses/harvey-labs.LICENSE b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/licenses/harvey-labs.LICENSE similarity index 100% rename from third_party/licenses/harvey-labs.LICENSE rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/licenses/harvey-labs.LICENSE diff --git a/third_party/licenses/job-bench-eval.LICENSE b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/licenses/job-bench-eval.LICENSE similarity index 100% rename from third_party/licenses/job-bench-eval.LICENSE rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/licenses/job-bench-eval.LICENSE diff --git a/third_party/licenses/opencode.LICENSE b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/licenses/opencode.LICENSE similarity index 100% rename from third_party/licenses/opencode.LICENSE rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/licenses/opencode.LICENSE diff --git a/third_party/licenses/workbuddy-bench.LICENSE b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/licenses/workbuddy-bench.LICENSE similarity index 100% rename from third_party/licenses/workbuddy-bench.LICENSE rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/licenses/workbuddy-bench.LICENSE diff --git a/third_party/patches/harvey-labs-judge-vertex-anthropic.diff b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/patches/harvey-labs-judge-vertex-anthropic.diff similarity index 100% rename from third_party/patches/harvey-labs-judge-vertex-anthropic.diff rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/patches/harvey-labs-judge-vertex-anthropic.diff diff --git a/third_party/patches/job-bench-eval-judge.diff b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/patches/job-bench-eval-judge.diff similarity index 100% rename from third_party/patches/job-bench-eval-judge.diff rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/patches/job-bench-eval-judge.diff diff --git a/third_party/patches/opencode-compat.diff b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/patches/opencode-compat.diff similarity index 100% rename from third_party/patches/opencode-compat.diff rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/patches/opencode-compat.diff diff --git a/third_party/patches/workbuddy-bench-runtime.diff b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/patches/workbuddy-bench-runtime.diff similarity index 100% rename from third_party/patches/workbuddy-bench-runtime.diff rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/patches/workbuddy-bench-runtime.diff diff --git a/third_party/wb_runner/LICENSE b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/LICENSE similarity index 100% rename from third_party/wb_runner/LICENSE rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/LICENSE diff --git a/third_party/wb_runner/PROVENANCE.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/PROVENANCE.md similarity index 68% rename from third_party/wb_runner/PROVENANCE.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/PROVENANCE.md index 4d099148..b33f37b4 100644 --- a/third_party/wb_runner/PROVENANCE.md +++ b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/PROVENANCE.md @@ -1,11 +1,11 @@ # workbuddy-bench runner assets These runners derive from the workbuddy-bench integration and retain its -[upstream license](LICENSE). See the [pinned revision and patches](../../../dependencies/PINNED.md). +[upstream license](LICENSE). See the [pinned revision and patches](../../dependencies_meta/PINNED.md). The integration adds editable harness installation and configurable models, paths, and endpoints. Local model-provider names, environment variables, and log paths use the `harness2` namespace. Setup installs these agents and harness configurations into the WorkBuddy checkout. -[ASSETS.json](../../../dependencies/ASSETS.json) records the shipped file checksums. -Verify them with `python scripts/check_provenance.py`. +[ASSETS.json](../../dependencies_meta/ASSETS.json) records the shipped file checksums. +Verify them with `python plugins/harness2-benchmarks/scripts/check_provenance.py`. diff --git a/third_party/wb_runner/agents/codex_agent.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/agents/codex_agent.py similarity index 99% rename from third_party/wb_runner/agents/codex_agent.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/agents/codex_agent.py index f42dbc39..715b2337 100644 --- a/third_party/wb_runner/agents/codex_agent.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/agents/codex_agent.py @@ -84,7 +84,7 @@ async def install(self, environment: BaseEnvironment) -> None: f'echo "ERROR: codex binary not found at {bin_path}." >&2; ' f'echo " The job must bind-mount the staged dist (read-only):" >&2; ' f'echo " environment_override.mounts: [{{type: bind, ' - f'source: dependencies/codex/dist, target: {self._bin_dir}}}]" >&2; ' + f'source: ~/.cache/harness2/benchmarks/codex/dist, target: {self._bin_dir}}}]" >&2; ' f"exit 1; fi && " f'ln -sf "{bin_path}" /usr/local/bin/codex && ' f'v="$(/usr/local/bin/codex --version 2>&1)" && ' diff --git a/third_party/wb_runner/agents/codex_harness_agent.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/agents/codex_harness_agent.py similarity index 100% rename from third_party/wb_runner/agents/codex_harness_agent.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/agents/codex_harness_agent.py diff --git a/third_party/wb_runner/agents/oc_agent.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/agents/oc_agent.py similarity index 100% rename from third_party/wb_runner/agents/oc_agent.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/agents/oc_agent.py diff --git a/third_party/wb_runner/agents/oc_harness_agent.py b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/agents/oc_harness_agent.py similarity index 100% rename from third_party/wb_runner/agents/oc_harness_agent.py rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/agents/oc_harness_agent.py diff --git a/third_party/wb_runner/configs/harnesses/codex-harness/_defaults.yaml b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/codex-harness/_defaults.yaml similarity index 77% rename from third_party/wb_runner/configs/harnesses/codex-harness/_defaults.yaml rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/codex-harness/_defaults.yaml index e29b33f6..622ac3e8 100644 --- a/third_party/wb_runner/configs/harnesses/codex-harness/_defaults.yaml +++ b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/codex-harness/_defaults.yaml @@ -1,4 +1,4 @@ -# The adapter mounts dependencies/codex/dist read-only at /opt/codex and supplies +# The adapter mounts ~/.cache/harness2/benchmarks/codex/dist read-only at /opt/codex and supplies # the model and container-accessible Responses endpoint. The job-local # Chat Completions proxy serves the benchmark judge separately. harness: diff --git a/third_party/wb_runner/configs/harnesses/codex-harness/versions/0.144.6.yaml b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/codex-harness/versions/0.144.6.yaml similarity index 100% rename from third_party/wb_runner/configs/harnesses/codex-harness/versions/0.144.6.yaml rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/codex-harness/versions/0.144.6.yaml diff --git a/third_party/wb_runner/configs/harnesses/oc-harness/_defaults.yaml b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/oc-harness/_defaults.yaml similarity index 100% rename from third_party/wb_runner/configs/harnesses/oc-harness/_defaults.yaml rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/oc-harness/_defaults.yaml diff --git a/third_party/wb_runner/configs/harnesses/oc-harness/versions/1.14.18.yaml b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/oc-harness/versions/1.14.18.yaml similarity index 100% rename from third_party/wb_runner/configs/harnesses/oc-harness/versions/1.14.18.yaml rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/oc-harness/versions/1.14.18.yaml diff --git a/third_party/wb_runner/configs/harnesses/opencode/CONFIG.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/CONFIG.md similarity index 100% rename from third_party/wb_runner/configs/harnesses/opencode/CONFIG.md rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/CONFIG.md diff --git a/third_party/wb_runner/configs/harnesses/opencode/_defaults.yaml b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/_defaults.yaml similarity index 100% rename from third_party/wb_runner/configs/harnesses/opencode/_defaults.yaml rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/_defaults.yaml diff --git a/third_party/wb_runner/configs/harnesses/opencode/docker/Dockerfile b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/docker/Dockerfile similarity index 100% rename from third_party/wb_runner/configs/harnesses/opencode/docker/Dockerfile rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/docker/Dockerfile diff --git a/third_party/wb_runner/configs/harnesses/opencode/presets/settings/v1_14.json b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/presets/settings/v1_14.json similarity index 100% rename from third_party/wb_runner/configs/harnesses/opencode/presets/settings/v1_14.json rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/presets/settings/v1_14.json diff --git a/third_party/wb_runner/configs/harnesses/opencode/versions/1.14.18.yaml b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/versions/1.14.18.yaml similarity index 100% rename from third_party/wb_runner/configs/harnesses/opencode/versions/1.14.18.yaml rename to plugins/harness2-benchmarks/harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/versions/1.14.18.yaml diff --git a/harness2/benchmarks/wb_render.py b/plugins/harness2-benchmarks/harness2_benchmarks/wb_render.py similarity index 99% rename from harness2/benchmarks/wb_render.py rename to plugins/harness2-benchmarks/harness2_benchmarks/wb_render.py index 3c5bee61..48568306 100644 --- a/harness2/benchmarks/wb_render.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/wb_render.py @@ -276,7 +276,7 @@ def _have(mod: str) -> bool: def _bridge_interpreter() -> str: """The renderer interpreter; empty when invoked as the bridge itself.""" try: - from .. import config + from . import config except ImportError: return "" return str(config.WB_RENDER_PY) diff --git a/harness2/benchmarks/workbuddy.py b/plugins/harness2-benchmarks/harness2_benchmarks/workbuddy.py similarity index 96% rename from harness2/benchmarks/workbuddy.py rename to plugins/harness2-benchmarks/harness2_benchmarks/workbuddy.py index 27a95b5c..45e78b34 100644 --- a/harness2/benchmarks/workbuddy.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/workbuddy.py @@ -35,16 +35,19 @@ from functools import lru_cache from pathlib import Path -from .. import config -from ..core import runmark as RM -from ..core import tree as HT -from ..core.selection import order_graded_first -from ..core.spec import BenchPrompts, BenchSpec, resolve_channel_model -from ..substrate import opencode as OC -from ..substrate.base import Install, RunRequest -from ..substrate.codex import CODEX_SURFACE, CodexSubstrate -from ..substrate.opencode import OPENCODE_SURFACE, OpencodeSubstrate +from harness2.core import runmark as RM +from harness2.core import tree as HT +from harness2.core.selection import order_graded_first +from harness2.core.spec import BenchPrompts, BenchSpec +from harness2.substrate import opencode as OC +from harness2.substrate.base import Install, RunRequest +from harness2.substrate.codex import CODEX_SURFACE, CodexSubstrate +from harness2.substrate.opencode import OPENCODE_SURFACE, OpencodeSubstrate + +from . import config from . import wb_render as RENDER +from .config import resolve_channel_model +from .preflight import codex as codex_preflight from .tasks import discover, select SCOREABLE = frozenset({"code", "web", "office"}) @@ -540,7 +543,7 @@ def _job_yaml( else "" ) head = ( - f"# GENERATED by harness2/benchmarks/workbuddy.py -- ephemeral, safe to " + f"# GENERATED by harness2_benchmarks/workbuddy.py -- ephemeral, safe to " f"delete. label={label}\n" ) return ( @@ -650,12 +653,12 @@ def run_batch( return sorted(set(_run_dirs(label_dir)) - before) -def spec_for(subset: str, substrate: str) -> BenchSpec: +def spec_for(subset: str, substrate: str, runs_root: Path) -> BenchSpec: s = SUBSETS[subset] channel = resolve_channel_model(None, None, effort="high") return BenchSpec( name=f"wb-{subset}", - runs_root=config.RUNS / "wb" / subset, + runs_root=Path(runs_root), install_mode="home_dir_jinja" if substrate == "oc" else "codex_root", prompts=workbuddy_prompts(subset), channels={"proposer": channel, "selector": channel}, @@ -672,7 +675,12 @@ class WbAdapter: """One (subset, substrate) cell of WB-Bench. A different cell is a different instance.""" def __init__( - self, subset: str = "office", *, substrate: str = "oc", split_file: Path | None = None + self, + subset: str = "office", + *, + runs_root: Path, + substrate: str = "oc", + split_file: Path | None = None, ): self.split_file = split_file if subset not in SUBSETS: @@ -681,7 +689,7 @@ def __init__( raise ValueError(f"unknown substrate {substrate!r}; one of {SUBSTRATES}") self.subset = subset self.subset_info = SUBSETS[subset] - self.spec = spec_for(subset, substrate) + self.spec = spec_for(subset, substrate, runs_root) self.substrate = ( OpencodeSubstrate(launcher=self._launch) if substrate == "oc" @@ -776,7 +784,7 @@ def _launch(self, req: RunRequest, install: Install) -> list[Path]: k = len(req.run_ids) label = self._label(req.run_ids[0], k) if self.substrate.name == "cx": - self.substrate.preflight(require_dist=True, require_bridge=True) + codex_preflight(require_bridge=True) def _have(task_id: str) -> int: return sum( @@ -936,13 +944,13 @@ def preflight(self) -> None: if shutil.which("uv") is None: raise RuntimeError( "`uv` is not on PATH; WB batches run as `uv run ./scripts/run.sh` inside " - "the benchmark repo (see docs/BENCHMARKS.md)" + "the benchmark repo (see plugins/harness2-benchmarks/docs/BENCHMARKS.md)" ) repo = config.wb_repo() if not (repo / "scripts" / "run.sh").is_file(): raise RuntimeError( f"the WorkBuddy-Bench checkout at {repo} has no scripts/run.sh — run " - f"scripts/setup_benchmark.py wb" + f"harness2 setup wb-{self.subset}" ) slugs = ( [_harness_oc(fork=False), _harness_oc(fork=True)] @@ -955,13 +963,11 @@ def preflight(self) -> None: if not entry.is_file(): raise RuntimeError( f"harness entry {slug!r} is missing at {entry} — the harness configs " - f"ship as third_party/wb_runner/configs/harnesses/; run " - f"python scripts/setup_benchmark.py wb" + f"ship as {config.THIRD_PARTY / 'wb_runner/configs/harnesses'}/; run " + f"harness2 setup wb-{self.subset}" ) if self.substrate.name == "cx": - self.substrate.preflight(require_dist=True, require_bridge=True) - else: - self.substrate.preflight() + codex_preflight(require_bridge=True) def _killed(self, run_dir: Path) -> bool: """True when the capture stream ends on a provider error object, i.e. the rollout was diff --git a/plugins/harness2-benchmarks/pyproject.toml b/plugins/harness2-benchmarks/pyproject.toml new file mode 100644 index 00000000..cca55f09 --- /dev/null +++ b/plugins/harness2-benchmarks/pyproject.toml @@ -0,0 +1,44 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[build-system] +requires = ["setuptools>=77", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "harness2-benchmarks" +version = "0.1.0" +license = "Apache-2.0" +requires-python = ">=3.12,<3.14" +dependencies = ["harness2"] + +[project.optional-dependencies] +render = ["openpyxl>=3.1,<4", "python-pptx>=1,<2"] + +[project.entry-points."harness2.suites"] +jb = "harness2_benchmarks.suites.jb" +lab = "harness2_benchmarks.suites.lab" +wb-office = "harness2_benchmarks.suites.wb_office" +wb-web = "harness2_benchmarks.suites.wb_web" +wb-code = "harness2_benchmarks.suites.wb_code" + +[tool.setuptools] +include-package-data = false + +[tool.setuptools.packages.find] +include = ["harness2_benchmarks*"] +namespaces = false + +[tool.setuptools.package-data] +harness2_benchmarks = ["third_party/**/*", "dependencies_meta/**/*"] diff --git a/scripts/check_provenance.py b/plugins/harness2-benchmarks/scripts/check_provenance.py similarity index 82% rename from scripts/check_provenance.py rename to plugins/harness2-benchmarks/scripts/check_provenance.py index 84dd8a36..90b28f06 100644 --- a/scripts/check_provenance.py +++ b/plugins/harness2-benchmarks/scripts/check_provenance.py @@ -13,17 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Verify the release's runner and patch files against dependencies/ASSETS.json.""" +"""Verify the release's runner and patch files against harness2_benchmarks/dependencies_meta/ASSETS.json.""" import hashlib import json -from pathlib import Path + +from harness2_benchmarks.config import PKG def main(): - root = Path(__file__).resolve().parents[1] + root = PKG try: - expected = json.loads((root / "dependencies/ASSETS.json").read_text()) + expected = json.loads((root / "dependencies_meta/ASSETS.json").read_text()) failures = [] for name, digest in expected.items(): path = root / name diff --git a/scripts/check_setup.py b/plugins/harness2-benchmarks/scripts/check_setup.py similarity index 79% rename from scripts/check_setup.py rename to plugins/harness2-benchmarks/scripts/check_setup.py index a6d231f4..a32dbe29 100644 --- a/scripts/check_setup.py +++ b/plugins/harness2-benchmarks/scripts/check_setup.py @@ -13,9 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Compatibility entry point for harness2.cli.check.""" +"""Compatibility entry point for harness2 doctor.""" -from harness2.cli.check import main +import subprocess +import sys if __name__ == "__main__": - raise SystemExit(main()) + raise SystemExit(subprocess.call(["harness2", "doctor", *sys.argv[1:]])) diff --git a/scripts/run_jobbench.sh b/plugins/harness2-benchmarks/scripts/run_jobbench.sh similarity index 59% rename from scripts/run_jobbench.sh rename to plugins/harness2-benchmarks/scripts/run_jobbench.sh index b00204e1..732e2551 100644 --- a/scripts/run_jobbench.sh +++ b/plugins/harness2-benchmarks/scripts/run_jobbench.sh @@ -14,23 +14,28 @@ # limitations under the License. # Run JobBench end to end: check, adapt, judge, report. -# bash scripts/run_jobbench.sh # every task -# bash scripts/run_jobbench.sh --only-domains biostatisticians --max-tasks 1 +# bash plugins/harness2-benchmarks/scripts/run_jobbench.sh # every task +# bash plugins/harness2-benchmarks/scripts/run_jobbench.sh --only-domains biostatisticians --max-tasks 1 # Settings: SUBSTRATE=oc|cx K=1 MODE=parallel|sequential # Extra arguments go to the adaptation command; --dry-run stops after it. set -uo pipefail -cd "$(dirname "$0")/.." -PATH="$PWD/.venv/bin:$PATH" +PATH="$(cd "$(dirname "$0")/../../.." && pwd)/.venv/bin:$PATH" SUBSTRATE=${SUBSTRATE:-oc} +case "$SUBSTRATE" in + oc) EXECUTOR=opencode ;; + cx) EXECUTOR=codex ;; + *) exit 2 ;; +esac K=${K:-1} MODE=${MODE:-parallel} -run=(--bench jb --substrate "$SUBSTRATE" --k "$K" --mode "$MODE") +name="jb-${EXECUTOR}-k${K}-${MODE}" +run=(--suite jb --name "$name" --k "$K" --mode "$MODE") -harness2-check --bench jb --substrate "$SUBSTRATE" || exit -harness2 "${run[@]}" "$@" +harness2 -c executor.name="$EXECUTOR" -c suite.name=jb doctor || exit +harness2 -c executor.name="$EXECUTOR" run "${run[@]}" "$@" status=$? [[ " $* " == *" --dry-run "* ]] && exit $status -harness2-judge "${run[@]}" +harness2 judge "$name" judged=$? -harness2-report "${run[@]}" +harness2 report "$name" exit $((status ? status : judged)) diff --git a/scripts/run_lab.sh b/plugins/harness2-benchmarks/scripts/run_lab.sh similarity index 59% rename from scripts/run_lab.sh rename to plugins/harness2-benchmarks/scripts/run_lab.sh index df94d803..7e81aad0 100644 --- a/scripts/run_lab.sh +++ b/plugins/harness2-benchmarks/scripts/run_lab.sh @@ -14,23 +14,28 @@ # limitations under the License. # Run LAB end to end: check, adapt, judge, report. -# bash scripts/run_lab.sh # every task -# bash scripts/run_lab.sh --only-domains antitrust-competition --max-tasks 1 +# bash plugins/harness2-benchmarks/scripts/run_lab.sh # every task +# bash plugins/harness2-benchmarks/scripts/run_lab.sh --only-domains antitrust-competition --max-tasks 1 # Settings: SUBSTRATE=oc|cx K=1 MODE=parallel|sequential # Extra arguments go to the adaptation command; --dry-run stops after it. set -uo pipefail -cd "$(dirname "$0")/.." -PATH="$PWD/.venv/bin:$PATH" +PATH="$(cd "$(dirname "$0")/../../.." && pwd)/.venv/bin:$PATH" SUBSTRATE=${SUBSTRATE:-oc} +case "$SUBSTRATE" in + oc) EXECUTOR=opencode ;; + cx) EXECUTOR=codex ;; + *) exit 2 ;; +esac K=${K:-1} MODE=${MODE:-parallel} -run=(--bench lab --substrate "$SUBSTRATE" --k "$K" --mode "$MODE") +name="lab-${EXECUTOR}-k${K}-${MODE}" +run=(--suite lab --name "$name" --k "$K" --mode "$MODE") -harness2-check --bench lab --substrate "$SUBSTRATE" || exit -harness2 "${run[@]}" "$@" +harness2 -c executor.name="$EXECUTOR" -c suite.name=lab doctor || exit +harness2 -c executor.name="$EXECUTOR" run "${run[@]}" "$@" status=$? [[ " $* " == *" --dry-run "* ]] && exit $status -harness2-judge "${run[@]}" +harness2 judge "$name" judged=$? -harness2-report "${run[@]}" +harness2 report "$name" exit $((status ? status : judged)) diff --git a/scripts/run_workbuddy.sh b/plugins/harness2-benchmarks/scripts/run_workbuddy.sh similarity index 60% rename from scripts/run_workbuddy.sh rename to plugins/harness2-benchmarks/scripts/run_workbuddy.sh index 8df4d798..8c3668d7 100644 --- a/scripts/run_workbuddy.sh +++ b/plugins/harness2-benchmarks/scripts/run_workbuddy.sh @@ -14,24 +14,29 @@ # limitations under the License. # Run WorkBuddy-Bench end to end: check, adapt, judge, report. -# bash scripts/run_workbuddy.sh # every task -# bash scripts/run_workbuddy.sh --only-domains doc-ops --max-tasks 1 +# bash plugins/harness2-benchmarks/scripts/run_workbuddy.sh # every task +# bash plugins/harness2-benchmarks/scripts/run_workbuddy.sh --only-domains doc-ops --max-tasks 1 # Settings: SUBSTRATE=oc|cx SUBSET=office|web|code K=1 MODE=parallel|sequential # Extra arguments go to the adaptation command; --dry-run stops after it. set -uo pipefail -cd "$(dirname "$0")/.." -PATH="$PWD/.venv/bin:$PATH" +PATH="$(cd "$(dirname "$0")/../../.." && pwd)/.venv/bin:$PATH" SUBSTRATE=${SUBSTRATE:-oc} SUBSET=${SUBSET:-office} +case "$SUBSTRATE" in + oc) EXECUTOR=opencode ;; + cx) EXECUTOR=codex ;; + *) exit 2 ;; +esac K=${K:-1} MODE=${MODE:-parallel} -run=(--bench wb --subset "$SUBSET" --substrate "$SUBSTRATE" --k "$K" --mode "$MODE") +name="wb-${SUBSET}-${EXECUTOR}-k${K}-${MODE}" +run=(--suite "wb-$SUBSET" --name "$name" --k "$K" --mode "$MODE") -harness2-check --bench wb --subset "$SUBSET" --substrate "$SUBSTRATE" || exit -harness2 "${run[@]}" "$@" +harness2 -c executor.name="$EXECUTOR" -c suite.name="wb-$SUBSET" doctor || exit +harness2 -c executor.name="$EXECUTOR" run "${run[@]}" "$@" status=$? [[ " $* " == *" --dry-run "* ]] && exit $status -harness2-judge "${run[@]}" +harness2 judge "$name" judged=$? -harness2-report "${run[@]}" +harness2 report "$name" exit $((status ? status : judged)) diff --git a/scripts/setup_benchmark.py b/plugins/harness2-benchmarks/scripts/setup_benchmark.py similarity index 86% rename from scripts/setup_benchmark.py rename to plugins/harness2-benchmarks/scripts/setup_benchmark.py index 871d1cb9..81a03ffc 100644 --- a/scripts/setup_benchmark.py +++ b/plugins/harness2-benchmarks/scripts/setup_benchmark.py @@ -13,9 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Compatibility entry point for harness2.cli.setup.""" +"""Compatibility entry point for harness2_benchmarks.setup.""" -from harness2.cli.setup import main +from harness2_benchmarks.setup import main if __name__ == "__main__": raise SystemExit(main()) diff --git a/scripts/split_tasks.py b/plugins/harness2-benchmarks/scripts/split_tasks.py similarity index 98% rename from scripts/split_tasks.py rename to plugins/harness2-benchmarks/scripts/split_tasks.py index 8a778e7c..59c11203 100644 --- a/scripts/split_tasks.py +++ b/plugins/harness2-benchmarks/scripts/split_tasks.py @@ -23,7 +23,7 @@ from collections import defaultdict from pathlib import Path -from harness2.benchmarks.tasks import discover +from harness2_benchmarks.tasks import discover def main(argv=None) -> int: diff --git a/plugins/harness2-benchmarks/tests/test_plugin.py b/plugins/harness2-benchmarks/tests/test_plugin.py new file mode 100644 index 00000000..554d51bc --- /dev/null +++ b/plugins/harness2-benchmarks/tests/test_plugin.py @@ -0,0 +1,425 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline coverage for the benchmark distribution boundary.""" + +from __future__ import annotations + +import importlib +import json +import os +import pkgutil +import shutil +import subprocess +import sys +import tempfile +import unittest +from dataclasses import dataclass +from importlib.metadata import entry_points +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import harness2_benchmarks +from harness2_benchmarks import config, install, jb_opencode_runner, preflight + +SUITES = ("jb", "lab", "wb-office", "wb-web", "wb-code") +PLUGIN = Path(__file__).resolve().parents[1] + + +class PluginTests(unittest.TestCase): + def context(self, root, executor="opencode"): + return SimpleNamespace( + settings={"executor.name": executor}, + project=root, + run_root=root / "runs" / "one", + split_file=root / "split.json", + ) + + def suites(self): + return [ + importlib.import_module("harness2_benchmarks.suites." + name.replace("-", "_")) + for name in SUITES + ] + + def test_import_every_module_and_entry_points(self): + for module in pkgutil.walk_packages( + harness2_benchmarks.__path__, harness2_benchmarks.__name__ + "." + ): + importlib.import_module(module.name) + entries = {e.name: e for e in entry_points(group="harness2.suites")} + for name, suite in zip(SUITES, self.suites()): + self.assertIs(entries[name].load(), suite) + self.assertEqual(install.missing_runners(), "") + + def test_suites_reject_unsupported_executor(self): + for name, suite in zip(SUITES, self.suites()): + for executor in ("claude", "command", "unknown"): + with self.subTest(suite=name, executor=executor): + with self.assertRaisesRegex( + config.ConfigError, + f"the {name} suite runs on the opencode or codex executor", + ): + suite.make_adapter(self.context(Path("/project"), executor)) + + def test_suite_paths_and_substrates(self): + with tempfile.TemporaryDirectory() as tmp: + for name, suite in zip(SUITES, self.suites()): + for executor, substrate in (("opencode", "oc"), ("codex", "cx")): + with self.subTest(suite=name, executor=executor): + ctx = self.context(Path(tmp), executor) + adapter = suite.make_adapter(ctx) + self.assertEqual(adapter.spec.name, name) + self.assertEqual(adapter.spec.runs_root, ctx.run_root) + self.assertEqual(adapter.substrate.name, substrate) + self.assertEqual(adapter.split_file, ctx.split_file) + if name == "jb": + self.assertEqual(adapter.store, ctx.run_root / "dataset") + elif name == "lab": + self.assertEqual( + adapter._results_root, ctx.run_root / "results" / substrate + ) + else: + self.assertEqual( + adapter._label_dir("example"), ctx.run_root / "rollouts/example" + ) + + def test_suite_check_and_setup(self): + for name, suite in zip(SUITES, self.suites()): + ctx = self.context(Path("/project")) + with self.subTest(suite=name), patch.object(suite, "make_adapter") as adapter: + with patch.object(preflight, "check", return_value=["missing"]) as check: + self.assertEqual(suite.check(ctx), ["missing"]) + adapter.assert_called_once_with(ctx) + check.assert_called_once_with(adapter.return_value, judge=True) + bench, _, subset = name.partition("-") + args = [bench] + (["--subset", subset] if subset else []) + ["--dry-run"] + with patch.object(suite, "main", return_value=7) as setup: + self.assertEqual(suite.setup(["--dry-run"]), 7) + setup.assert_called_once_with(args) + + def test_channels_keep_legacy_validation(self): + with patch.object(config, "PROPOSER_MODEL_BY_CHANNEL", {"opencode": ""}): + self.assertEqual(config.resolve_channel_model(None, None).model, "") + self.assertEqual( + config.resolve_channel_model("opencode", "provider/model").model, "provider/model" + ) + self.assertEqual(config.resolve_channel_model("claude_cli", "claude-x").model, "claude-x") + for kind, model in ( + (None, "x"), + ("opencode", None), + ("unknown", "x"), + ("opencode", ""), + ("opencode", "bare"), + ("claude_cli", "provider/model"), + ): + with self.subTest(kind=kind, model=model), self.assertRaises(ValueError): + config.resolve_channel_model(kind, model) + + def test_channels_supply_new_runtime_fields_when_available(self): + @dataclass + class Channel: + kind: str + model: str + effort: str + steps: int + timeout: int + bin: str = "" + provider: object = None + + with ( + patch.object(config, "ChannelSpec", Channel), + patch.object(config, "VERTEX_PROJECT", "test-project"), + patch.object(config, "VERTEX_REGION", "test-region"), + ): + ch = config.resolve_channel_model("opencode", "provider/model", steps=3, timeout=4) + self.assertEqual( + ch.bin, + f"{config.BUN} run --cwd {config.OPENCODE_DIR}/packages/opencode " + "--conditions=browser src/index.ts", + ) + self.assertEqual(ch.provider.project, "test-project") + self.assertEqual(ch.provider.region, "test-region") + self.assertEqual((ch.steps, ch.timeout), (3, 4)) + self.assertEqual(config.resolve_channel_model("claude_cli", "claude-x").bin, "") + with ( + patch.object(config, "ChannelSpec", Channel), + patch.object(config, "VERTEX_PROJECT", ""), + ): + self.assertIsNone(config.resolve_channel_model("opencode", "provider/model").provider) + + def test_dependency_paths_use_cache_or_env(self): + with tempfile.TemporaryDirectory() as tmp: + env = {**os.environ, "XDG_CACHE_HOME": tmp} + env.pop("HARNESS2_BENCH_DEPS", None) + code = "from harness2_benchmarks.config import DEPENDENCIES; print(DEPENDENCIES)" + for override, expected in ( + (None, Path(tmp) / "harness2/benchmarks"), + (str(Path(tmp) / "custom"), Path(tmp) / "custom"), + ): + if override: + env["HARNESS2_BENCH_DEPS"] = override + result = subprocess.run( + [sys.executable, "-B", "-c", code], + env=env, + cwd=tmp, + capture_output=True, + text=True, + check=True, + ) + self.assertEqual(result.stdout.strip(), str(expected.resolve())) + self.assertEqual( + config.THIRD_PARTY, Path(harness2_benchmarks.__file__).parent / "third_party" + ) + self.assertFalse(hasattr(config, "RUNS")) + + def test_preflight_checks_pinned_assets_without_core_opencode_preflight(self): + with tempfile.TemporaryDirectory() as tmp: + deps = Path(tmp) + oc = deps / "opencode" + bun = deps / "bun" + (oc / "node_modules").mkdir(parents=True) + index = oc / "packages/opencode/src/index.ts" + index.parent.mkdir(parents=True) + index.touch() + bun.touch() + adapter = SimpleNamespace( + spec=SimpleNamespace(name="jb"), tasks=Mock(), preflight=Mock() + ) + with ( + patch.multiple( + config, + DEPENDENCIES=deps, + OPENCODE_DIR=oc, + BUN=bun, + SOLVER_MODEL_OC="provider/model", + IMPROVER_MODEL="provider/model", + validate_models=Mock(), + ), + patch( + "harness2_benchmarks.preflight.subprocess.run", + return_value=SimpleNamespace(returncode=0), + ), + patch( + "harness2.substrate.opencode.OpencodeSubstrate.preflight", + side_effect=AssertionError, + ), + ): + self.assertEqual(preflight.check(adapter), []) + index.unlink() + bun.unlink() + errors = preflight.check(adapter) + self.assertTrue(any("OpenCode checkout missing" in e for e in errors)) + self.assertTrue(any("bun not found" in e for e in errors)) + + def test_codex_preflight_uses_plugin_paths_and_validates_endpoint(self): + with tempfile.TemporaryDirectory() as tmp: + binary = Path(tmp) / "codex" + binary.touch() + with ( + patch.multiple( + config, + CODEX_DIST=tmp, + CODEX_BASE_URL="http://localhost:4000/v1", + CODEX_CONTAINER_URL="http://host.docker.internal:4000/v1", + ), + patch.dict( + os.environ, + HARNESS2_CODEX_BASE_URL="http://localhost:4000/v1", + HARNESS2_CODEX_API_KEY="test-key", + ), + patch("harness2_benchmarks.preflight.socket.create_connection") as connect, + ): + preflight.codex(require_bridge=True) + connect.assert_called_once_with(("localhost", 4000), timeout=5) + with patch.object(config, "CODEX_BASE_URL", "file:///tmp/key"): + with self.assertRaises(config.ConfigError): + preflight.codex() + with patch.object(config, "CODEX_CONTAINER_URL", ""): + with self.assertRaisesRegex(RuntimeError, "HARNESS2_CODEX_CONTAINER_URL"): + preflight.codex(require_bridge=True) + binary.unlink() + with self.assertRaisesRegex(RuntimeError, "Codex is not installed"): + preflight.codex() + + def test_jobbench_runner_defaults_use_plugin_dependencies(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + render = root / "render" + (render / ".opencode").mkdir(parents=True) + oc = root / "opencode" + (oc / "packages/opencode").mkdir(parents=True) + with ( + patch.dict( + os.environ, + { + "MODEL_ID": "provider/model", + "LABEL": "test", + "TASK_IDS": "task", + "HARNESS_RENDER_DIR": str(render), + "OPENCODE_DIR": "", + "TASKS_BASE_DIR": "", + }, + ), + patch.object(config, "OPENCODE_DIR", oc), + patch.object(jb_opencode_runner, "JOBBENCH", root / "jb"), + ): + settings = jb_opencode_runner._settings() + self.assertEqual(settings.opencode_dir, oc) + self.assertEqual(settings.tasks_base, root / "jb/dataset/main") + + def test_lab_overlay_paths_resolve_in_downloaded_checkout(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + with patch.object(config, "DEPENDENCIES", root): + install.overlays("harvey-labs") + repo = root / "harvey-labs" + parsers = repo / "sandbox/parsers" + parsers.mkdir(parents=True) + (parsers / "parse_doc.py").touch() + code = ( + "import json, runpy, sys; from pathlib import Path; repo = Path(sys.argv[1]); " + "r = runpy.run_path(str(repo / 'external/adapt_runs_cx/run_codex_lab.py')); " + "p = runpy.run_path(str(repo / 'opencode/scripts/read_doc.py')); " + "print(json.dumps([str(r['REPO']), str(r['C'].BENCH_ROOT), " + "str(r['READ_DOC']), str(p['_find_parsers_dir']())]))" + ) + env = dict(os.environ) + env.pop("LAB_BENCH_ROOT", None) + result = subprocess.run( + [sys.executable, "-B", "-c", code, str(repo)], + capture_output=True, + text=True, + check=True, + env=env, + ) + self.assertEqual( + json.loads(result.stdout), + [str(repo), str(repo), str(repo / "opencode/scripts/read_doc.py"), str(parsers)], + ) + + def test_codex_shell_runner_defaults_use_benchmark_cache(self): + runner = config.THIRD_PARTY / "jb_runner/run_benchmark_codex_cli_adapt.sh" + with tempfile.TemporaryDirectory() as tmp: + env = {**os.environ, "XDG_CACHE_HOME": tmp, "BENCHMARK_MODELS": "test"} + env.pop("TASKS_BASE_DIR", None) + env.pop("HARNESS2_BENCH_DEPS", None) + env.pop("SPLIT", None) + for override, expected in ( + (None, Path(tmp) / "harness2/benchmarks/job-bench-eval/dataset/main"), + (str(Path(tmp) / "custom"), Path(tmp) / "custom/job-bench-eval/dataset/main"), + ): + if override: + env["HARNESS2_BENCH_DEPS"] = override + result = subprocess.run( + [ + "bash", + "-c", + 'source "$1"; printf "%s" "$TASKS_BASE_DIR"', + "bash", + str(runner), + ], + env=env, + capture_output=True, + text=True, + check=True, + ) + self.assertEqual(result.stdout, str(expected)) + + def test_scripts_use_new_cli_and_keep_project_directory(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + scripts = root / "plugins/harness2-benchmarks/scripts" + scripts.mkdir(parents=True) + binary = root / ".venv/bin/harness2" + binary.parent.mkdir(parents=True) + binary.write_text( + f"#!{sys.executable}\nimport json, os, sys\n" + "with open(os.environ['CALLS'], 'a') as f:\n" + " f.write(json.dumps([os.getcwd(), sys.argv[1:]]) + '\\n')\n" + ) + binary.chmod(0o755) + for filename, suite in ( + ("run_jobbench.sh", "jb"), + ("run_lab.sh", "lab"), + ("run_workbuddy.sh", "wb-web"), + ): + script = scripts / filename + shutil.copy2(PLUGIN / "scripts" / filename, script) + for substrate, executor in (("oc", "opencode"), ("cx", "codex")): + for dry in (False, True): + calls = root / "calls.jsonl" + calls.unlink(missing_ok=True) + env = { + **os.environ, + "CALLS": str(calls), + "SUBSTRATE": substrate, + "SUBSET": "web", + "K": "3", + "MODE": "sequential", + } + result = subprocess.run( + ["bash", str(script), *(["--dry-run"] if dry else [])], + cwd=root, + env=env, + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + rows = [json.loads(line) for line in calls.read_text().splitlines()] + self.assertTrue(all(row[0] == str(root.resolve()) for row in rows)) + self.assertEqual( + rows[0][1], + [ + "-c", + f"executor.name={executor}", + "-c", + f"suite.name={suite}", + "doctor", + ], + ) + self.assertEqual( + rows[1][1], + [ + "-c", + f"executor.name={executor}", + "run", + "--suite", + suite, + "--name", + f"{suite}-{executor}-k3-sequential", + "--k", + "3", + "--mode", + "sequential", + ] + + (["--dry-run"] if dry else []), + ) + self.assertEqual(len(rows), 2 if dry else 4) + if not dry: + self.assertEqual( + [r[1] for r in rows[2:]], + [ + ["judge", f"{suite}-{executor}-k3-sequential"], + ["report", f"{suite}-{executor}-k3-sequential"], + ], + ) + result = subprocess.run( + ["bash", str(script)], env={**os.environ, "SUBSTRATE": "bad"} + ) + self.assertEqual(result.returncode, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/pyproject.toml b/pyproject.toml index d5892bea..09bab60e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,14 +20,13 @@ authors = [ dependencies = [] [project.optional-dependencies] -render = ["openpyxl>=3.1,<4", "python-pptx>=1,<2"] bridge = ["litellm[proxy]==1.95.0", "fastapi==0.139.2", "google-auth==2.56.3"] [project.scripts] harness2 = "harness2.cli.run_sweep:main" harness2-judge = "harness2.cli.judge:main" harness2-report = "harness2.cli.report:main" -harness2-setup = "harness2.cli.setup:main" +harness2-setup = "harness2_benchmarks.setup:main" harness2-check = "harness2.cli.check:main" harness2-demo = "harness2.cli.demo:main" harness2-bridge = "harness2.cli.bridge:main" From 84e125458c30c93f4e69becb369e5176a6c4bce7 Mon Sep 17 00:00:00 2001 From: Bae Seongmin <83870598+3ae3ae@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:40:08 +0900 Subject: [PATCH 04/21] feat: provider-aware improver channels, run events, spending limits (#4) Improver, selector and reflector sessions now run on the claude, codex or opencode CLI found on PATH (or a configured binary), each with the environment its provider profile produces and an isolated config dir, so neither Vertex AI nor the user's global agent setup is assumed. A "human" channel lets the CLI answer selections in the terminal, and the reflector can run on its own channel. core/events.py appends one JSON line per rollout, session, selection and task to the run's events.jsonl and enforces --budget-usd and --max-calls across resumes: work stops before the next call once a limit is reached, and Ctrl-C stops the same way. core.configure() sets phase width, transport attempts and session concurrency at run time. Persisted run-directory paths are stored relative to the run root and the session cache keys on paths relative to the audit dir, so a run directory can be moved and still resumes without repeating work. The benchmarks plugin passes its pinned OpenCode command and Vertex provider through the new ChannelSpec fields directly. Co-authored-by: Claude Opus 5.5 --- harness2.egg-info/PKG-INFO | 227 +++++++ harness2.egg-info/SOURCES.txt | 54 ++ harness2.egg-info/dependency_links.txt | 1 + harness2.egg-info/entry_points.txt | 8 + harness2.egg-info/requires.txt | 5 + harness2.egg-info/top_level.txt | 1 + harness2/core/__init__.py | 25 + harness2/core/events.py | 112 ++++ harness2/core/judge.py | 3 +- harness2/core/proposer.py | 183 ++++-- harness2/core/recursion.py | 93 ++- harness2/core/runmark.py | 15 + harness2/core/schedule.py | 98 ++- harness2/core/selection.py | 14 +- harness2/core/spec.py | 47 +- harness2/demo.py | 26 +- harness2/substrate/codex.py | 7 +- .../harness2_benchmarks.egg-info/PKG-INFO | 11 + .../harness2_benchmarks.egg-info/SOURCES.txt | 77 +++ .../dependency_links.txt | 1 + .../entry_points.txt | 6 + .../harness2_benchmarks.egg-info/requires.txt | 5 + .../top_level.txt | 1 + .../harness2_benchmarks/config.py | 7 +- .../harness2-benchmarks/tests/test_plugin.py | 19 +- tests/test_runtime.py | 620 ++++++++++++++++++ tests/test_workflow.py | 24 +- 27 files changed, 1489 insertions(+), 201 deletions(-) create mode 100644 harness2.egg-info/PKG-INFO create mode 100644 harness2.egg-info/SOURCES.txt create mode 100644 harness2.egg-info/dependency_links.txt create mode 100644 harness2.egg-info/entry_points.txt create mode 100644 harness2.egg-info/requires.txt create mode 100644 harness2.egg-info/top_level.txt create mode 100644 harness2/core/events.py create mode 100644 plugins/harness2-benchmarks/harness2_benchmarks.egg-info/PKG-INFO create mode 100644 plugins/harness2-benchmarks/harness2_benchmarks.egg-info/SOURCES.txt create mode 100644 plugins/harness2-benchmarks/harness2_benchmarks.egg-info/dependency_links.txt create mode 100644 plugins/harness2-benchmarks/harness2_benchmarks.egg-info/entry_points.txt create mode 100644 plugins/harness2-benchmarks/harness2_benchmarks.egg-info/requires.txt create mode 100644 plugins/harness2-benchmarks/harness2_benchmarks.egg-info/top_level.txt create mode 100644 tests/test_runtime.py diff --git a/harness2.egg-info/PKG-INFO b/harness2.egg-info/PKG-INFO new file mode 100644 index 00000000..11273dff --- /dev/null +++ b/harness2.egg-info/PKG-INFO @@ -0,0 +1,227 @@ +Metadata-Version: 2.4 +Name: harness2 +Version: 0.1.0 +Summary: Harness²: Recursive Agent Harnessing for an Open World +Author: Google LLC +License-Expression: Apache-2.0 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Operating System :: POSIX +Requires-Python: <3.14,>=3.12 +Description-Content-Type: text/markdown +License-File: LICENSE +Provides-Extra: bridge +Requires-Dist: litellm[proxy]==1.95.0; extra == "bridge" +Requires-Dist: fastapi==0.139.2; extra == "bridge" +Requires-Dist: google-auth==2.56.3; extra == "bridge" +Dynamic: license-file + +

+ Harness²
+ Recursive Agent Harnessing for an Open World +

+ +

+ Paper + Website + Benchmark guides + License +

+ +

+ + Install, set up, and run JobBench: uv pip install -e '.[render]', harness2-setup jb, then harness2 --bench jb --k 1 --mode parallel. Click for prerequisites and commands. + +

+ +A **harness** is the system around a model that enables it to act as an agent: +prompts, tools, memory, skills, orchestration, and more. **Harnessing** is the +process of optimizing that system. + +Harness² is a plug-and-play framework for **recursive agent harnessing** in an +open world, where agents encounter tasks unseen during harness design. It +connects two levels of recursion while keeping model weights fixed: + +* **Task-level recursion** refines the execution harness for the current task + through a contrastive **Propose–Probe–Compose** step: propose edits, probe + their effects on the same task, and compose a refined harness from behavioral + contrasts between executions, without ground-truth feedback. +* **Harnessing-level recursion** updates a persistent *improvement harness* + with adopted harnesses and evidence-supported lessons from each task to guide + future refinement. + +

+ + Harness² pipeline. Task-level recursion: an agent executor runs the initial harness, a harness editor proposes invariant and domain edits, candidate probes execute them, and a harness composer combines them into the final harness. Harnessing-level recursion: an experience reflector reflects on the step and updates the improvement harness used for the next task. + +

+ +Across three professional benchmarks and four model–harness configurations, a +single Harness² step improves GLM-5 and Gemini 3.5 Flash over their base +harnesses, with gains of up to 11.5 points on WorkBuddy-Bench, 4.5 points on +JobBench, and 11.8 percentage points on LAB's partial pass rate. Increasing the +task-level recursion budget brings further gains. + +The release supports **parallel and sequential recursion**, **OpenCode and +Codex executors**, and **JobBench, LAB, and WorkBuddy-Bench**. + +## Video overview + +https://github.com/user-attachments/assets/76b10548-b14b-4c8f-9aa5-a93662df4d56 + +## Contents + +- [Video overview](#video-overview) +- [Harness components](#harness-components) +- [Install](#install) +- [Run a benchmark](#run-a-benchmark) +- [Experiments](#experiments) +- [Web access](#web-access) +- [Results and resume](#results-and-resume) +- [Citation](#citation) +- [Contributing](#contributing) +- [License](#license) +- [Disclaimer](#disclaimer) + +## Harness components + +Harness² exposes eight editable components around the model: **system prompt, +guardrails, memory, tool guidance, subagents, skills, scripts, and plugins**. +They control context construction, the tool interface, delegation, and behavior +inside the tool loop. + +

+ + Eight harness components grouped by runtime role: system prompt, guardrails, memory, and skills construct context; tool guidance and scripts define the tool interface; subagents orchestrate delegated work; plugins control middleware. + +

+ +## Install + +Use **Linux, Python 3.12, and [uv](https://docs.astral.sh/uv/getting-started/installation/)**. +From the repository root: + +```bash +uv venv --python 3.12 .venv +source .venv/bin/activate +uv pip install -e '.[render]' +harness2-demo --output runs/demo --k 2 +``` + +The local demo runs both recursion modes with deterministic editors and real +Python executions. It requires no model credentials. + +For model-backed benchmarks, install **Node.js 20+, the Google Cloud CLI**, and +the [benchmark dependencies](docs/SETUP.md#requirements). LAB needs Pandoc and +Poppler; WorkBuddy-Bench needs Docker. Then configure your models: + +```bash +cp .env.example .env +# Edit .env with your Vertex AI project and model IDs. +gcloud auth application-default login +``` + +See the [setup guide](docs/SETUP.md) for model settings, authentication +alternatives, and the [Codex executor](docs/SETUP.md#codex-executor). + +## Run a benchmark + +The following example installs LAB and runs one task, then evaluates and reports +its result. Model-backed runs incur API costs; start with a small task selection. + +```bash +harness2-setup lab +harness2-check --bench lab +harness2 --bench lab --k 1 --mode parallel \ + --only-domains antitrust-competition --max-tasks 1 +harness2-judge --bench lab --k 1 --mode parallel +harness2-report --bench lab --k 1 --mode parallel +``` + +Sequential recursion runs the same commands with `--mode sequential`. `--k` is +the task-level recursion budget; see [Experiments](#experiments). Judging and +reporting must use the same benchmark, executor, subset, `--k`, `--mode`, and +seed as the refinement run. Add `--dry-run` to `harness2` to preview task +selection without model calls. + +| Benchmark | Tasks | Install | Guide | +|---|---|---|---| +| JobBench | Professional work | `harness2-setup jb` | [JobBench](docs/BENCHMARKS.md#jobbench) | +| LAB | Legal work | `harness2-setup lab` | [LAB](docs/BENCHMARKS.md#lab) | +| WorkBuddy-Bench | Office, web, and code | `harness2-setup wb` | [WorkBuddy](docs/BENCHMARKS.md#workbuddy-bench) | + +Choose one benchmark per command. WorkBuddy also takes `--subset office`, +`--subset web`, or `--subset code` on every command after setup. The +[benchmark guides](docs/BENCHMARKS.md) cover each one's data, domains, and +grading. + +## Experiments + +Scale task-level recursion in two ways: **parallel recursion** explores multiple +candidate harnesses and composes refinements from their execution evidence; +**sequential recursion** carries the refined harness and accumulated evidence +through successive Propose–Probe–Compose steps. + +

+ + Two ways to scale task-level recursion. Parallel: probe k general and k domain-specific candidate harnesses, then produce k compositions from the shared evidence. Sequential: carry the refined harness and accumulated evidence through k Propose–Probe–Compose steps. + +

+

Parallel recursion explores wider; sequential recursion refines deeper.

+ +| Option | Behavior | +|---|---| +| `--k 1` | One Propose–Probe–Compose step | +| `--k K --mode parallel` | Probe `K` general and `K` domain-specific candidates, then produce `K` compositions | +| `--k K --mode sequential` | Refine the harness through `K` successive steps | +| `--substrate cx` | Use Codex instead of the default OpenCode executor | + +After each task, harnessing-level recursion retains the adopted harness and +evidence-supported lessons to guide future refinement. See the +[experiment guide](docs/EXPERIMENTS.md) for task selection, concurrency, +reproducibility, and baseline comparisons. + +## Web access + +Executor network access is unrestricted, and Codex runs with its sandbox +disabled. Public benchmark rubrics may be accessible online. Before evaluation, +configure [network restrictions](docs/EXPERIMENTS.md#web-access) to prevent +access to benchmark answers and grading criteria. + +## Results and resume + +Results are saved under `runs/`, organized by benchmark and subset where +applicable. They include task summaries, deliverables, adopted harnesses, +reflection notes, and evaluation results. + +Repeat the same command to resume. Use a new `HARNESS2_RUNS_DIR` when changing +models or task selection. See [results and resume](docs/EXPERIMENTS.md#results-and-resume) +for details; each command also supports `--help`. + +## Citation + +```bibtex +@article{xu2026harness2, + title = {Harness$^2$: Recursive Agent Harnessing for an Open World}, + author = {Xu, Ruiyao and Chen, Yanfei and CuiZhu, Zhongying and Dalvi Mishra, Bhavana and Ming, Yifei and Yu, Han and Han, Rujun and Lee, Chen-Yu and Pfister, Tomas}, + journal = {arXiv preprint arXiv:2610.14224}, + year = {2026}, + url = {https://arxiv.org/abs/2610.14224} +} +``` + +## Contributing + +See [CONTRIBUTING.md](CONTRIBUTING.md). + +## License + +Harness² is licensed under [Apache 2.0](LICENSE). Third-party code under +[`third_party/`](third_party/README.md), benchmarks, and datasets retain their +upstream licenses; see +[third-party notices](dependencies/README.md). + +## Disclaimer + +This is not an officially supported Google product. This project is not eligible +for the [Google Open Source Software Vulnerability Rewards Program](https://bughunters.google.com/open-source-security). diff --git a/harness2.egg-info/SOURCES.txt b/harness2.egg-info/SOURCES.txt new file mode 100644 index 00000000..868f2df9 --- /dev/null +++ b/harness2.egg-info/SOURCES.txt @@ -0,0 +1,54 @@ +LICENSE +MANIFEST.in +README.md +pyproject.toml +assets/Harness2-wordmark.png +assets/components.png +assets/harness2_pipeline.png +assets/harness2_recursion_modes.png +assets/logo.png +harness2/__init__.py +harness2/config.py +harness2/demo.py +harness2/providers.py +harness2/registry.py +harness2/settings.py +harness2.egg-info/PKG-INFO +harness2.egg-info/SOURCES.txt +harness2.egg-info/dependency_links.txt +harness2.egg-info/entry_points.txt +harness2.egg-info/requires.txt +harness2.egg-info/top_level.txt +harness2/cli/__init__.py +harness2/cli/_common.py +harness2/cli/bridge.py +harness2/cli/check.py +harness2/cli/demo.py +harness2/cli/judge.py +harness2/cli/report.py +harness2/cli/run_sweep.py +harness2/core/__init__.py +harness2/core/context.py +harness2/core/events.py +harness2/core/judge.py +harness2/core/ledger.py +harness2/core/library.py +harness2/core/prompts.py +harness2/core/proposer.py +harness2/core/recursion.py +harness2/core/report.py +harness2/core/runmark.py +harness2/core/schedule.py +harness2/core/selection.py +harness2/core/spec.py +harness2/core/tree.py +harness2/substrate/__init__.py +harness2/substrate/base.py +harness2/substrate/codex.py +harness2/substrate/opencode.py +scripts/render_readme_typing.py +tests/test_providers.py +tests/test_registry.py +tests/test_runtime.py +tests/test_settings.py +tests/test_workflow.py \ No newline at end of file diff --git a/harness2.egg-info/dependency_links.txt b/harness2.egg-info/dependency_links.txt new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/harness2.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/harness2.egg-info/entry_points.txt b/harness2.egg-info/entry_points.txt new file mode 100644 index 00000000..7e26df27 --- /dev/null +++ b/harness2.egg-info/entry_points.txt @@ -0,0 +1,8 @@ +[console_scripts] +harness2 = harness2.cli.run_sweep:main +harness2-bridge = harness2.cli.bridge:main +harness2-check = harness2.cli.check:main +harness2-demo = harness2.cli.demo:main +harness2-judge = harness2.cli.judge:main +harness2-report = harness2.cli.report:main +harness2-setup = harness2_benchmarks.setup:main diff --git a/harness2.egg-info/requires.txt b/harness2.egg-info/requires.txt new file mode 100644 index 00000000..293682b6 --- /dev/null +++ b/harness2.egg-info/requires.txt @@ -0,0 +1,5 @@ + +[bridge] +litellm[proxy]==1.95.0 +fastapi==0.139.2 +google-auth==2.56.3 diff --git a/harness2.egg-info/top_level.txt b/harness2.egg-info/top_level.txt new file mode 100644 index 00000000..12ad3db7 --- /dev/null +++ b/harness2.egg-info/top_level.txt @@ -0,0 +1 @@ +harness2 diff --git a/harness2/core/__init__.py b/harness2/core/__init__.py index d59d80d5..2138c7af 100644 --- a/harness2/core/__init__.py +++ b/harness2/core/__init__.py @@ -17,3 +17,28 @@ Per-task harness adaptation over a c1..c8 component tree, two recursion modes at an identical budget, blind answer-free selection, and a per-domain experience library. """ + +from __future__ import annotations + +import threading + + +def configure(*, phase_width=None, transport_attempts=None, proposer_concurrency=None) -> None: + """Apply runtime scheduling settings before starting work.""" + from . import proposer, recursion, schedule + + values = { + "phase_width": phase_width, + "transport_attempts": transport_attempts, + "proposer_concurrency": proposer_concurrency, + } + for name, value in values.items(): + if value is not None and (type(value) is not int or value < 1): + raise ValueError(f"{name} must be a positive integer") + if phase_width is not None: + recursion.PHASE_WIDTH = schedule.ROLLOUT_PARALLEL = phase_width + if transport_attempts is not None: + recursion.TRANSPORT_ATTEMPTS = transport_attempts + if proposer_concurrency is not None: + proposer.MAX_CONCURRENCY = proposer_concurrency + proposer._SEM = threading.BoundedSemaphore(proposer_concurrency) diff --git a/harness2/core/events.py b/harness2/core/events.py new file mode 100644 index 00000000..0188925f --- /dev/null +++ b/harness2/core/events.py @@ -0,0 +1,112 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Thread-safe runtime events and resumable spending limits.""" + +from __future__ import annotations + +import json +import threading +from datetime import datetime, timezone +from pathlib import Path + + +class Stopped(RuntimeError): + """Execution stopped at a user request or a spending limit.""" + + +_lock = threading.Lock() +_active = False +_path: Path | None = None +_spent_usd = 0.0 +_calls = 0 +_budget_usd = 0.0 +_max_calls = 0 +_stopped: str | None = None + + +def _accumulate(record: dict) -> None: + global _spent_usd, _calls + _spent_usd += record.get("cost_usd", 0) or 0 + if record["kind"] == "rollout" or (record["kind"] == "session" and not record.get("cached")): + _calls += 1 + + +def start(path: Path | None, *, budget_usd: float = 0.0, max_calls: int = 0) -> None: + """Reset limits and restore accounting from a previous event stream.""" + if budget_usd < 0 or max_calls < 0: + raise ValueError("budget_usd and max_calls must be non-negative") + global _active, _path, _spent_usd, _calls, _budget_usd, _max_calls, _stopped + with _lock: + _active = True + _path = Path(path) if path is not None else None + _spent_usd, _calls = 0.0, 0 + _budget_usd, _max_calls, _stopped = budget_usd, max_calls, None + if _path is not None: + _path.parent.mkdir(parents=True, exist_ok=True) + if _path.is_file(): + for line in _path.read_text(encoding="utf-8").splitlines(): + if line.strip(): + _accumulate(json.loads(line)) + + +def emit(kind: str, **fields) -> None: + """Append one event and count completed work, under the same lock.""" + with _lock: + if not _active: + return + record = { + "t": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "kind": kind, + **fields, + } + if _path is not None: + with _path.open("a", encoding="utf-8") as stream: + stream.write(json.dumps(record) + "\n") + _accumulate(record) + + +def _reason() -> str | None: + if _stopped: + return _stopped + if _active and _budget_usd and _spent_usd >= _budget_usd: + return "budget_usd" + if _active and _max_calls and _calls >= _max_calls: + return "max_calls" + return None + + +def check() -> None: + """Raise before starting more work when a limit or stop request applies.""" + with _lock: + reason = _reason() + if reason is not None: + raise Stopped(reason) + + +def request_stop(reason: str) -> None: + global _stopped + with _lock: + _stopped = reason + + +def totals() -> dict: + with _lock: + return { + "spent_usd": round(_spent_usd, 6), + "calls": _calls, + "budget_usd": _budget_usd, + "max_calls": _max_calls, + "stopped": _reason(), + } diff --git a/harness2/core/judge.py b/harness2/core/judge.py index cf4e7b9f..d28cc857 100644 --- a/harness2/core/judge.py +++ b/harness2/core/judge.py @@ -26,6 +26,7 @@ from pathlib import Path from typing import Sequence +from . import runmark from .spec import BenchmarkAdapter @@ -120,7 +121,7 @@ def judge_selections( sel["k"], sel["chosen_label"], sel.get("chosen_run_id", ""), - Path(rd), + runmark.resolve_path(rd, adapter.spec.runs_root), chosen_is_base=bool(sel.get("chosen_is_base")), ) ] = sel diff --git a/harness2/core/proposer.py b/harness2/core/proposer.py index 4a235075..a0aae876 100644 --- a/harness2/core/proposer.py +++ b/harness2/core/proposer.py @@ -25,6 +25,7 @@ import json import os import re +import shlex import shutil import subprocess import tempfile @@ -33,23 +34,16 @@ from dataclasses import dataclass, field from pathlib import Path -from .. import config +from .. import providers +from ..substrate.codex import parse_items_text +from . import events from .spec import ChannelSpec SYSTEM_ARG_MAX = 120_000 -MAX_CONCURRENCY = int(config.PROPOSER_CONCURRENCY) -_SEM = threading.BoundedSemaphore(MAX_CONCURRENCY) if MAX_CONCURRENCY > 0 else None - - -_DROP_ENV = ( - "ANTHROPIC_API_KEY", - "ANTHROPIC_AUTH_TOKEN", - "ANTHROPIC_BASE_URL", - "ANTHROPIC_BEDROCK_BASE_URL", - "CLAUDE_CODE_USE_BEDROCK", -) +MAX_CONCURRENCY = 2 +_SEM = threading.BoundedSemaphore(MAX_CONCURRENCY) def _obj(d: dict, key: str) -> dict: @@ -94,35 +88,39 @@ class Proposer: spec: ChannelSpec audit_dir: Path | None = None attempts: int = 2 - project: str = field(default_factory=lambda: config.VERTEX_PROJECT) - region: str = field(default_factory=lambda: config.VERTEX_REGION) _calls: list[Reply] = field(default_factory=list) def __post_init__(self) -> None: - if not self.spec.model.strip(): - raise ValueError("Set HARNESS2_IMPROVER_MODEL before starting model sessions") + if self.spec.kind != "human" and not self.spec.model.strip(): + raise ValueError("model must not be empty for model sessions") def env(self) -> dict[str, str]: - e = {k: v for k, v in os.environ.items() if k not in _DROP_ENV} - if self.spec.kind == "claude_cli": - e.update( - { - "CLAUDE_CODE_USE_VERTEX": "1", - "ANTHROPIC_VERTEX_PROJECT_ID": self.project, - "CLOUD_ML_REGION": self.region, - "ANTHROPIC_MODEL": self.spec.model, - } - ) - else: - e.update( - { - "GOOGLE_VERTEX_PROJECT": self.project, - "GOOGLE_VERTEX_LOCATION": self.region, - "VERTEXAI_LOCATION": self.region, - } - ) + e = {k: v for k, v in os.environ.items() if k not in providers.STRIP_ENV} + if self.spec.provider is not None: + if self.spec.kind == "claude": + e.update(providers.claude_env(self.spec.provider)) + elif self.spec.kind == "opencode": + e.update(providers.opencode_config(self.spec.provider, self._model())[1]) return e + def _model(self) -> str: + return self.spec.model.split("/", 1)[-1] + + def _bin(self) -> list[str]: + return shlex.split(self.spec.bin) if self.spec.bin else [self.spec.kind] + + def _emit(self, reply: Reply) -> None: + events.emit( + "session", + label=reply.label, + channel=self.spec.kind, + model=self.spec.model, + ok=reply.ok, + cached=reply.cached, + cost_usd=reply.cost_usd, + wall_s=reply.wall_s, + ) + def call( self, mission: str, @@ -140,10 +138,13 @@ def call( directory is destroyed -- e.g. "context/harness", which is how an edited tree comes back. """ - cache_key = _call_fingerprint(label, mission, system, context_dir) + if self.spec.kind == "human": + raise ProposerError("the human channel needs an interactive selector") + cache_key = _call_fingerprint(label, mission, system, context_dir, self.audit_dir) cached = self._cached(label, cache_key, require_workspace=bool(harvest)) if cached is not None: self._calls.append(cached) + self._emit(cached) return cached wall = int(timeout or self.spec.timeout) @@ -154,14 +155,15 @@ def call( for attempt in range(1, tries + 1): if attempt > 1: time.sleep(10 * (attempt - 1)) - tmp = Path(tempfile.mkdtemp(prefix="harness2-prop-")) + tmp = Path(tempfile.mkdtemp(prefix="harness2-prop-")).resolve() try: if context_dir and Path(context_dir).is_dir(): shutil.copytree(context_dir, tmp / "context", dirs_exist_ok=True) with _SEM if _SEM is not None else contextlib.nullcontext(): - text, cost, err = ( - self._claude_cli if self.spec.kind == "claude_cli" else self._opencode - )(tmp, mission, system, wall) + events.check() + text, cost, err = getattr(self, f"_{self.spec.kind}")( + tmp, mission, system, wall + ) spent_usd += cost if not text.strip(): last = err or "empty reply" @@ -181,7 +183,10 @@ def call( _drop_workspace(self._paths(label, cache_key)) self._write_audit(r, mission, system, cache_key) self._calls.append(r) + self._emit(r) return r + except events.Stopped: + raise except subprocess.TimeoutExpired: last = f"timeout after {wall}s" except Exception as exc: @@ -201,24 +206,28 @@ def call( ) self._calls.append(r) self._write_audit(r, mission, system, cache_key) + self._emit(r) raise ProposerError( f"[{label}] {self.spec.kind}/{self.spec.model}: {last}", attempts=tries, cost_usd=spent_usd, ) - def _claude_cli( - self, cwd: Path, mission: str, system: str, wall: int - ) -> tuple[str, float, str]: - cli = shutil.which("claude") or "claude" + def _claude(self, cwd: Path, mission: str, system: str, wall: int) -> tuple[str, float, str]: + home = cwd / ".claude-home" + home.mkdir() cmd = [ - cli, + *self._bin(), "--dangerously-skip-permissions", "-p", "--output-format", "json", "--effort", self.spec.effort, + "--model", + self._model(), + "--setting-sources", + "project,local", ] stdin = mission if system: @@ -234,7 +243,7 @@ def _claude_cli( cmd, input=stdin, cwd=cwd, - env=self.env(), + env=self.env() | {"CLAUDE_CONFIG_DIR": str(home)}, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -250,6 +259,49 @@ def _claude_cli( err = f"rc={p.returncode}: {(p.stderr or '')[-300:]}" return text, float(payload.get("total_cost_usd") or 0.0), err + def _codex(self, cwd: Path, mission: str, system: str, wall: int) -> tuple[str, float, str]: + home = cwd / ".codex-home" + home.mkdir() + model = self._model() + config = ( + providers.codex_config(self.spec.provider, model, self.spec.effort) + if self.spec.provider is not None + else f"model = {json.dumps(model)}\n" + f"model_reasoning_effort = {json.dumps(self.spec.effort)}\n" + ) + (home / "config.toml").write_text(config, encoding="utf-8") + p = subprocess.run( + [ + *self._bin(), + "exec", + "--json", + "--skip-git-repo-check", + "--dangerously-bypass-approvals-and-sandbox", + "--ephemeral", + "-C", + str(cwd), + "-", + ], + input=f"# SYSTEM INSTRUCTIONS\n\n{system}\n\n# END\n\n{mission}" if system else mission, + cwd=cwd, + env=self.env() | {"CODEX_HOME": str(home)}, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=wall, + ) + text, errors = "", [] + for event in parse_items_text(p.stdout): + if event.get("type") in ("turn.failed", "error"): + error = event.get("error") or event.get("message") or event["type"] + errors.append(str(error)) + item = _obj(event, "item") + if item.get("type") == "agent_message": + text = str(item.get("text") or "") + if errors or p.returncode: + return "", 0.0, "; ".join(errors)[-300:] or f"rc={p.returncode}: {p.stderr[-300:]}" + return text, 0.0, "" + def _opencode(self, cwd: Path, mission: str, system: str, wall: int) -> tuple[str, float, str]: (cwd / "MISSION.md").write_text( (f"{system}\n\n---\n\n" if system else "") + mission, encoding="utf-8" @@ -257,14 +309,14 @@ def _opencode(self, cwd: Path, mission: str, system: str, wall: int) -> tuple[st out = cwd / "output" out.mkdir(exist_ok=True) - (cwd / "opencode.json").write_text( - json.dumps({"agent": {"build": {"steps": self.spec.steps}}}), encoding="utf-8" - ) + config = {"agent": {"build": {"steps": self.spec.steps}}} + if self.spec.provider is not None: + config.update(providers.opencode_config(self.spec.provider, self._model())[0]) + (cwd / "opencode.json").write_text(json.dumps(config), encoding="utf-8") prompt = ( "Read MISSION.md first. Do what it asks. Reply with the requested output as " "your final message AND save it verbatim to output/answer.md." ) - oc = _opencode_dir() env = self.env() env["OPENCODE_PERMISSION"] = json.dumps( @@ -274,12 +326,7 @@ def _opencode(self, cwd: Path, mission: str, system: str, wall: int) -> tuple[st env["XDG_STATE_HOME"] = str(cwd / ".ocstate") cmd = [ - str(config.BUN), - "run", - "--cwd", - str(oc / "packages" / "opencode"), - "--conditions=browser", - "src/index.ts", + *self._bin(), "run", prompt, "--dir", @@ -393,17 +440,6 @@ def _drop_workspace(paths: dict[str, Path] | None) -> None: shutil.rmtree(paths["workspace"], ignore_errors=True) -def _opencode_dir() -> Path: - """The upstream OpenCode checkout to run through bun: config.OPENCODE_DIR.""" - oc = config.OPENCODE_DIR - if not (oc / "packages" / "opencode" / "src" / "index.ts").is_file(): - raise FileNotFoundError( - f"no OpenCode checkout at {oc} (config.OPENCODE_DIR); run " - f"scripts/setup_benchmark.py to clone the pinned commit there" - ) - return oc - - def _parse_json(stdout: str) -> dict: """The claude CLI's one JSON object, tolerating banners and trailing noise around it. @@ -504,7 +540,13 @@ def _opencode_cost(stdout: str) -> float: return round(total, 6) -def _call_fingerprint(label: str, mission: str, system: str, context_dir: Path | str | None) -> str: +def _call_fingerprint( + label: str, + mission: str, + system: str, + context_dir: Path | str | None, + anchor: Path | None = None, +) -> str: """Content-address one call, including its evidence and its mode/task workspace path. The resolved path is part of the key: the two modes can have byte-identical k=1 evidence @@ -516,7 +558,12 @@ def _call_fingerprint(label: str, mission: str, system: str, context_dir: Path | h.update(b"\0") if context_dir: root = Path(context_dir) - h.update(str(root.resolve()).encode("utf-8", errors="replace")) + location = ( + os.path.relpath(root.resolve(), Path(anchor).resolve()) + if anchor + else str(root.resolve()) + ) + h.update(location.encode("utf-8", errors="replace")) h.update(b"\0") if root.is_dir(): for p in sorted(root.rglob("*")): diff --git a/harness2/core/recursion.py b/harness2/core/recursion.py index dda6b98c..99154ce1 100644 --- a/harness2/core/recursion.py +++ b/harness2/core/recursion.py @@ -21,21 +21,21 @@ from __future__ import annotations import re +import time from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from pathlib import Path from typing import Callable, Sequence -from .. import config -from . import context, prompts +from . import context, events, prompts, runmark from . import tree as harness_tree from .ledger import Ledger from .proposer import Proposer, ProposerError from .spec import BenchmarkAdapter, RunView, resolve_prompts STAGE_ATTEMPTS = 2 -TRANSPORT_ATTEMPTS = config.TRANSPORT_ATTEMPTS -PHASE_WIDTH = config.PHASE_WIDTH +TRANSPORT_ATTEMPTS = 2 +PHASE_WIDTH = 2 @dataclass @@ -149,6 +149,7 @@ def _run_stage( decisions: dict[str, str] | None = None, ) -> Stage: """Build evidence, propose an edit, and retry an invalid proposal once.""" + events.check() surface = adapter.substrate.surface() try: @@ -163,6 +164,8 @@ def _run_stage( decisions=decisions, ) mission = context.mission(tree, job=job) + except events.Stopped: + raise except Exception as exc: return Stage( label=label, @@ -257,9 +260,53 @@ def _record_sessions(ledger: Ledger, stages: Sequence[Stage]) -> None: ledger.session(stage.label, stage.usd, cached=stage.cached, attempts=stage.attempts) +def _cached_rollout(adapter, work_dir, label, run_id, harness_dir) -> RunView | None: + """Reuse completed rollouts without executing or charging for them again.""" + cache = work_dir / "rollouts" / label + if not runmark.is_complete(cache, harness_dir=harness_dir): + return None + record = runmark.read_mark(cache) + if record.get("run_id") != run_id: + return None + data = dict(record["rollout"]) + data["run_dir"] = runmark.resolve_path(data["run_dir"], adapter.spec.runs_root) + if not data["run_dir"].is_dir(): + return None + if data["delivered"] and not adapter.delivered(data["run_dir"]): + return None + intact = getattr(adapter, "artifacts_intact", None) + if callable(intact) and not intact(data["run_dir"]): + return None + return RunView(**data) + + +def _finish_rollout(adapter, work_dir, task_id, rv, harness_dir, started) -> None: + rv.wall_s = round(time.monotonic() - started, 3) + cost = getattr(adapter, "rollout_cost", None) + rv.cost_usd = float(cost(rv.run_dir)) if callable(cost) and rv.run_dir.is_dir() else 0.0 + if rv.ok: + data = vars(rv) | {"run_dir": runmark.relative_path(rv.run_dir, adapter.spec.runs_root)} + runmark.mark_complete( + work_dir / "rollouts" / rv.label, + run_id=rv.run_id, + harness_dir=harness_dir, + extra={"rollout": data}, + ) + events.emit( + "rollout", + task=task_id, + label=rv.label, + run_id=rv.run_id, + ok=rv.ok, + delivered=rv.delivered, + cost_usd=rv.cost_usd, + wall_s=rv.wall_s, + ) + + def _roll( adapter: BenchmarkAdapter, - ledger: Ledger, + ledger: Ledger | None, task_id: str, label: str, tree: harness_tree.Tree | None, @@ -275,9 +322,16 @@ def _roll( adapter.substrate.surface(), tree, work_dir / "harnesses" / label ) run_id = run_id_of(label) + cached = _cached_rollout(adapter, work_dir, label, run_id, hdir) + if cached is not None: + if record and ledger is not None: + ledger.rollout(label, cached=True) + return cached + events.check() + started = time.monotonic() try: - rd = adapter.execute(task_id, hdir, run_id) - exists = Path(rd).exists() + rd = Path(adapter.execute(task_id, hdir, run_id)) + exists = rd.exists() rv = RunView( label=label, run_id=run_id, @@ -286,6 +340,8 @@ def _roll( delivered=exists and adapter.delivered(rd), error="" if exists else "adapter returned a missing run directory", ) + except events.Stopped: + raise except Exception as exc: rv = RunView( label=label, @@ -295,8 +351,9 @@ def _roll( delivered=False, error=f"{type(exc).__name__}: {exc}", ) - if record: - ledger.rollout(label) + _finish_rollout(adapter, work_dir, task_id, rv, hdir, started) + if record and ledger is not None: + ledger.rollout(label, rv.cost_usd) return rv @@ -335,7 +392,7 @@ def _roll_many( ) ) for (label, _tree), rv in zip(items, got): - ledger.rollout(label) + ledger.rollout(label, rv.cost_usd) out[label] = rv return out @@ -351,14 +408,16 @@ def run_parallel( run_id_of: Callable[[str], str], base_tree: harness_tree.Tree, library_dir: Path | None = None, - propose_workers: int = PHASE_WIDTH, - rollout_workers: int = PHASE_WIDTH, + propose_workers: int | None = None, + rollout_workers: int | None = None, ) -> ModeResult: """WIDTH: 2k iid candidates from shared evidence, then k INDEPENDENT compositions.""" + propose_workers = PHASE_WIDTH if propose_workers is None else propose_workers + rollout_workers = PHASE_WIDTH if rollout_workers is None else rollout_workers surface = adapter.substrate.surface() ledger = Ledger(mode="agg", task_id=task_id, k=k) for r in base: - ledger.rollout(r.label) + ledger.rollout(r.label, r.cost_usd) result = ModeResult(mode="agg", task_id=task_id, k=k, ledger=ledger) result.rollouts.update({r.label: r for r in base}) ctx_dir = work_dir / "context" @@ -460,14 +519,16 @@ def run_sequential( run_id_of: Callable[[str], str], base_tree: harness_tree.Tree, library_dir: Path | None = None, - propose_workers: int = 2, - rollout_workers: int = PHASE_WIDTH, + propose_workers: int | None = None, + rollout_workers: int | None = None, ) -> ModeResult: """DEPTH: k nested passes over the same shared base evidence.""" + propose_workers = PHASE_WIDTH if propose_workers is None else propose_workers + rollout_workers = PHASE_WIDTH if rollout_workers is None else rollout_workers surface = adapter.substrate.surface() ledger = Ledger(mode="seq", task_id=task_id, k=k) for r in base: - ledger.rollout(r.label) + ledger.rollout(r.label, r.cost_usd) result = ModeResult(mode="seq", task_id=task_id, k=k, ledger=ledger) result.rollouts.update({r.label: r for r in base}) ctx_dir = work_dir / "context" diff --git a/harness2/core/runmark.py b/harness2/core/runmark.py index 1de5794b..b4d58ed2 100644 --- a/harness2/core/runmark.py +++ b/harness2/core/runmark.py @@ -33,6 +33,21 @@ VERSION = 1 +def relative_path(path: Path | str, root: Path) -> str: + """Persist paths inside the run root without tying them to its current location.""" + resolved = Path(path).resolve() + return str( + resolved.relative_to(root.resolve()) + if resolved.is_relative_to(root.resolve()) + else resolved + ) + + +def resolve_path(path: Path | str, root: Path) -> Path: + """Read both relocatable records and legacy absolute paths.""" + return root / Path(path) + + def _ignored(p: Path) -> bool: """Byproducts that are not part of the harness and must not change its identity.""" return "__pycache__" in p.parts or p.suffix == ".pyc" or p.name == ".DS_Store" diff --git a/harness2/core/schedule.py b/harness2/core/schedule.py index f91f450b..87d853d8 100644 --- a/harness2/core/schedule.py +++ b/harness2/core/schedule.py @@ -24,6 +24,7 @@ import os import random import re +import time import traceback from collections import defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed @@ -32,8 +33,8 @@ from pathlib import Path from typing import Mapping, Sequence +from . import events, recursion, runmark, selection from . import library as library_store -from . import recursion, runmark, selection from . import tree as harness_tree from .proposer import Proposer from .spec import BenchmarkAdapter, RunView @@ -188,9 +189,7 @@ def _selector_for( if prop.spec == spec: return prop audit = Path(prop.audit_dir) / "selector" if prop.audit_dir else None - return Proposer( - spec, audit_dir=audit, attempts=prop.attempts, project=prop.project, region=prop.region - ) + return Proposer(spec, audit_dir=audit, attempts=prop.attempts) def _tree_is_base(surface, base_tree: harness_tree.Tree, tree: harness_tree.Tree) -> bool: @@ -391,7 +390,7 @@ def _resume( sel = cached_sels[mode] namespace = Namespace(adapter.spec.name, adapter.substrate.name, mode, k, seed, tag) harness_dir = namespace.task_dir(runs_root, task_id) / "harnesses" / sel["chosen_label"] - run_dir = Path(sel.get("chosen_run_dir") or "") + run_dir = runmark.resolve_path(sel.get("chosen_run_dir") or "", adapter.spec.runs_root) try: if not sel.get("chosen_run_dir") or not run_dir.is_dir(): raise ValueError("adopted run directory is missing") @@ -434,8 +433,19 @@ def _roll_bases( if not pending: return {} rolled: dict[str, RunView] = {} + work_dir = dead_dir.parent + for label, rid in pending: + cached = recursion._cached_rollout(adapter, work_dir, label, rid, None) + if cached is not None: + rolled[label] = cached + pending = [(label, rid) for label, rid in pending if label not in rolled] + if not pending: + return rolled execute_many = getattr(adapter, "execute_many", None) if len(pending) > 1 and callable(execute_many): + for _ in pending: + events.check() + started = time.monotonic() try: dirs = list(execute_many(task_id, None, [rid for _lbl, rid in pending])) for i, (label, rid) in enumerate(pending): @@ -454,27 +464,29 @@ def _roll_bases( delivered=exists and adapter.delivered(rd), error="" if exists else "adapter returned a missing run directory", ) + except events.Stopped: + raise except Exception as exc: for label, rid in pending: rolled[label] = _dead(label, rid, dead_dir, f"{type(exc).__name__}: {exc}") + for label, _rid in pending: + recursion._finish_rollout(adapter, work_dir, task_id, rolled[label], None, started) return rolled def _one(label_rid: tuple[str, str]) -> RunView: label, rid = label_rid - try: - rd = adapter.execute(task_id, None, rid) - return RunView( - label=label, run_id=rid, run_dir=rd, ok=True, delivered=adapter.delivered(rd) - ) - except Exception as exc: - return _dead(label, rid, dead_dir, f"{type(exc).__name__}: {exc}") + return recursion._roll( + adapter, None, task_id, label, None, lambda _label: rid, work_dir, record=False + ) if len(pending) == 1: rv = _one(pending[0]) - return {rv.label: rv} - width = min(len(pending), ROLLOUT_PARALLEL) - with ThreadPoolExecutor(max_workers=max(1, width)) as pool_r: - return {rv.label: rv for rv in pool_r.map(_one, pending)} + rolled[rv.label] = rv + else: + width = min(len(pending), ROLLOUT_PARALLEL) + with ThreadPoolExecutor(max_workers=max(1, width)) as pool_r: + rolled.update({rv.label: rv for rv in pool_r.map(_one, pending)}) + return rolled def _run_one_mode( @@ -493,6 +505,7 @@ def _run_one_mode( surface, pool_policy: str, res: TaskResult, + reflector_prop: Proposer | None = None, ) -> None: """Run one recursion mode over the shared bases, select a final, and record it on `res`.""" work_dir = n.task_dir(runs_root, task_id) @@ -510,6 +523,7 @@ def _run_one_mode( library_dir=lib_dir, ) res.modes[mode] = r.to_dict() + events.check() candidates = selection.selection_pool(mode, k, r.rollouts, base) if pool_policy == "finals" and k == 1: sel = selection.trivial_selection( @@ -526,19 +540,23 @@ def _run_one_mode( work_dir=work_dir, pool_kind="finals", ) + events.emit("selection", task=task_id, mode=mode, chosen=sel.chosen_label) if mode_lib and sel.ok: + events.check() selection.reflect_adoption( task_id, sel, r.rollouts, adapter=adapter, - prop=selector, + prop=reflector_prop if reflector_prop is not None else selector, work_dir=work_dir, decisions={stage.label: stage.notes or stage.error for stage in r.stages}, ) d = sel.to_dict() - d["chosen_run_dir"] = str( - candidates[sel.chosen_label].run_dir if sel.chosen_label in candidates else "" + d["chosen_run_dir"] = ( + runmark.relative_path(candidates[sel.chosen_label].run_dir, adapter.spec.runs_root) + if sel.chosen_label in candidates + else "" ) chosen_tree = r.trees.get(sel.chosen_label, base_tree) @@ -576,6 +594,7 @@ def run_task( library: library_store.Library | None = None, libraries: dict[str, library_store.Library] | None = None, selector_prop: Proposer | None = None, + reflector_prop: Proposer | None = None, pool_policy: str = "finals", ) -> TaskResult: """k shared reference executions -> every mode -> blind selection per mode -> summary.json. @@ -613,6 +632,8 @@ def run_task( ): return res + events.check() + events.emit("task_start", task=task_id, domain=domain, status="running") surface = adapter.substrate.surface() base_tree = harness_tree.read_tree(surface, adapter.base_harness()) @@ -625,11 +646,19 @@ def run_task( rolled = _roll_bases(task_id, adapter, pending, dead_dir) base: list[RunView] = [rolled[label] for label in labels] - res.base = [{"label": b.label, "run_id": b.run_id, "run_dir": str(b.run_dir)} for b in base] + res.base = [ + { + "label": b.label, + "run_id": b.run_id, + "run_dir": runmark.relative_path(b.run_dir, adapter.spec.runs_root), + } + for b in base + ] if not any(b.ok for b in base): res.status = "base-failed" res.errors.append("no base rollout produced anything") _write_summary(summary_path, res) + events.emit("task_done", task=task_id, domain=domain, status=res.status) return res for mode in modes: @@ -643,6 +672,7 @@ def run_task( adapter=adapter, prop=prop, selector=selector, + reflector_prop=reflector_prop, n=n, runs_root=runs_root, mode_lib=mode_libs.get(mode), @@ -651,6 +681,8 @@ def run_task( pool_policy=pool_policy, res=res, ) + except events.Stopped: + raise except Exception as exc: res.modes[mode] = { "ok": False, @@ -663,6 +695,7 @@ def run_task( res.status = _merged_status(res.to_dict()) _write_summary(summary_path, res) + events.emit("task_done", task=task_id, domain=domain, status=res.status) return res @@ -679,6 +712,7 @@ def run_domain( seed: int | None = None, tag: str = "", selector_prop: Proposer | None = None, + reflector_prop: Proposer | None = None, pool_policy: str = "finals", ) -> dict: """Every task in one domain, in `stream_order()` order, threading the library forward.""" @@ -711,6 +745,7 @@ def run_domain( tag=tag, libraries=libs, selector_prop=selector_prop, + reflector_prop=reflector_prop, pool_policy=pool_policy, ) if r.status in ("ok", "skipped"): @@ -724,6 +759,8 @@ def run_domain( "error": "; ".join(r.errors) if r.errors else r.status, "library_entries_before": before, } + except events.Stopped: + raise except Exception as exc: out["failed"] += 1 row = {"task": tid, "status": "exception", "error": f"{type(exc).__name__}: {exc}"} @@ -791,6 +828,7 @@ def run_sweep( domain_parallel: int = 1, only_domains: Sequence[str] = (), selector_prop: Proposer | None = None, + reflector_prop: Proposer | None = None, pool_policy: str = "finals", max_tasks: int | None = None, only_tasks: Sequence[str] = (), @@ -824,15 +862,25 @@ def run_sweep( seed=seed, tag=tag, selector_prop=selector, + reflector_prop=reflector_prop, pool_policy=pool_policy, ): d for d in order } - for f in as_completed(futs): - try: - rows.append(f.result()) - except Exception as exc: - rows.append({"domain": futs[f], "error": f"{type(exc).__name__}: {exc}"}) + stopped = None + try: + for f in as_completed(futs): + try: + rows.append(f.result()) + except events.Stopped as exc: + stopped = exc + except Exception as exc: + rows.append({"domain": futs[f], "error": f"{type(exc).__name__}: {exc}"}) + except KeyboardInterrupt: + events.request_stop("interrupted") + raise + if stopped is not None: + raise stopped return { "benchmark": adapter.spec.name, "substrate": adapter.substrate.name, diff --git a/harness2/core/selection.py b/harness2/core/selection.py index 2cf04f50..6d88bdb7 100644 --- a/harness2/core/selection.py +++ b/harness2/core/selection.py @@ -25,7 +25,7 @@ from pathlib import Path from typing import Callable, Sequence, TypeVar -from . import context +from . import context, runmark from . import prompts as P from .proposer import Proposer, ProposerError from .spec import BenchmarkAdapter, BenchPrompts, RunView, resolve_prompts @@ -352,7 +352,9 @@ def run_selection( sel.chosen_run_id = pool[sel.chosen_label].run_id sel.chosen_is_base = _is_base(sel.chosen_label) - _persist(sel, root, str(pool[sel.chosen_label].run_dir if sel.chosen_label in pool else "")) + _persist( + sel, root, runmark.relative_path(pool[sel.chosen_label].run_dir, adapter.spec.runs_root) + ) return sel @@ -395,7 +397,7 @@ def trivial_selection( root = Path(work_dir) / "selection" / f"{mode}{k}" shutil.rmtree(root, ignore_errors=True) root.mkdir(parents=True, exist_ok=True) - _persist(sel, root, str(pool[label].run_dir)) + _persist(sel, root, runmark.relative_path(pool[label].run_dir, adapter.spec.runs_root)) return sel @@ -447,7 +449,11 @@ def reflect_adoption( task_id, sel, candidates, adapter=adapter, prop=prop, root=root, layout=evidence.layout ) sel.experience = experience - _persist(sel, Path(work_dir) / "selection" / f"{sel.mode}{sel.k}", str(chosen.run_dir)) + _persist( + sel, + Path(work_dir) / "selection" / f"{sel.mode}{sel.k}", + runmark.relative_path(chosen.run_dir, adapter.spec.runs_root), + ) return experience diff --git a/harness2/core/spec.py b/harness2/core/spec.py index e58d1485..3738cc5d 100644 --- a/harness2/core/spec.py +++ b/harness2/core/spec.py @@ -25,12 +25,12 @@ from pathlib import Path from typing import Protocol, runtime_checkable -from .. import config +from ..providers import Provider from .tree import SurfaceSpec ModelId = str -CHANNELS = ("claude_cli", "opencode") +CHANNELS = ("claude", "codex", "opencode", "human") @dataclass(frozen=True) @@ -42,50 +42,16 @@ class ChannelSpec: effort: str = "high" steps: int = 120 timeout: int = 3600 + bin: str = "" + provider: Provider | None = None def __post_init__(self) -> None: + if self.kind == "claude_cli": + object.__setattr__(self, "kind", "claude") if self.kind not in CHANNELS: raise ValueError(f"unknown channel {self.kind!r}; expected one of {CHANNELS}") -def resolve_channel_model( - kind: str | None, - model: str | None, - *, - effort: str = "high", - steps: int = 120, - timeout: int = 3600, -) -> ChannelSpec: - """The one channel->model mapping; a lone override of either raises.""" - use_default = kind is None and model is None - if (kind is None) != (model is None): - raise ValueError( - "channel and model must be overridden TOGETHER (a bare-id/provider-prefixed " - f"mismatch fails silently downstream); got kind={kind!r}, model={model!r}" - ) - if kind is None: - kind = config.DEFAULT_PROPOSER_CHANNEL - if kind not in CHANNELS: - raise ValueError(f"unknown channel {kind!r}; expected one of {CHANNELS}") - if model is None: - model = config.PROPOSER_MODEL_BY_CHANNEL[kind] - if not model.strip(): - if use_default: - return ChannelSpec(kind, "", effort=effort, steps=steps, timeout=timeout) - raise ValueError("model override must not be empty") - if kind == "opencode" and "/" not in model: - raise ValueError( - f"opencode channel needs a provider-prefixed model id " - f"in HARNESS2_IMPROVER_MODEL or HARNESS2_SOLVER_MODEL, got {model!r}" - ) - if kind == "claude_cli" and "/" in model: - raise ValueError( - f"claude_cli channel takes a bare Anthropic id " - f"(e.g. {config.PROPOSER_MODEL_BY_CHANNEL['claude_cli']!r}), got {model!r}" - ) - return ChannelSpec(kind, model, effort=effort, steps=steps, timeout=timeout) - - @dataclass(frozen=True) class BenchPrompts: """The benchmark-shaped text the shared prompts interpolate.""" @@ -271,7 +237,6 @@ def resolve_prompts(adapter: BenchmarkAdapter, task_id: str) -> BenchPrompts: "RunView", "ModelId", "resolve_prompts", - "resolve_channel_model", "CHANNELS", "INSTALL_MODES", ] diff --git a/harness2/demo.py b/harness2/demo.py index 487de02d..01c7c9e8 100644 --- a/harness2/demo.py +++ b/harness2/demo.py @@ -22,8 +22,8 @@ from decimal import Decimal from pathlib import Path -from .core import tree -from .core.proposer import Proposer, Reply +from .core import runmark, tree +from .core.proposer import Proposer from .core.spec import BenchPrompts, BenchSpec, ChannelSpec from .substrate.opencode import OpencodeSubstrate @@ -86,8 +86,14 @@ def validate_tree(self, harness: tree.Tree) -> None: def execute( self, task_id: str, harness_dir: Path | None, run_id: str, *, timeout: int | None = None ) -> Path: + run_dir = self.spec.runs_root / "executions" / run_id + if ( + runmark.is_complete(run_dir, harness_dir=harness_dir) + and self.delivered(run_dir) + and (run_dir / "trajectory.md").is_file() + ): + return run_dir self.execution_count += 1 - run_dir = self.root / "executions" / run_id run_dir.mkdir(parents=True, exist_ok=True) script = (harness_dir or self.base_harness()) / "scripts/summarize.py" result = subprocess.run( @@ -102,6 +108,7 @@ def execute( "[ACTION] Run scripts/summarize.py on ledger.csv\n[OBSERVATION]\n" + result.stdout, encoding="utf-8", ) + runmark.mark_complete(run_dir, run_id=run_id, harness_dir=harness_dir) return run_dir def delivered(self, run_dir: Path) -> bool: @@ -120,10 +127,9 @@ def deliverable_files(self, run_dir: Path) -> dict[str, str]: class DemoProposer(Proposer): """Scripted edits and selection; no model service or benchmark judge is used.""" - def call(self, mission, context_dir, label, **kwargs) -> Reply: - context = Path(context_dir) - workspace = None - if label.startswith("select."): + def _opencode(self, cwd, mission, system, wall): + context = cwd / "context" + if (context / "candidates").is_dir(): with (context / "source/ledger.csv").open() as source: expected = sum( (Decimal(row["amount"]) for row in csv.DictReader(source)), Decimal(0) @@ -135,7 +141,7 @@ def call(self, mission, context_dir, label, **kwargs) -> Reply: text = ( f"The signed total matches the source ledger.\n```choice\n{chosen}\n```" ) - elif label.startswith("experience."): + elif (context / "adoption.json").is_file(): changed = (context / "harness/scripts/summarize.py").read_text() != ( context / "base_harness/scripts/summarize.py" ).read_text() @@ -158,6 +164,4 @@ def call(self, mission, context_dir, label, **kwargs) -> Reply: script = workspace / "scripts/summarize.py" script.write_text(script.read_text().replace(" if amount > 0", ""), encoding="utf-8") text = "Keep signed amounts: the base total omitted credits in the source ledger." - reply = Reply(label=label, text=text, ok=True, attempts=1, workspace=workspace) - self._calls.append(reply) - return reply + return text, 0.0, "" diff --git a/harness2/substrate/codex.py b/harness2/substrate/codex.py index fd34eacd..a434b435 100644 --- a/harness2/substrate/codex.py +++ b/harness2/substrate/codex.py @@ -268,7 +268,12 @@ def parse_items(path: Path) -> list[dict]: raw = Path(path).read_text(encoding="utf-8", errors="replace") except OSError as e: return [{"type": "_read_error", "message": str(e)}] - raw = raw.replace("\x00", "") + return parse_items_text(raw) + + +def parse_items_text(text: str) -> list[dict]: + """Parse an exec JSONL string, keeping the last state of each thread item.""" + raw = text.replace("\x00", "") events: list[dict] = [] by_id: dict[str, int] = {} diff --git a/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/PKG-INFO b/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/PKG-INFO new file mode 100644 index 00000000..da5fd38f --- /dev/null +++ b/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/PKG-INFO @@ -0,0 +1,11 @@ +Metadata-Version: 2.4 +Name: harness2-benchmarks +Version: 0.1.0 +License-Expression: Apache-2.0 +Requires-Python: <3.14,>=3.12 +License-File: LICENSE +Requires-Dist: harness2 +Provides-Extra: render +Requires-Dist: openpyxl<4,>=3.1; extra == "render" +Requires-Dist: python-pptx<2,>=1; extra == "render" +Dynamic: license-file diff --git a/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/SOURCES.txt b/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/SOURCES.txt new file mode 100644 index 00000000..c3da0e76 --- /dev/null +++ b/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/SOURCES.txt @@ -0,0 +1,77 @@ +LICENSE +pyproject.toml +harness2_benchmarks/__init__.py +harness2_benchmarks/config.py +harness2_benchmarks/install.py +harness2_benchmarks/jb_opencode_runner.py +harness2_benchmarks/jobbench.py +harness2_benchmarks/lab.py +harness2_benchmarks/preflight.py +harness2_benchmarks/setup.py +harness2_benchmarks/tasks.py +harness2_benchmarks/wb_render.py +harness2_benchmarks/workbuddy.py +harness2_benchmarks.egg-info/PKG-INFO +harness2_benchmarks.egg-info/SOURCES.txt +harness2_benchmarks.egg-info/dependency_links.txt +harness2_benchmarks.egg-info/entry_points.txt +harness2_benchmarks.egg-info/requires.txt +harness2_benchmarks.egg-info/top_level.txt +harness2_benchmarks/dependencies_meta/ASSETS.json +harness2_benchmarks/dependencies_meta/PINNED.md +harness2_benchmarks/dependencies_meta/README.md +harness2_benchmarks/suites/__init__.py +harness2_benchmarks/suites/jb.py +harness2_benchmarks/suites/lab.py +harness2_benchmarks/suites/wb_code.py +harness2_benchmarks/suites/wb_office.py +harness2_benchmarks/suites/wb_web.py +harness2_benchmarks/third_party/README.md +harness2_benchmarks/third_party/jb_runner/LICENSE +harness2_benchmarks/third_party/jb_runner/PROVENANCE.md +harness2_benchmarks/third_party/jb_runner/run_benchmark_codex_cli_adapt.sh +harness2_benchmarks/third_party/jb_runner/codex_base_harness/AGENTS.md +harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/LongTermMEMORY.md +harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/guardrails.md +harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/harness.yaml +harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/systemprompt.md +harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/plugin/README.md +harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/scripts/README.md +harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/sub_agents/README.md +harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/tool_descriptions/tool_guidance.md +harness2_benchmarks/third_party/lab_runner/LICENSE +harness2_benchmarks/third_party/lab_runner/PROVENANCE.md +harness2_benchmarks/third_party/lab_runner/adapt_runs_cx/run_codex_lab.py +harness2_benchmarks/third_party/lab_runner/opencode/README.md +harness2_benchmarks/third_party/lab_runner/opencode/__init__.py +harness2_benchmarks/third_party/lab_runner/opencode/render.py +harness2_benchmarks/third_party/lab_runner/opencode/render_tree.py +harness2_benchmarks/third_party/lab_runner/opencode/run_opencode.py +harness2_benchmarks/third_party/lab_runner/opencode/session.py +harness2_benchmarks/third_party/lab_runner/opencode/scripts/read_doc.py +harness2_benchmarks/third_party/lab_runner/tts_lab/__init__.py +harness2_benchmarks/third_party/lab_runner/tts_lab/config.py +harness2_benchmarks/third_party/licenses/harvey-labs.LICENSE +harness2_benchmarks/third_party/licenses/job-bench-eval.LICENSE +harness2_benchmarks/third_party/licenses/opencode.LICENSE +harness2_benchmarks/third_party/licenses/workbuddy-bench.LICENSE +harness2_benchmarks/third_party/patches/harvey-labs-judge-vertex-anthropic.diff +harness2_benchmarks/third_party/patches/job-bench-eval-judge.diff +harness2_benchmarks/third_party/patches/opencode-compat.diff +harness2_benchmarks/third_party/patches/workbuddy-bench-runtime.diff +harness2_benchmarks/third_party/wb_runner/LICENSE +harness2_benchmarks/third_party/wb_runner/PROVENANCE.md +harness2_benchmarks/third_party/wb_runner/agents/codex_agent.py +harness2_benchmarks/third_party/wb_runner/agents/codex_harness_agent.py +harness2_benchmarks/third_party/wb_runner/agents/oc_agent.py +harness2_benchmarks/third_party/wb_runner/agents/oc_harness_agent.py +harness2_benchmarks/third_party/wb_runner/configs/harnesses/codex-harness/_defaults.yaml +harness2_benchmarks/third_party/wb_runner/configs/harnesses/codex-harness/versions/0.144.6.yaml +harness2_benchmarks/third_party/wb_runner/configs/harnesses/oc-harness/_defaults.yaml +harness2_benchmarks/third_party/wb_runner/configs/harnesses/oc-harness/versions/1.14.18.yaml +harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/CONFIG.md +harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/_defaults.yaml +harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/docker/Dockerfile +harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/presets/settings/v1_14.json +harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/versions/1.14.18.yaml +tests/test_plugin.py \ No newline at end of file diff --git a/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/dependency_links.txt b/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/dependency_links.txt new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/entry_points.txt b/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/entry_points.txt new file mode 100644 index 00000000..f9e6a524 --- /dev/null +++ b/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/entry_points.txt @@ -0,0 +1,6 @@ +[harness2.suites] +jb = harness2_benchmarks.suites.jb +lab = harness2_benchmarks.suites.lab +wb-code = harness2_benchmarks.suites.wb_code +wb-office = harness2_benchmarks.suites.wb_office +wb-web = harness2_benchmarks.suites.wb_web diff --git a/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/requires.txt b/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/requires.txt new file mode 100644 index 00000000..22f09d90 --- /dev/null +++ b/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/requires.txt @@ -0,0 +1,5 @@ +harness2 + +[render] +openpyxl<4,>=3.1 +python-pptx<2,>=1 diff --git a/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/top_level.txt b/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/top_level.txt new file mode 100644 index 00000000..bd82b800 --- /dev/null +++ b/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/top_level.txt @@ -0,0 +1 @@ +harness2_benchmarks diff --git a/plugins/harness2-benchmarks/harness2_benchmarks/config.py b/plugins/harness2-benchmarks/harness2_benchmarks/config.py index 2b59f9de..e16daba0 100644 --- a/plugins/harness2-benchmarks/harness2_benchmarks/config.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/config.py @@ -18,7 +18,6 @@ import os import sys -from dataclasses import fields from pathlib import Path from harness2.core.spec import ChannelSpec @@ -172,13 +171,11 @@ def resolve_channel_model( ) -> ChannelSpec: """The one channel->model mapping; a lone override of either raises.""" kwargs = {"effort": effort, "steps": steps, "timeout": timeout} - channel_fields = {field.name for field in fields(ChannelSpec)} - # shortcut: W2 adds bin/provider, remove the field guards when it is integrated. - if "bin" in channel_fields and (kind or DEFAULT_PROPOSER_CHANNEL) == "opencode": + if (kind or DEFAULT_PROPOSER_CHANNEL) == "opencode": kwargs["bin"] = ( f"{BUN} run --cwd {OPENCODE_DIR}/packages/opencode --conditions=browser src/index.ts" ) - if "provider" in channel_fields and VERTEX_PROJECT: + if VERTEX_PROJECT: kwargs["provider"] = Provider( "vertex", "vertex", project=VERTEX_PROJECT, region=VERTEX_REGION ) diff --git a/plugins/harness2-benchmarks/tests/test_plugin.py b/plugins/harness2-benchmarks/tests/test_plugin.py index 554d51bc..c24b68ee 100644 --- a/plugins/harness2-benchmarks/tests/test_plugin.py +++ b/plugins/harness2-benchmarks/tests/test_plugin.py @@ -25,7 +25,6 @@ import sys import tempfile import unittest -from dataclasses import dataclass from importlib.metadata import entry_points from pathlib import Path from types import SimpleNamespace @@ -127,19 +126,8 @@ def test_channels_keep_legacy_validation(self): with self.subTest(kind=kind, model=model), self.assertRaises(ValueError): config.resolve_channel_model(kind, model) - def test_channels_supply_new_runtime_fields_when_available(self): - @dataclass - class Channel: - kind: str - model: str - effort: str - steps: int - timeout: int - bin: str = "" - provider: object = None - + def test_channels_supply_pinned_opencode_and_vertex_provider(self): with ( - patch.object(config, "ChannelSpec", Channel), patch.object(config, "VERTEX_PROJECT", "test-project"), patch.object(config, "VERTEX_REGION", "test-region"), ): @@ -153,10 +141,7 @@ class Channel: self.assertEqual(ch.provider.region, "test-region") self.assertEqual((ch.steps, ch.timeout), (3, 4)) self.assertEqual(config.resolve_channel_model("claude_cli", "claude-x").bin, "") - with ( - patch.object(config, "ChannelSpec", Channel), - patch.object(config, "VERTEX_PROJECT", ""), - ): + with patch.object(config, "VERTEX_PROJECT", ""): self.assertIsNone(config.resolve_channel_model("opencode", "provider/model").provider) def test_dependency_paths_use_cache_or_env(self): diff --git a/tests/test_runtime.py b/tests/test_runtime.py new file mode 100644 index 00000000..da0d86bc --- /dev/null +++ b/tests/test_runtime.py @@ -0,0 +1,620 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline runtime contracts: channels, accounting, stop/resume, and relocation.""" + +from __future__ import annotations + +import json +import os +import shutil +import sys +import tempfile +import threading +import tomllib +import unittest +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from unittest.mock import Mock, patch + +from harness2.core import configure, events, judge, recursion, runmark, schedule +from harness2.core.proposer import SYSTEM_ARG_MAX, Proposer, ProposerError, _call_fingerprint +from harness2.core.spec import CHANNELS, ChannelSpec +from harness2.demo import DemoAdapter, DemoProposer +from harness2.providers import STRIP_ENV, Provider +from harness2.substrate.codex import parse_items, parse_items_text + + +class EventsTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.path = Path(temporary.name) / "nested/events.jsonl" + self.addCleanup(events.start, None) + events.start(None) + + def test_disabled_until_started_and_request_stop(self): + with patch.object(events, "_active", False): + events.emit("session", cost_usd=2) + events.check() + self.assertEqual(events.totals()["calls"], 0) + events.request_stop("requested") + with self.assertRaisesRegex(events.Stopped, "requested"): + events.check() + events.start(None) + events.check() + self.assertIsNone(events.totals()["stopped"]) + + def test_budget_resume_and_cached_sessions(self): + events.start(self.path, budget_usd=0.5, max_calls=9) + events.emit("task_start", task="one") + events.emit("session", cost_usd=0.2, cached=False) + events.emit("session", cost_usd=0, cached=True) + events.emit("rollout", cost_usd=0.3) + self.assertEqual( + events.totals(), + { + "spent_usd": 0.5, + "calls": 2, + "budget_usd": 0.5, + "max_calls": 9, + "stopped": "budget_usd", + }, + ) + with self.assertRaisesRegex(events.Stopped, "budget_usd"): + events.check() + events.start(self.path, max_calls=3) + events.check() + events.emit("session") + with self.assertRaisesRegex(events.Stopped, "max_calls"): + events.check() + records = [json.loads(line) for line in self.path.read_text().splitlines()] + self.assertEqual(len(records), 5) + self.assertTrue(all(record["t"].endswith("Z") for record in records)) + for kwargs in ({"budget_usd": -1}, {"max_calls": -1}): + with self.assertRaises(ValueError): + events.start(None, **kwargs) + + def test_concurrent_events_are_whole_lines_and_counted_once(self): + events.start(self.path) + with ThreadPoolExecutor(max_workers=4) as pool: + list(pool.map(lambda i: events.emit("rollout", task=str(i), cost_usd=0.1), range(40))) + self.assertEqual(len([json.loads(line) for line in self.path.read_text().splitlines()]), 40) + self.assertEqual((events.totals()["calls"], events.totals()["spent_usd"]), (40, 4)) + + +FAKE_CLI = r"""import json +import os +import sys +from pathlib import Path + +cwd = Path.cwd() +kind = Path(sys.argv[0]).name +record = {"argv": sys.argv[1:], "env": dict(os.environ), + "stdin": sys.stdin.read() if kind != "opencode" else "", + "cwd": str(cwd)} +for key, name in (("CODEX_HOME", "config.toml"), ("CLAUDE_CONFIG_DIR", "")): + if key in os.environ: + home = Path(os.environ[key]) + record[key + "_exists"] = home.is_dir() + if name: + record["config"] = (home / name).read_text() +if (cwd / "opencode.json").is_file(): + record["config"] = json.loads((cwd / "opencode.json").read_text()) + record["mission"] = (cwd / "MISSION.md").read_text() +with open(os.environ["FAKE_RECORD"], "a") as output: + output.write(json.dumps(record) + "\n") +if "FAKE_OUTPUT" in os.environ: + print(os.environ["FAKE_OUTPUT"]) +elif kind == "claude": + print(json.dumps({"result": "done", "total_cost_usd": 0.25, "subtype": "success"})) +elif kind == "codex": + print(json.dumps({"type": "item.completed", "item": { + "id": "first", "type": "agent_message", "text": "earlier"}})) + print(json.dumps({"type": "item.completed", "item": { + "id": "last", "type": "agent_message", "text": "done"}})) +else: + print(json.dumps({"type": "text", "part": {"text": "done"}})) + print(json.dumps({"type": "step_finish", "part": {"cost": 0.125}})) +sys.exit(int(os.environ.get("FAKE_RC", "0"))) +""" + + +class ProposerTests(unittest.TestCase): + maxDiff = None + + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + binary = self.root / "bin" + binary.mkdir() + for kind in ("claude", "codex", "opencode"): + path = binary / kind + path.write_text(f"#!{sys.executable}\n" + FAKE_CLI) + path.chmod(0o755) + self.record = self.root / "record.jsonl" + self.base_env = { + "PATH": f"{binary}:{os.defpath}", + "FAKE_RECORD": str(self.record), + "CUSTOM_KEY": "fake-key", + "LC_ALL": "C", + } + self.environment = patch.dict( + os.environ, self.base_env | dict.fromkeys(STRIP_ENV, "stale"), clear=True + ) + self.environment.start() + self.addCleanup(self.environment.stop) + self.addCleanup(events.start, None) + events.start(None) + + def records(self): + return [json.loads(line) for line in self.record.read_text().splitlines()] + + def test_channels_alias_empty_and_human(self): + self.assertEqual(CHANNELS, ("claude", "codex", "opencode", "human")) + self.assertEqual(ChannelSpec("claude_cli", "local/model").kind, "claude") + with self.assertRaises(ValueError): + ChannelSpec("missing", "model") + with self.assertRaises(ValueError): + Proposer(ChannelSpec("claude", "")) + human = Proposer(ChannelSpec("human", "")) + with self.assertRaisesRegex(ProposerError, "interactive selector"): + human.call("choose", None, "one") + self.assertFalse(self.record.exists()) + + def test_channel_argv_env_and_provider_config(self): + providers = { + "claude": Provider("local", "anthropic", "https://claude.invalid", "CUSTOM_KEY"), + "codex": Provider("local", "openai", "https://responses.invalid/v1", "CUSTOM_KEY"), + "opencode": Provider("local", "openai", "https://chat.invalid/v1", "CUSTOM_KEY"), + } + for kind, provider in providers.items(): + with self.subTest(kind=kind): + prop = Proposer(ChannelSpec(kind, "local/test-model", provider=provider)) + reply = prop.call("mission", None, kind, system="system") + self.assertEqual(reply.text, "done") + self.assertEqual( + reply.cost_usd, {"claude": 0.25, "codex": 0, "opencode": 0.125}[kind] + ) + record = self.records()[-1] + cwd = record["cwd"] + expected_env = dict(self.base_env) + if kind == "claude": + self.assertEqual( + record["argv"], + [ + "--dangerously-skip-permissions", + "-p", + "--output-format", + "json", + "--effort", + "high", + "--model", + "test-model", + "--setting-sources", + "project,local", + "--append-system-prompt", + "system", + ], + ) + expected_env |= { + "ANTHROPIC_API_KEY": "fake-key", + "ANTHROPIC_BASE_URL": provider.base_url, + "CLAUDE_CONFIG_DIR": cwd + "/.claude-home", + } + self.assertTrue(record["CLAUDE_CONFIG_DIR_exists"]) + self.assertEqual(record["stdin"], "mission") + elif kind == "codex": + self.assertEqual( + record["argv"], + [ + "exec", + "--json", + "--skip-git-repo-check", + "--dangerously-bypass-approvals-and-sandbox", + "--ephemeral", + "-C", + cwd, + "-", + ], + ) + expected_env["CODEX_HOME"] = cwd + "/.codex-home" + self.assertTrue(record["CODEX_HOME_exists"]) + self.assertEqual( + record["stdin"], "# SYSTEM INSTRUCTIONS\n\nsystem\n\n# END\n\nmission" + ) + config = tomllib.loads(record["config"]) + self.assertEqual(config["model"], "test-model") + self.assertEqual(config["model_provider"], "harness2-local") + self.assertEqual( + config["model_providers"]["harness2-local"]["env_key"], "CUSTOM_KEY" + ) + else: + self.assertEqual( + record["argv"], + [ + "run", + "Read MISSION.md first. Do what it asks. " + "Reply with the requested output as your final message AND save it verbatim to output/answer.md.", + "--dir", + cwd, + "--model", + "local/test-model", + "--format", + "json", + "--variant", + "high", + ], + ) + expected_env |= { + "OPENCODE_PERMISSION": json.dumps( + { + "external_directory": { + "*": "deny", + cwd: "allow", + cwd + "/**": "allow", + } + } + ), + "XDG_DATA_HOME": cwd + "/.ocdata", + "XDG_STATE_HOME": cwd + "/.ocstate", + } + self.assertEqual(record["stdin"], "") + self.assertEqual(record["mission"], "system\n\n---\n\nmission") + self.assertEqual(record["config"]["agent"]["build"]["steps"], 120) + self.assertEqual( + record["config"]["provider"]["local"]["models"], {"test-model": {}} + ) + observed_env = dict(record["env"]) + # macOS injects this into the child after subprocess receives its environment. + observed_env.pop("__CF_USER_TEXT_ENCODING", None) + self.assertEqual(observed_env, expected_env) + normalized = json.dumps( + {"argv": [kind, *record["argv"]], "env": record["env"]}, sort_keys=True + ) + normalized = normalized.replace(cwd, "").replace(str(self.root), "") + print(f"PROPOSER {kind}: {normalized}") + + def test_no_provider_custom_binary_long_system_and_codex_config(self): + system = "s" * (SYSTEM_ARG_MAX + 1) + binary = f'"{self.root}/bin/claude" --wrapper' + Proposer(ChannelSpec("claude", "bare-model", bin=binary)).call( + "mission", None, "long", system=system + ) + record = self.records()[-1] + self.assertEqual(record["argv"][0], "--wrapper") + self.assertEqual(record["stdin"], f"# SYSTEM INSTRUCTIONS\n\n{system}\n\n# END\n\nmission") + self.assertFalse(any(key in record["env"] for key in STRIP_ENV)) + Proposer(ChannelSpec("codex", "bare-model")).call("mission", None, "plain") + self.assertEqual( + tomllib.loads(self.records()[-1]["config"]), + {"model": "bare-model", "model_reasoning_effort": "high"}, + ) + + def test_vertex_env_is_explicit(self): + provider = Provider("vertex", "vertex", project="test-project", region="test-region") + prop = Proposer(ChannelSpec("opencode", "google-vertex/test-model", provider=provider)) + prop.call("mission", None, "vertex") + record = self.records()[-1] + self.assertEqual(record["env"]["GOOGLE_VERTEX_PROJECT"], "test-project") + self.assertEqual(record["env"]["GOOGLE_VERTEX_LOCATION"], "test-region") + self.assertEqual(record["env"]["VERTEXAI_LOCATION"], "test-region") + self.assertNotIn("provider", record["config"]) + + def test_codex_failure_events_and_returncode_emit_failed_sessions(self): + for output, rc in ( + (json.dumps({"type": "turn.failed", "error": {"message": "failed"}}), "0"), + (json.dumps({"type": "error", "message": "bad"}), "0"), + ( + json.dumps( + {"type": "item.completed", "item": {"type": "agent_message", "text": "partial"}} + ), + "2", + ), + ): + with ( + self.subTest(output=output, rc=rc), + patch.dict(os.environ, {"FAKE_OUTPUT": output, "FAKE_RC": rc}), + ): + events.start(None) + prop = Proposer(ChannelSpec("codex", "local/model"), attempts=1) + with self.assertRaises(ProposerError): + prop.call("mission", None, "failure") + self.assertEqual(events.totals()["calls"], 1) + self.assertFalse(prop._calls[-1].ok) + + def test_cache_budget_harvest_and_fingerprint_relocation(self): + context = self.root / "run/context/task" + (context / "harness").mkdir(parents=True) + (context / "harness/CLAUDE.md").write_text("base") + audit = self.root / "run/audit" + prop = Proposer(ChannelSpec("claude", "local/model"), audit_dir=audit) + events.start(None, max_calls=1) + reply = prop.call("mission", context, "one", harvest="context/harness") + self.assertEqual((reply.workspace / "CLAUDE.md").read_text(), "base") + fingerprint = _call_fingerprint("one", "mission", "", context, audit) + moved = self.root / "moved" + shutil.move(self.root / "run", moved) + context, audit = moved / "context/task", moved / "audit" + self.assertEqual(_call_fingerprint("one", "mission", "", context, audit), fingerprint) + prop = Proposer(prop.spec, audit_dir=audit) + self.assertTrue(prop.call("mission", context, "one", harvest="context/harness").cached) + self.assertEqual((len(self.records()), events.totals()["calls"]), (1, 1)) + with self.assertRaises(events.Stopped): + prop.call("different", context, "two") + shutil.rmtree(prop._calls[-1].workspace) + with self.assertRaises(events.Stopped): + prop.call("mission", context, "one", harvest="context/harness") + + def test_jsonl_text_and_path_share_deduplication(self): + text = ( + "junk\n" + + "\n".join( + json.dumps(record) + for record in [ + { + "type": "item.started", + "item": {"id": "x", "type": "agent_message", "text": "old"}, + }, + {"type": "turn.completed"}, + { + "type": "item.completed", + "item": {"id": "x", "type": "agent_message", "text": "last"}, + }, + ] + ) + + "\n{invalid\n" + ) + path = self.root / "stream.jsonl" + path.write_text(text) + self.assertEqual(parse_items_text(text), parse_items(path)) + self.assertEqual( + [record["type"] for record in parse_items_text(text)], + ["item.completed", "turn.completed"], + ) + self.assertEqual(parse_items_text(text)[0]["item"]["text"], "last") + self.assertEqual(parse_items(self.root / "missing")[0]["type"], "_read_error") + + +class RuntimeWorkflowTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) / "original" + self.adapter = DemoAdapter(self.root) + self.prop = DemoProposer( + self.adapter.spec.channels["proposer"], audit_dir=self.adapter.spec.runs_root / "audit" + ) + self.addCleanup(events.start, None) + self.addCleanup(configure, phase_width=2, transport_attempts=2, proposer_concurrency=2) + events.start(None) + + def sweep(self, k=1, **kwargs): + return schedule.run_sweep( + k, + adapter=self.adapter, + prop=self.prop, + runs_root=self.adapter.spec.runs_root, + library_root=self.root / "library", + **kwargs, + ) + + def test_budget_stop_resume_without_repeated_work(self): + event_path = self.adapter.spec.runs_root / "events.jsonl" + events.start(event_path, max_calls=3) + with self.assertRaisesRegex(events.Stopped, "max_calls"): + self.sweep() + self.assertEqual(events.totals()["calls"], 3) + before = self.adapter.execution_count + self.assertEqual(before, 1) + events.start(event_path, max_calls=100) + result = self.sweep() + self.assertEqual(result["domains"][0]["done"], 2) + records = [json.loads(line) for line in event_path.read_text().splitlines()] + rollouts = [record["run_id"] for record in records if record["kind"] == "rollout"] + self.assertEqual(len(rollouts), len(set(rollouts))) + self.assertEqual(self.adapter.execution_count, 14) + self.assertEqual(events.totals()["calls"], 30) + executions, calls = self.adapter.execution_count, len(self.prop._calls) + self.sweep() + self.assertEqual((self.adapter.execution_count, len(self.prop._calls)), (executions, calls)) + print( + f"BUDGET STOP/RESUME: stopped=3 calls; executions={executions}; " + f"total_calls={events.totals()['calls']}; repeated_rollouts=0" + ) + + def test_relocation_skips_executions_and_proposals_and_resolves_judge_paths(self): + self.sweep(k=2) + moved = self.root.with_name("moved") + shutil.move(self.root, moved) + self.root = moved + self.adapter = DemoAdapter(moved) + self.prop = DemoProposer(self.prop.spec, audit_dir=self.adapter.spec.runs_root / "audit") + self.adapter.score = Mock( + side_effect=lambda rd, task: ( + {"n_passed": 1, "n_criteria": 1} if self.adapter.delivered(rd) else None + ) + ) + result = self.sweep(k=2) + self.assertEqual( + [row["status"] for row in result["domains"][0]["tasks"]], ["skipped", "skipped"] + ) + self.assertEqual((self.adapter.execution_count, len(self.prop._calls)), (0, 0)) + selections = [] + for summary in self.adapter.spec.runs_root.glob("demo-oc-agg2/*/summary.json"): + data = json.loads(summary.read_text()) + for row in data["base"]: + self.assertFalse(Path(row["run_dir"]).is_absolute()) + self.assertTrue( + runmark.resolve_path(row["run_dir"], self.adapter.spec.runs_root).is_dir() + ) + for selection in data["selections"].values(): + self.assertFalse(Path(selection["chosen_run_dir"]).is_absolute()) + selections.append(selection) + for record in self.adapter.spec.runs_root.rglob("selection.json"): + selection = json.loads(record.read_text()) + self.assertFalse(Path(selection["chosen_run_dir"]).is_absolute()) + for record in self.adapter.spec.runs_root.rglob(runmark.MARK): + data = json.loads(record.read_text()) + if "rollout" in data: + self.assertFalse(Path(data["rollout"]["run_dir"]).is_absolute()) + rows = judge.judge_selections(self.adapter, selections) + self.assertTrue(all(row.graded for row in rows)) + absolute = dict(selections[0]) + absolute["chosen_run_dir"] = str( + runmark.resolve_path(absolute["chosen_run_dir"], self.adapter.spec.runs_root) + ) + self.assertTrue(judge.judge_selections(self.adapter, [absolute])[0].graded) + print(f"RELOCATION: executions=0; proposer_calls=0; judged={len(rows)}; relative_paths=ok") + + def test_reflector_prop_and_distinct_selector_are_threaded(self): + reflector = DemoProposer(self.prop.spec) + selector = DemoProposer(self.prop.spec) + self.sweep(k=2, selector_prop=selector, reflector_prop=reflector) + self.assertTrue(selector._calls) + self.assertTrue(all(call.label.startswith("select.") for call in selector._calls)) + self.assertEqual(len(reflector._calls), 4) + self.assertTrue(all(call.label.startswith("experience.") for call in reflector._calls)) + self.assertFalse(any(call.label.startswith("experience.") for call in self.prop._calls)) + spec = ChannelSpec("codex", "local/model") + self.adapter.spec.channels["selector"] = spec + resolved = schedule._selector_for(self.adapter, self.prop, None) + self.assertEqual(resolved.spec, spec) + + def test_configure_reaches_existing_functions_without_default_capture(self): + configure(phase_width=1, transport_attempts=1, proposer_concurrency=1) + self.assertEqual( + (recursion.PHASE_WIDTH, schedule.ROLLOUT_PARALLEL, recursion.TRANSPORT_ATTEMPTS), + (1, 1, 1), + ) + import harness2.core.proposer as proposer_module + + self.assertTrue(proposer_module._SEM.acquire(blocking=False)) + self.assertFalse(proposer_module._SEM.acquire(blocking=False)) + proposer_module._SEM.release() + widths = [] + real_pool = ThreadPoolExecutor + + def pool(*args, **kwargs): + widths.append(kwargs["max_workers"]) + return real_pool(*args, **kwargs) + + with patch.object(recursion, "ThreadPoolExecutor", side_effect=pool): + self.sweep() + self.assertTrue(widths) + self.assertEqual(set(widths), {1}) + for kwargs in ( + {"phase_width": 0}, + {"transport_attempts": -1}, + {"proposer_concurrency": False}, + ): + with self.assertRaises(ValueError): + configure(**kwargs) + + def test_rollout_cost_failures_cache_and_stage_stop(self): + self.adapter.rollout_cost = Mock(return_value=0.5) + ledger = recursion.Ledger("agg", "ledger-1", 1) + work = self.adapter.spec.runs_root / "work" + rv = recursion._roll( + self.adapter, ledger, "ledger-1", "base1", None, lambda label: label, work + ) + self.assertEqual((rv.cost_usd, events.totals()["spent_usd"]), (0.5, 0.5)) + recursion._roll(self.adapter, ledger, "ledger-1", "base1", None, lambda label: label, work) + self.assertEqual((self.adapter.execution_count, events.totals()["calls"]), (1, 1)) + events.request_stop("requested") + with self.assertRaises(events.Stopped): + recursion._run_stage( + label="inv1", + dim="INV", + job="edit", + fallback={}, + adapter=self.adapter, + prop=self.prop, + ctx_dir=work, + task_id="ledger-1", + rollouts=[], + harness_dir=None, + library_dir=None, + ) + events.start(None) + with patch.object(self.adapter, "execute", side_effect=RuntimeError("failed")): + failed = recursion._roll( + self.adapter, ledger, "ledger-1", "failed", None, lambda label: label, work + ) + self.assertFalse(failed.ok) + self.assertIn("failed", failed.error) + with patch.object(self.adapter, "execute", side_effect=events.Stopped("requested")): + with self.assertRaises(events.Stopped): + recursion._roll( + self.adapter, ledger, "ledger-1", "stopped", None, lambda label: label, work + ) + + def test_batch_bases_reuse_cache_and_propagate_stop(self): + self.adapter.execute_many = Mock( + side_effect=lambda task, harness, ids: [ + self.adapter.execute(task, harness, rid) for rid in ids + ] + ) + pending = [("base1", "first"), ("base2", "second")] + work = self.adapter.spec.runs_root / "batch" + got = schedule._roll_bases("ledger-1", self.adapter, pending, work / "missing") + self.assertEqual(set(got), {"base1", "base2"}) + self.assertEqual(events.totals()["calls"], 2) + schedule._roll_bases("ledger-1", self.adapter, pending, work / "missing") + self.adapter.execute_many.assert_called_once() + self.adapter.execute_many.side_effect = events.Stopped("batch stopped") + with self.assertRaises(events.Stopped): + schedule._roll_bases( + "ledger-1", + self.adapter, + [("base3", "third"), ("base4", "fourth")], + work / "missing", + ) + + def test_sweep_collects_other_domains_before_raising_and_interrupt_requests_stop(self): + completed = threading.Event() + + def domain(name, *args, **kwargs): + if name == "first": + raise events.Stopped("limit") + completed.set() + return {"domain": name} + + with ( + patch.object(schedule, "task_stream", return_value={"first": ["a"], "second": ["b"]}), + patch.object(schedule, "run_domain", side_effect=domain), + ): + with self.assertRaisesRegex(events.Stopped, "limit"): + self.sweep(domain_parallel=2) + self.assertTrue(completed.is_set()) + with ( + patch.object(schedule, "run_domain", return_value={}), + patch.object(schedule, "as_completed", side_effect=KeyboardInterrupt), + ): + with self.assertRaises(KeyboardInterrupt): + self.sweep() + self.assertEqual(events.totals()["stopped"], "interrupted") + + def test_relative_and_legacy_absolute_paths(self): + inside = self.root / "runs/one" + outside = self.root / "elsewhere" + self.assertEqual(runmark.relative_path(inside, self.root / "runs"), "one") + self.assertEqual(runmark.relative_path(outside, self.root / "runs"), str(outside.resolve())) + self.assertEqual(runmark.resolve_path("one", self.root / "runs"), inside) + self.assertEqual(runmark.resolve_path(outside, self.root / "runs"), outside) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_workflow.py b/tests/test_workflow.py index 2b7ebeb1..52b11143 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -20,11 +20,11 @@ from pathlib import Path from unittest.mock import Mock, patch -from harness2 import config from harness2.core import context, judge, recursion, schedule, tree from harness2.core.library import Library from harness2.core.proposer import Proposer, ProposerError, Reply -from harness2.core.spec import resolve_channel_model +from harness2.core.runmark import resolve_path +from harness2.core.spec import ChannelSpec from harness2.demo import BASE_SCRIPT, DemoAdapter, DemoProposer @@ -48,13 +48,8 @@ def run_stream(self, k=2, proposer=None): ) def test_recursion_budget_and_resume(self): - with patch.dict(config.PROPOSER_MODEL_BY_CHANNEL, {"opencode": "", "claude_cli": ""}): - unresolved = resolve_channel_model(None, None) - self.assertEqual(unresolved.model, "") - with self.assertRaises(ValueError): - Proposer(unresolved) - with self.assertRaises(ValueError): - resolve_channel_model("opencode", "") + with self.assertRaises(ValueError): + Proposer(ChannelSpec("opencode", "")) for k in (1, 2): with self.subTest(k=k): result = self.run_stream(k) @@ -74,7 +69,11 @@ def test_recursion_budget_and_resume(self): selection = summary["selections"][mode] expected = "Total: 85.15\n" if task_id == "ledger-1" else "Total: 20.25\n" self.assertEqual( - self.adapter.deliverable_text(Path(selection["chosen_run_dir"])), + self.adapter.deliverable_text( + resolve_path( + selection["chosen_run_dir"], self.adapter.spec.runs_root + ) + ), expected, ) executions, calls = self.adapter.execution_count, len(self.proposer._calls) @@ -101,7 +100,9 @@ def test_recursion_budget_and_resume(self): repaired = self.run_stream() self.assertEqual([row["status"] for row in repaired["tasks"]], ["ok", "skipped"]) chosen = json.loads(summary_path.read_text())["selections"]["agg"] - (Path(chosen["chosen_run_dir"]) / "answer.txt").unlink() + ( + resolve_path(chosen["chosen_run_dir"], self.adapter.spec.runs_root) / "answer.txt" + ).unlink() repaired = self.run_stream() self.assertEqual([row["status"] for row in repaired["tasks"]], ["ok", "skipped"]) @@ -138,7 +139,6 @@ def run_opencode(command, **kwargs): proposer = Proposer(self.proposer.spec) with ( - patch("harness2.core.proposer._opencode_dir", return_value=self.root), patch("harness2.core.proposer.subprocess.run", side_effect=run_opencode), ): for index in range(2): From f6906533555471469f415c40413e060fe240fb9a Mon Sep 17 00:00:00 2001 From: Bae Seongmin <83870598+3ae3ae@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:41:27 +0900 Subject: [PATCH 05/21] feat: Claude Code and command executors, native harness import/export (#5) Adds two executors next to Codex and OpenCode: - claude: runs the Claude Code CLI with an isolated CLAUDE_CONFIG_DIR. Its harness is CLAUDE.md, .claude/skills, .claude/scripts, subagents installed as .claude/agents/.md and hooks merged into .claude/settings.json; stream-json output renders into the shared trajectory grammar. - command: runs any agent through a user-supplied command template; its editable surface is a list of glob patterns. Each executor can now import a harness from a project's native layout, strip it from a workspace, install a candidate into one, and export an adopted harness back, touching only the files its surface manages (and refusing symlinked paths). Codex harness scripts move to .agents/scripts so they never collide with a project's scripts/ directory. Executors resolve their CLI from PATH by default (or a configured binary) instead of pinned downloads, and the bridge serves any provider profile as a Responses endpoint instead of Vertex only. Co-authored-by: Claude Opus 5.5 --- harness2.egg-info/SOURCES.txt | 5 + harness2/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 243 bytes .../__pycache__/providers.cpython-312.pyc | Bin 0 -> 7462 bytes harness2/__pycache__/settings.cpython-312.pyc | Bin 0 -> 23181 bytes harness2/cli/bridge.py | 85 ++- .../core/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 1452 bytes .../core/__pycache__/prompts.cpython-312.pyc | Bin 0 -> 76063 bytes .../core/__pycache__/runmark.cpython-312.pyc | Bin 0 -> 6126 bytes .../core/__pycache__/spec.cpython-312.pyc | Bin 0 -> 11596 bytes .../core/__pycache__/tree.cpython-312.pyc | Bin 0 -> 25937 bytes harness2/core/spec.py | 9 +- harness2/core/tree.py | 41 +- harness2/substrate/__init__.py | 11 +- .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 718 bytes .../__pycache__/_native.cpython-312.pyc | Bin 0 -> 5433 bytes .../__pycache__/base.cpython-312.pyc | Bin 0 -> 3794 bytes .../__pycache__/claude.cpython-312.pyc | Bin 0 -> 17276 bytes .../__pycache__/codex.cpython-312.pyc | Bin 0 -> 25842 bytes .../__pycache__/command.cpython-312.pyc | Bin 0 -> 6789 bytes .../__pycache__/opencode.cpython-312.pyc | Bin 0 -> 19071 bytes harness2/substrate/_native.py | 109 ++++ harness2/substrate/claude.py | 326 ++++++++++ harness2/substrate/codex.py | 133 ++-- harness2/substrate/command.py | 128 ++++ harness2/substrate/opencode.py | 32 +- .../harness2_benchmarks/jobbench.py | 2 + tests/fixtures/claude-stream.jsonl | 7 + tests/test_bridge.py | 139 ++++ tests/test_executors.py | 592 ++++++++++++++++++ 29 files changed, 1486 insertions(+), 133 deletions(-) create mode 100644 harness2/__pycache__/__init__.cpython-312.pyc create mode 100644 harness2/__pycache__/providers.cpython-312.pyc create mode 100644 harness2/__pycache__/settings.cpython-312.pyc create mode 100644 harness2/core/__pycache__/__init__.cpython-312.pyc create mode 100644 harness2/core/__pycache__/prompts.cpython-312.pyc create mode 100644 harness2/core/__pycache__/runmark.cpython-312.pyc create mode 100644 harness2/core/__pycache__/spec.cpython-312.pyc create mode 100644 harness2/core/__pycache__/tree.cpython-312.pyc create mode 100644 harness2/substrate/__pycache__/__init__.cpython-312.pyc create mode 100644 harness2/substrate/__pycache__/_native.cpython-312.pyc create mode 100644 harness2/substrate/__pycache__/base.cpython-312.pyc create mode 100644 harness2/substrate/__pycache__/claude.cpython-312.pyc create mode 100644 harness2/substrate/__pycache__/codex.cpython-312.pyc create mode 100644 harness2/substrate/__pycache__/command.cpython-312.pyc create mode 100644 harness2/substrate/__pycache__/opencode.cpython-312.pyc create mode 100644 harness2/substrate/_native.py create mode 100644 harness2/substrate/claude.py create mode 100644 harness2/substrate/command.py create mode 100644 tests/fixtures/claude-stream.jsonl create mode 100644 tests/test_bridge.py create mode 100644 tests/test_executors.py diff --git a/harness2.egg-info/SOURCES.txt b/harness2.egg-info/SOURCES.txt index 868f2df9..dca95e0f 100644 --- a/harness2.egg-info/SOURCES.txt +++ b/harness2.egg-info/SOURCES.txt @@ -43,10 +43,15 @@ harness2/core/selection.py harness2/core/spec.py harness2/core/tree.py harness2/substrate/__init__.py +harness2/substrate/_native.py harness2/substrate/base.py +harness2/substrate/claude.py harness2/substrate/codex.py +harness2/substrate/command.py harness2/substrate/opencode.py scripts/render_readme_typing.py +tests/test_bridge.py +tests/test_executors.py tests/test_providers.py tests/test_registry.py tests/test_runtime.py diff --git a/harness2/__pycache__/__init__.cpython-312.pyc b/harness2/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..359a090d946dc56a32bc95526ed3e1805ecc111a GIT binary patch literal 243 zcmX@j%ge<81jqhe&8lZ&V0aATzyKqZ@tFh2n9h*G5XDf*sL51n0pxll7UiWD7a!VW zr4W>wTv}9|S(d8cn4X$fq5zf3%u82D%P&$$%v10$NX=6S&o9bJ(eu+}yu}_LpOT*( zAHR~}GsuQtZu+6csYS*5#)+x=dIgEasrmZl`9;|!MX9O9`i5ntMP(I+7W(Cex~UaF zt4i{LN;4qV8|lZ#XXa&=#K-FuRNmsS$<0qG%}KQ@Vgnira(yw#b03%)85uvxF)-Q} Hu>iRME5t@7 literal 0 HcmV?d00001 diff --git a/harness2/__pycache__/providers.cpython-312.pyc b/harness2/__pycache__/providers.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e20f1ffaebe0a5f52283c0e31fa72613226c3960 GIT binary patch literal 7462 zcmb7JeQX=ab>HPK$>owFC0f+C?Ug@8*^=z2=d4sYJbA5&4k9GV+I|o)4B=6qa!Z*!k;RAV4m~p}96GLP;)4Z(YDekUxU{BS!&R z9EzfEW|tHt(@ruHXJ_8Lc{@A%-tWCP`}2|#o`Uj^|M#zBe^o(Izrvkdu^J6}{c~vC zp(H9qNwj2%(&Ltpg~n|r%7hqlWkW1n*{F5g7P65sYt%l@g*a&2Bzx2`&WCttb0MeX z2)QhjY?JuUouARhJuFlgCAmJLq>|4p2IVv3LWN4Ca>@M>8*)oN$pcr9R3VkZwN$E< z%HUciRY_jBdZlV<8(g>DVEi?yjzJWTsnM_|3kejTP$U`Q#;_8VRY8eqaX}78MuqXX zBu9lvRFPxaepQIXW5dcw5`{G-9&7#$UV@+2o#9w4t`VBr3w?(a*20l!SXH6JB~esO zL>28f_)#!-2N4oLD31KB?g+=UQ4~)o5#1I~$g!}Z+a_eB$y2(0NS07Ma?Q`{W$nF# z-TlG7o({3SuSYx^xU82KkS+{%i-WSG@LT5ni05ReP5N}S`E4B_8b9GP0 zx!{FP@%%ZlKhV_^?A1NNzCdq#PyT*`up_o%6i>;qi1|jzg)utx`UlXsL&;Q#2K};J z0IdRzvdFB&oPZJ1VoA#yvXY+7?Ab`qZuab?$C*8j^c=W%gZDdiuFuqgRG?jmg~w${ zkYiFJ4vHX5D%z+JL9zrYpoF8UfaGBr!E%9V=mzjjC@3Bqhf!f7jFj+DRBqN8RYN+9 zCF!hoBO&XQ&JBfCSp;nWeMu)0{s(3v>l!b?ZDMeTnSJ;q}HLPzUgYYpvp*XX${LQ z%?Vgfyk#u#SdNOjgxzwdBl1ZBE=kRUnO2 zerTv~L-m)`f3m?Be92$*WJy(CG1EO8S!FA-mDMw6=eDh~)!A*nnXcJ^Ro0jFmd~7? z?Rd(Tqy0Ha5!2M6Pf-|JKm~HIKTVAljuIAl`E(8*-Ur@kdfGD0q%DX|)5wx$RF7sV z5mu8l^@&AdK4o$(AleMO|n%?qTbVnkLGzEhMvcqD*%o)%coy+Z^?oef53BwNS zRw7nqflq*f8la&&31r}RFGtZKpu7b?wG*ltDqFwj!S?&xm(x-zxzXIxj;8HP*D)13k zv{?f36sN;10t;z+;B`-9B~P0H#hea>Qi=O zw!gF&aWI7HaD}%8;ngF&^*Vi6w~a*OL*b~OLEA7HuK<;rE?t z?I0#1bbvMsFCCqF*VE3M zckh4qarlezAB}(6zq-GBwj)!zd)c)IGqru(mTmAq;ve#ux@Xz7H|z1v@;QN3EB5La z`~2Ah2Ok}Kcr3e9xHq*hwX(By>F7&4TMA~(f*JD^=ODMQUa{9?OZ`i&8Bh4@>%W}( z=c&i%zO4UQ!%rJBt!IIj;li1#qR}Y413*NkCb;VE%W!?qd0(cgdFlG2iH8$QV^2!H zie&;9G8GqB`S&vHdj?G8K@=Xo6n+FIYTgVJ;1H0r5PmllOkzHwrWuI^)yzW$NHq&6 zqh*?vtQIQGr1S9p3B-tml2=rq*a^jwlMEE@6&?qn6uncyJl^Cg0$2rHPLm5;$CgsS z<@(eEE~TW`(EJ6g@>3f+AA|s}y+`Pm)kHj|%4#vyKQ4fao-j0=#2(+txnN-kDbk26 zG&LE%>O{_+oN75iD5t&yh9wC`+yW+M#FI_LaB{ptSQ8Yjc@z5M7Xju>ulGUY4yEON zZ83~bgWvr;kIrLe5>U7!61?6Vdc$T|drw1<=aOS;63Jp%jVOvve=lWwKY8fT+rxfF zXUD*K>Q*T^o&WF~4U(t1)Rv4#R4d*oY_(566)vz}PmPzp{CT+MIG+<#DKn$6b6b_QO3V51@kM zKisL}Cl$Oa2?ct)P);#+@qVj;Y&d>EHMq*hbue@s6I!6sZN@UtY20BFilpKF@g?eCc0DkHhP;Foth8_ThH49xL(d!I@g9gUI|CXn%ee@bfl~l}y?+o7_p1YLc zcg%FW;Jvrsn)m(i#9Z?lMc3CotE*q!{mGg66VPke^K6fQY2V+TU;H-oS~rh*pLr|i z>3g;X+Yg)OcH+4e&njx?+wYxOII~)@KjUivn&0=#S2KV3-pPfNt3H3Gyb)*zcCVG# z9lO^Uht+1_QepUkH;Ld<1yFxa__7O-r*JMz#l>{&mT8NE-EI+VN?XzpLf$1oV{ykS zv8H3j>mfz5=CA<*&f?D8HbJAE+QtlfaXxU`oR-I)%}yHvG6rT936E499_1EOaFvV@ z#V7H_=cRyGa^gD8NG>z{DiS399{Kq0MP`A_5>sZunsQBZZ5GKLqa_b`gHn)EB!o(1 zA5x4Co1LjXZdCK7-OkQ>Hhf;I{;4v>#6P>75|nkGk^{TAXIj3^6- z_2>w$NTg)_Gk}IRou1ZhfCy*h8*l~ySsEnqsp>xsD^03&t;GcOji4!2mK(|^C8!BM zBxD`2#5e&D-5yWibVSvSJV*6&2D}LP-XuaIK+}f1BzGiegK?)2Z4hDa$4u`+wH0bS zxra5O84Uu{HO#mZKzuXWWXrsFs%}@UlciCByLT?$zBoU% z;@h9yzT;kKAq3-&Wv(XcZ+ukou;TI2760*Nes89I5O9(6W?dz>PTxG8t*psbRb!yb z*6zyI3X8ku54?0(eeN~J?&AT*T(&hY#qn#~94>fJhn0U7)$&Q%{2QaK2@0JDOMq?O zS|BE}q-nqkM#A_|1od#dC9!t_^6-rBHt^y^74yFl1lEv3N@U%kRrV)Vans!X6%K82`m~5 zH_W;bS}A8-Z3aum2SY6|N*uM_JoO=LI){7Mwh1Vuc;m=V3@a@rKpB(VCLJOXa!BMc z`#9tw#|wDvDb747%-_oso>WESJc*nnp*+s`?HdVGvGfRG1c2|$;FXOZS#{WUzh!I z`#){}+LJkcX}Rn&*uRk-&6nM)Sg2S$x>9{0Tfgf;&Hb9$({pXht{qvAZ?>c`i+NF5 zzqog?GUM}O@ydeo2c=83yB7nCZ)d6xd`bU|`zg2V>B(?Ce@GI9W6)g5pG`HAU}MBM z4atobT+x7*H@RYFGM0Dx82|8`Jb_&QOXte=z^o1RWR>ax!UbqIu=sSczmI zCe#R{MVzd<$^fYi%vzyYevm|8?g{*K!WSwR%Q$J{#q^96HN&D@+ zDO(#PcbZ_>e2LL9O&iW3*ilVO=vG{-r&IN9LOyHA9Z5n?*F8Ac*D&B0ASUp;4Ofki zU(r!q9fJyo4iIxlM$$k|M~HtX4jt>RZc`(pGCr9eipQh64UR1VPIUev&bG|Y62$(= zc_p06jH^0>&#iGj2KfaBd;o%6N(Qnb*U%^s{~CU37AgP$PuWbj>1*%yW3PGdgM|;4 z+8>>Nc>d}3wk%hA%X8BMG5(Q-Bl8W5`ngt$ycwLoes5x7VtzEUuPam6{nXWyb#9vp{?G3mRB6p`DcVt!EwBDK zu*N{^8x@~#{X;`rCrf?BIy=40S7kKReij{s0aOhYKE~IQD3kiWLmk~IMXHp1S*6P=F~a$EIHFThpOC(qs-~d@4F8V z0BKrDPy5e2iT7RH?cMv`?|%2&?)~ecA_Ir#um0(N-w(EN+~3g;`LRn8{^4Kn95>Dh zTsJ51f})=vP;@JJiYxn--AeXWb*tE0-K}PCO}7SbRljyX*R5k^)&2SbL$`s&HT}i` zQ@3fLsJn>$*7lnREZr6s*Y#TmY~8j2d$*nEyf#rI=mkT6@qnY-fqbK25{hms1hZff ztb*;fQm`Y%Boqq{!E}xjoVT^PloFv7Ic2xCK92sOopM2mu!8-o6spj}>e~jP2Bnq= zF0{2&s1@p1tRCN3c3a==RB+ysx3UoFE>&>*IHBQ1PFU_OdrK)Lcb5yBg%vNVx+{b& zLL=Ul!d77=-c|VBgm<;DO=!luM%XUw5L#YTcP|lk3av=xF&sOhc~o)p%jje22^umSI7LYwde-VH*#uo3U&!d_t$ z-Yc#s8~24a9CBarieAArBo6j@JwaE%8w~pV=K{@?a?$H|`#s)fm)kG6P8>OW$mJD$ z!9eTJs7H-_#_0C@2ZQdQZ_pp;#Bc3k_s|e(BdQa;-XYK6(3M6-MlZO7Zco2E5Xh)| zecpZn`KqJt;CbW~oj858y=Tw<_B{tXI`?IiyZu*w2K3_Y@EJYzz$bV`R13NT7Xld$ z6;yurfH$MMG$>xk@U0v^~6_XYxMSFti7b3Q>F_6Hi38Ev+NXhypk6{Ti0=+3Z! zcFz0Go%iXq7>-u7Qoed%Q1JF=lxKbZjK&9u&u~D<=m*@Fdpzg|O1S;O z^JvcJ$=G&xp4fl<$kC2HJ-d%~^c-wI4JfLDSBAV9-C6qju-K2eAM*8F@LuWh`Y&cQ zgF`?SU#R?iFgO(0vT7AFTcxyCfNa&pH5v1fqwSr$J8~^5#1;%zT#KMl52y_Lg5HdJ z2v`vSbT4~7!@)s-$v4385V3%_9|(#xz3A=DM%8^ZmW&bu6N}K2hrur!5E|j(AJV*z zb6&2S2hCCle7Ew70wXf+8T9x1&b5o;pcvZS;lJqa_X#dQ5{bhi!|JUruiJCpH30l_ zpYyuAAx_r0zZ1yXqunFtBQ!+qC-@r{a*KKEs!Y#Lvn2OR+YQM9hk z@IwJwwJw*~hF3OU)9#~okYg~vcdj~AX@ks zXb~ug*B=a^cc3YNS%79R+a*(BKg1)*IDx+aExD`QM<&~i!=s0<9l6>*YqTV-$73fG zRqNs>-#qjBnR`|1Qr6>%6Hlj%rxU8v40}ANm3wlC^D>ggxnK@$g4ulv@LmKl`aohA zJ_7$%w)SN{P|6py)kkAr@W{w0C}>XQnu3yk0lie68Ozgc`+D{q>D=3~uV?>}!|fq0 zh!3sikR!KZq=eSs;6VQ!wMd`ICAQ*?%$Ib>W_FaXv=OusgZZ}Si! znBqb*0g#+pRp~;^D7?502j`j zo)x0?D?W>CfdH|Z0RrfMQ!#EBGmM+Z%!wtdQr6Y?tOwq0`%cF@9e;T6?uz9411aOd z5BieErxU8DKRanEw8ADWjL10HvHWuA%Pj|FWgh`lGlX~!c@@MSba8cDP%g6Q;t6Wx z>|RAZC#rd%hevT5H7{|Om8UrnEM;i(QMVZIy4(_@){pHJ3`Vo77rQ%=)w6ET1y^to zB-b_Ue{Og%2&OL(6p_^$QeAXycV+mC*uk{4GWvaIGwS|9w-Cr!i0y0hdcYL1jWVM< z=)J;XKs=@BzEp^GMvx@12iQo5y<#u=d>(&+9SE*+b1J2}c-B&KV`OwB_Eg++>zR~g z<;1$AWmQ7A>eE?M@r4dTMA2?oJoVM6sHq0pzdlaBG@j^v4_l?S}%0K3} zDm_GnJd#?C2GT}SFzT@>Q}jMDE)YkX!Dz0 zSQDfvW{a$I=6heitCaef6!`ip9v})-F7A$^QzS91QIk;v0R{pYgD>FoV|&0xn^6fqPf$FG z{EVgt1n!(yr0+E9kOK}qqhS*VVmK)J&Y>3AIu^-jhQM<9E@zbB_cCf$Dx(nw!CVS# zmvUtvwjGN7Xp;p)hy<=60L3Y(8t)(L|GIzF5K%@>e%R0)-5cE$TNCSwwI^-0lXVfz ztkD_u-aavHteq{cjtQ@wzjZzycy-|I6Vt_O(^WN5RkUkNKWnl_*TDC-qy;u_iZiO7q2ZwTWuz0~3p|AY zb8NEQ=osyYdaiX6V_}XAU*9roE>Bdgo-Cf!CN_2?Dh{N~2NSx3=_32py|dP`nDgtF zh$?L_iD{xWqnbG-XLEh5GDOx#OJb$*hWo0<&yrkPGi(C408B4wU(!JnL0SlIfFI#` z2vIL7`4t2n`eezgB<^dndz z_o!EKQSQ?-m{tDzn#cCZI|eY+`D@sFRdO7?`;v+q z)ru{_O8N7GbI}!*TOobba>@LAR4cc*@cjZ0!dlWP(jpa!wU{SR67>Zl^AG<9k*_HV zMVv5~7jon!B(5W_?jk4JMiq-qC1l_%o6m?(ZdX9#*{6W`K>0etgJdbgInfB-+K=FC=~V&51jRiOf42Npi`16wN+e2wVbV_&dXKJ4kE)F z8ZDxQ+NO<2^dZRT+S>Q-K6&Uwm-qtxRP_z|{P_uI>?CV5qeS*FgXTBcNgs((-a7b{Ym4kk7p`ruH) zdio!giX!tT8qQu7YkyJf^u$~dS5`k~=8TpbD@IpDH^gdF zhU%+(X0^6x-Ob>%w(7^lUySTYJ4(m*jqQu=dF4=~eb!POZNK)VH|mm>WfRWHrEgVa zQ8`{855&ERro9Q*zWdhwb6U>f<{#=f$Ewe1Vcpy6cyrRyI_aHy?hpGGw9=7q9k_2j zNUf}z({c8a@v^b9SW{d~*;ZaXKpk9pTbS0?r7M@sR5m3mn|@T;GI|ia^)lkhy>EaU zUnw`aa^l3q-o)A`6Rth?t!-&r$$0r#dF+Xlt#+buX62K~l~3NYJt;595?9<2*CZOY zPPP4M$9o-jeKUKXP40a*wXG+y^@|CI8*s&P)pJ#?56qm-3dS*3|61d%#y8sDJpB6M z$rY*kt*MG_Df9M(ZaY?}&ML8nnDpEOwvZnFVH9Cr;d>GcBFs_>uS!sos00&^T2Q^6 zW4vIR(jbqCQJ=}v3OeNJsh(HY%fpytfWbx&1CtR(6a$%=@HPrgp$Kmi-e$augc8Am zw;3iUE8dnXw#KrIt}AO?2|aOKQsqGe&n7WKORCYp;IIfv3zEWA&x}k9AjlA@TOS$8 zYT^pJIT2ve|hN3ER$EY+I_lNCe(hBe#;m?LubirjYVsKTA#ikAyyU)P#mCE9CcNM3iSCU(@zu@;Ms7(Ho8mh>V;LHGwg4NI z5+V@;C{niq1Vop-VXLTXp`v*`B}O~oLV#U6Z7Ysw{CF-0M&D@Pw5~jD zaYl+Bvnc9O{qv?gzbHT}UPD22l@MRXdE!6{ah`Fe#N>snxqfEZz6=$m^5`kB{qpGL znCjY>6T0fpc8y3NTd&)Ih3FbD@F0JdJRiTP=7?LA*yeKR=c2MY71Lfv8cK-ho_KG;H1IIx<04uqW#MQIKM;%{fLUSMtueT3x*Jg zH_}vX+zquy;e!VIj1DR4Fw`btHTClq#dU)U=_BecgglE~;u?)Gt%}gQa3WSLCvK)- z3xZIQ%#@3R{a#n7S>niLlQA~GtjV|;hT5#jm^gH1G?p4UhzFF!6L^gF8`kHf8rDOrEsOIbJnZ73)jc z8Y3E%jc&bXtRY7C%Gi~7MXIzpvOjGpjp>t?+W5J9mX@@&WV~psD0Y6@x}1t{xo50S zo9tv01kcrct2rJ_RWwhVTGE#NP|xK`SfdA$rkY6Ghs~=~mNj?U;$5H~PsR3(eIaRe zO|&H~YZAIOzqn^v6QJS!nZD@Ba@9N%4e?Z35+5T{ptr+Re-1L|P?Pu&#y{Bd1Xcio zLEok(!kyP2yub^Z5tV`i0t3AkK4HF>hXx+45yCB|Rmg7;H`F&Y9{Ab-QGvQSn*osd zB5AOUKwJ$_)&i7Oj%Y^IZ}7j3eTgA1`wi697HAnHQOqXetSpkbLM2;sUI$+0cEg-r zS|KTttR}9eU=0PQ5o9!?_Z(;wa4avV5ejBytfN^;k8Lktit>oaq^&X51oOfPNP_4W zk&@p~WScgL=}ck_*>W^cPWpSQL1H+BEZvfioaN)sj6D-?N;z922Evw5(zq0unsBX| zte-4PY(A1Wdg6nNiJGTU_EQPdskE&FXlhv+xiVu}N+P?-`ar3$8qy7`CmqR#CuSP9 zB^$O)H|&^pbS4{i+!A8$xb5!JyBGhgDS9l%kDW*~?6~LXWOe2^h2HRsA3HjkiRgWY zX?L^v{iSMz^AL~dB%{Ssh|JgMEJklXOV1Xgv@)WACUbp+@e3XpwH?2%Q9I0x+K)p= zdF#?Jqcz*VG6pyuz#9oB08zpXrRS0$9z;LnU6|N8(&EFbhHb;y=@s8adI7zm0zXGm zU>5?YRUnJDjqe}ZAM1MMaLQ6UV`)iRT2hu(pav4Lxo2y7!|`U>>t)}nmt-z5 zl9p{L%Z`L@2ippXWV4SeA%PIi28TtzNCe=Hlp%f_aq*8RpcwXIiiZkB7r26*K`XS| z78Jffg%_rV>@sc!TIB_h?La|QA_t9DaR)UBl{P2tqF^_IjHRciz?ZG3M{SXFvphtxHz;6>jtI9nNdbX$f#q4eNof?&*aAlo{3iEP)vk|<%2KMb5ACI~lhgLP ztL>5XDOKHvOX??Y*zuKlOIF_0DX*GyJa&HWD-~h&Vpv}w=;e$zdEO$dhS2#W zcn6IhCeuEmgA1NQ`F=zTyF5{Q_VmI0u<(JwH7hBM=nF>^)`wL=+VRgF4K{`#=?vKuHUz<2Nl#8D z!Il^{fX}oAOXWPl$hJw-!fmp!C22eSYFYtxX3O31K3>&2OyQ0XO_^+P_4k`K` zKVr-RV?c^nXdbc5|5n%x^@^pS&2lJ9NT14{yg3g0(e_*_Q?AHv%ghUfXf1o-WuoG_ z(_ECl{_UV$PR$$hm>){%h^_!TG+_;}LmSo=GGk%g3lQr^bYIq8;$>eIc)OIzE%sH( zvHTm~p5L>`8YxA@59VLJ#!4YvSuz!ujdt8BAp zuG45TW8J^|cxQW8*V-QF$M<#|Y8PKZYkA{BmVmoL&IK$X8%1k~T%x6@I7$_XOQYYe zj6TWZWL5;d(!N9(k1RoYP!Q$&_uZJQ<6hMQ%vjq#3YRqJd~ z*|jqf_+X~BW~#3%x?vhM-Fzm#Jmp+Dt!w(R)frLEDb)Jie7dA!+&|`@I2ZM&N}h=9 zn=>f&?fk5>YW&RDnfS((vnjF{CJDVI5`;W<>J86C{kMGgb*tw}Ia|qG8CS71QQkay z?%mq&tbAwXcUs?RP3-PUtvzvd--lXry0~2~*pw&KwEed^&bMekobAD&<@~xqRyByXWqnPHa6gtvfpFEFJG2>z;8gOFEbR z$hmw}54(&9V#JM|qdQ|;?;Dp-v`x1C-XVC2%-HLa_PY4NlzsJ#{fVUgiIjcQjD1(q zzU!{zZXjhp7}+yxvPK=z!0m$*%2ZYJ#JMTQlrZ(wy{t6 z)+BXR5#?x4#2bBTR%eQwh^~vNW0xj0_jRl0l-$y_AFGNY!iz^{jhM{X34qzylu$K& z`pFv3x)dI2dS}Mun6;MPu1Z)M==W}(#&05{$Yky0vB^D&noTMD=Beirrd_{;BO40* z@)I3rEuFD6B&8~tM3R#r_e^Rg&raEtut81n?Zq9-)!(z)J8IS6t5qYOWB<@X-sSr?Q%L0% zQHz?jm*=8B>u1`UwT2W9CC?VuT~dqZ|WDJqsY6q%Nx2_!~A`6?4{fW zDcu&Y5ke6`Pa(cvDVG-va*HqUP?~hDC3WJK*v8GW_!)%`?NRS8we7%%p;6 z;UAJzlXHJ}f;Z+1CYb-M--3zDI=z>YPMVzFCDo`xBGbQt=I$uO{dkFAL{O+76%QaG zRN0gDG0j`(b8hadfnfm}o8;!+(D^;}-8FA-nRI{Gk!sp8y?n>_)W2Y6{}(E&`3(OB zF@m!5#svvQa_e7|_fI|XsoNY8qp#L@+hM7G8Z*;XVJTMdxzt2*r5s`G|l3M7uDE8YF&|HaED7a1m zF?di>2Jk9y&K1*?M?oWv>syFG(1*Qu+3n(KZ8b#x8==uqY}tf&+Ol2}=ck^!dpcz~ zlF%KI$~mUBHM16bbi=jKjAcpEvLwaqJhNJ3r2RL(bTe?fJ=&kt)@F+&bmjSKZ>&oa z5~LbBr|UZ5*&97EetPV5l3<#u*^+W@Nm{l{!EUrGq1!cUa3)HgNg2*0RA(f$Iy#(t z2<%wP(s_?URnB6@E^^Jut!Aicu;TKKPl^RWKIg@MIi;$zVhwkSC(DU@_ELX}i>21#-n(uhO~hsg5IWD6agqol)2 z88thKkl}q9CEUgqnBK9rn8V1*KJB$ZP8<6nkWya6f)mEKl*%ZIPRXdtU%7q zu9)+c?lDW0pDiwl4!^P`Zu{!abVcQDPrUwB-|w`54Yl3bH#t0Y?48T^ns=tl%45fF zX<|Hh#mcJ13Z)_D#9p zIe6DLUBBnnb064}j)Rf*w5jN3arERX6{ANYyWwFZH=um^Sun!mjbn|mw)oPNy&gV5 zOKRh1f9J~0x~Oooal(TH$%zRCZ1HumgELjF$*R_gV5(~Uv}r@yT8svD`pB_SO@yCo z;*2H>Jf=B6_SIT-tY}*P+zJbTjpYx^}I_6bb`71dELXW@|w>ZcEY*k zHKKxsZ*eqN3F@~sGI`p-?1+nL5@m*Q1a8^%Pp-k!X`Nyq6?^(}%%7aXanI*0saQao zzQtHP z5tCpWDT3~(TJA?!CokhHxqmlYB-n|495K`J@5hfpQ1|8ZmvW0Bak_9(*qrZ={Dlz< zpp880Ti?=rc{wJ-vae1~$-f1s+=_S;TbDI#8LcQpT1ht7 z#xQ1{jdm(*h9=Vxwg7+r;9ouifoL$N<@{%2K(YLFL0d5DEKbM}`)p3%W3WZ86@u7T z!Hqb=j#s%LWESZO=L8>U4@>AU);Clm&Tz3<5_Sru;o`-#t}-6dDqtJ1Gwd9#kYeY! zNCo`#%3joki}T{zH~Fx0jF<4flJma|ct2*9iL?#A`?x-iRIWp*8qMjOWoUu{G|OKK zs7EWpC5%#(;G3KCW#m}c6fTihBQDeU99ki_H&XVis7<&`E=$xkT-KMm%FTf(I9U7~ihW_Z2mFFDQba;qv_VuW0MIH4uO(%}c00c~YE zb@3u*PP|0HcPStp6inBDf>%cC_g5GqY7I3 zS;dHPo?=8gIId*coeb|0AEIE^Kj5n5^LhX&p{f9`36VUHKYNqGOFZfjK7T6dc$T%j zw`tNHcTX(;w-?^qBx!ek!(!*>b(w^B*5!YZR)@*ifI3r7jZohSg0gGFukQM9QP=Z| zo$6ihEt{;r?VC`&Hu&DMUx*)~F)@Q6WXzg$00&p7F{|UrDs?0!51AH#Jf!&X39hVF z1YJ9f>U50>R6IR{7aEsJZpl}uWm3;DT|!1hH>8mJHmnN&nNmNX;CmFfDL6}k2SLWb zPINM)#9cVUL;YUyXOv0$o{XlS30F+$DzLE}qC6EH1D8BaU!V#kMKSPYltY6P>VXS5H=i+H646r6*(*VBps`Z&br7ko00mz{kkMw< zff&LUPhc=FK__W4p44 zN^*EFmY{P_5xNWkbw8!6Gk)DOuCrzck=F*V?weC+)Y~Dy-8emZIIB>LK&ZJbsf}pe9C>kw_3e&nuoLxdK97lRma6QFTH+gVtDf8)Vjo~ovG!!Qsuj&nzYq9 zXW&e>sB?7NjIkzZtVtPNGsdQ*v1ww@v~e}8TM$=^%i;8nBQYo5SU+uF@pjjw2K(phfF;ncsw6*pVHHWh$^Q*q~EDT!9 z*GxLz4Nf;~yXlR!-|W9zAK8biE27?e#>%v_EV7s6{EMSsj(aECU+hFR zPo0|TNYw0&8b2(qOgomu*G-iDhpIbOZ`CC0HYOb#qX*O0%Gjl(wIOOu*Bwt0c|?$? zJAU^9f~Y#`jqOQTU4R3LF3iFBo^ccq-W01Vvfa%l8WY+r11e0RJz{`ADLiAxi{DZ^wqQQigv z*iS9am^Z$6;@Im4V2GQk*_5o=H07AC*^#cUkB5@gYoZ3Zjgol9w5d5=WVvx_^i=fZ zwP&yHol_{)`*Qf%JFTmp(V|q z?yz@Sl|Qhm5&uD1@ezgc2Tk_Q?aCi)S0nyI^`@ho@`pQlgb8)=(Y4A%ne%9iGSQ-@ z_y+wki!!mpbj+Yk8q|o-+s=u#0~f+SjAH|yfBi|`%C?whnb{6OR_P&LWicJ6EabI8 z$9;wjKMLry!P_i+d9nNs#gTGZmARz{mJrEd_ zB=}uX$|6gnIdt?895`IK&u;ruyAPdg&vNH4_ul95>cpMqICXF%4EwVyfpt6^$k2fk zFC)0ifamdfEKaZ{(E9{>QGbS z=&Sd<7noWabHC=h<$LwQy_yXZ$0qCUJe{i9@LnKmy!ZqEbjk7j4myMNkHOS88iD&l&Ys2M{wGC&@WjuFH@!eNkJC{eN z9B$pjsf492W!V5f91Ul6kiB5W;sWbrvER5ddga=eBB~F;f&|B(AA3F?m@aLa2u#)A zd7kw3S^t&I)5c}+u(_>_b-Zd$l&webGv6}D1F5p6(IQ-o za}3r3v+ag&)OYPdMERk`e!DJ7(~zp(GhIQ7ePWx8W5!pCB5HQS%C#*q+siv)#5i{A z^2CNo`<>1AmaLz0PCfTt+1>8s<|A~#aB1wxD2^DC3E|}E8KjIK96K0S#oZ~#iV2t- znr0kpla941$9kMX)){UXMh!E%>ZGnZ=813kk*@K<5;(kk3MS1|OuLokL!Fs6V9TR` z4FG(hvYqGtlGnEvDgV;MBZS{Ip4r+Nj;|#F|V@_{MSVJK<*zSHB_Cw164YH%Vf1Lz>{4%iF-x@!c7#!2l8EE zA7I`4j+DjlIX9q?o_qzU#WT3Phao(J-D0E4_f=I;GN0NgIlSY3%HCo45}rav*dx(XwuA8a7$KBiirbflFny8LZfBt(v?jQw5npm z1GUz!dC& HY>@vCmPIpc literal 0 HcmV?d00001 diff --git a/harness2/cli/bridge.py b/harness2/cli/bridge.py index 3e340bf8..79124870 100644 --- a/harness2/cli/bridge.py +++ b/harness2/cli/bridge.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Serve a Responses-API endpoint on Vertex AI for the Codex executor.""" +"""Serve a Responses-API endpoint for a configured provider/model.""" from __future__ import annotations @@ -24,19 +24,8 @@ import sys from pathlib import Path -from harness2 import config - -LITELLM_CONFIG = """model_list: - - model_name: {model} - litellm_params: - model: vertex_ai/{model} - vertex_project: os.environ/GOOGLE_CLOUD_PROJECT - vertex_location: os.environ/GOOGLE_CLOUD_LOCATION -litellm_settings: - drop_params: true -general_settings: - master_key: os.environ/HARNESS2_BRIDGE_KEY -""" +from harness2 import providers, settings +from harness2.settings import ConfigError def _free_port(host: str) -> int: @@ -46,7 +35,7 @@ def _free_port(host: str) -> int: def _endpoint(path: Path, host: str, port: int) -> dict: - """Reuse the saved port and key, so the Codex config written by setup stays valid.""" + """Reuse the saved port and key across bridge invocations.""" try: saved = json.loads(path.read_text(encoding="utf-8")) except (OSError, ValueError): @@ -63,43 +52,51 @@ def _endpoint(path: Path, host: str, port: int) -> dict: def main(argv=None): p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--model", default=config.SOLVER_MODEL_CX, help="Vertex model ID to serve") - p.add_argument( - "--host", - default="127.0.0.1", - help="interface to bind; use 0.0.0.0 so WorkBuddy containers can reach it", - ) - p.add_argument("--port", type=int, default=0, help="default: the saved port, else a free one") + p.add_argument("--model", required=True) + p.add_argument("--host", default="127.0.0.1") + p.add_argument("--port", type=int, default=0) args = p.parse_args(argv) litellm = Path(sys.executable).with_name("litellm") if not litellm.is_file(): - p.exit(1, "LiteLLM is not installed; run: uv pip install -e '.[render,bridge]'\n") - if not config.VERTEX_PROJECT: - p.exit(1, "Set GOOGLE_CLOUD_PROJECT in .env\n") - - root = config.DEPENDENCIES / "codex" + p.exit(1, "LiteLLM is not installed; run: uv pip install -e '.[bridge]'\n") + try: + config = settings.load(settings.find_project(Path.cwd())) + provider_name, model = providers.split_model(args.model) + provider = config.provider(provider_name) + params = providers.litellm_params(provider, model) + except ConfigError as exc: + p.exit(1, f"{exc}\n") + + root = settings.cache_dir() / "bridge" endpoint = _endpoint(root / "bridge.json", args.host, args.port) port = endpoint["port"] litellm_config = root / "bridge.yaml" - litellm_config.write_text(LITELLM_CONFIG.format(model=args.model), encoding="utf-8") + # JSON is valid YAML and safely preserves arbitrary model and endpoint strings. + litellm_config.write_text( + json.dumps( + { + "model_list": [{"model_name": model, "litellm_params": params}], + "litellm_settings": {"drop_params": True}, + "general_settings": {"master_key": "os.environ/HARNESS2_BRIDGE_KEY"}, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) - settings = { - "HARNESS2_CODEX_BASE_URL": f"http://127.0.0.1:{port}/v1", - "HARNESS2_CODEX_API_KEY": endpoint["key"], - "HARNESS2_CODEX_MODEL": args.model, - } - if args.host != "127.0.0.1": - settings["HARNESS2_CODEX_CONTAINER_URL"] = f"http://host.docker.internal:{port}/v1" - print("Add these to .env, then run harness2-setup --substrate cx:") - print("\n".join(f"{name}={value}" for name, value in settings.items()), flush=True) - - env = { - **os.environ, - "HARNESS2_BRIDGE_KEY": endpoint["key"], - "GOOGLE_CLOUD_PROJECT": config.VERTEX_PROJECT, - "GOOGLE_CLOUD_LOCATION": config.VERTEX_REGION, - } + print("Provider configuration:") + print( + "[providers.bridge]\n" + 'type = "openai"\n' + f'base_url = "http://127.0.0.1:{port}/v1"\n' + 'api_key_env = "HARNESS2_BRIDGE_KEY"', + ) + print("\nEnvironment:") + print(f"HARNESS2_BRIDGE_KEY={endpoint['key']}", flush=True) + + env = {**os.environ, "HARNESS2_BRIDGE_KEY": endpoint["key"]} command = [str(litellm), "--config", str(litellm_config), "--host", args.host] os.execve(str(litellm), [*command, "--port", str(port)], env) diff --git a/harness2/core/__pycache__/__init__.cpython-312.pyc b/harness2/core/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4117a5dd85b7210f190225498148dc68d5d5a771 GIT binary patch literal 1452 zcmZ8hO-y4&9G~fXZC_iyg2r@RAS1A@jnE2fvTU+37Pi7hC`gMRMDz0K8|Z_+kIb6~ zl)M@+9xz!CUR)E`ghXSqSq~mOn0WE-rDTaV|MB~O zz27=JlL*!i`1kdb7($Qbp}o{I2A^bsxrDJ5hx$G-OvB>_w_Jy~o0JiQRK~K|$^~iP?p8sa__RhV(#yyxRD*c%V!F0rIfQP*!2JbbNe=@b30Y82=iX z(QhJcolmd|NcOryw2ebm*DmCu`URp8@5KSDs-HHQln5hxgJC38^a~XHQd@KV#WweB z4=?9%P>LIFXMeAlQ2w7{6lL~3CsUn=PD-M&?x5j~6Zv$OL{Wql#Pn4g_3%!=Of zV!l{fm|raG`Et21w@@yL?z#N?`qX@JYH4w?P@MWeBzTok!-Q}|&!ii`Aekj<8(tM6 z6{=n;%w-Zx_7Q`LMl3GL|3JnzB_u%U7&aBrvy&_vX|I3Z&MiTo`nhWc&1F4kYd0s8 zC&$gtjco>Ob9-zew>36Ow_)zM3!>*!JD#fmlANwvj>UC7>(xa9+I`hp3m9en;6)bN z{~pLL`b*6ns=4DBN)NRLUTcxppT^a$*zQ}$Nt7PA8-5Xf`SE9|eRZFFvd>@RIL8pf$-2{)C1rL27ff0?=1a;dhu<1YU4j+RFm5P literal 0 HcmV?d00001 diff --git a/harness2/core/__pycache__/prompts.cpython-312.pyc b/harness2/core/__pycache__/prompts.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b587b845558d281ab67abc2cf28199b39f198eba GIT binary patch literal 76063 zcmeFaYjB)bdgs@T8-WIR0ZEC}g;I-@L}ElVb(xW7D2Xx%G)YJ#zyUx};*1S6&<(Ij z+}zzDMGQDgUv@3|Hj2GC$*JwDya%+${`n}R82OM)H)wh zP^46C^dXhs|2gNp{WjR7sF6JO?g}#k8*jhwdC%oJ&+~t-=YLqgK9%6-zx)6G>P*w4 z$;AJrKf+J5`;h$T50Z(*n~8klR3e|uH%upI8csDN^|^7n@l+%KHcdCpG@ojoX*t!x z_2zub^tzeWQ?2%S-E`Yb>Qsu)t*6@aU(dJwdefl`HoW?^XpG-%5ON; zncsM7bAHpQE&0wdyQ(@>{uo8~@(HzuWV7p1Lc)?bO}*{rRW!+kd_J z)IIsTPTiZo`_zv7J*V!=-+Su*{Ek!U{C%e$$lrfzXFh#uSN?%h59W8CdZ-~$XwL8Y z;e+oo7X36N4kz*t{dyw*@VgD}%De8L#Hok#&*XRWw%z$Xr}pHZ&3Ey+i|4zdJHC=X zkpBu-AIU$Le}sSc=6@l-mw&tSJ^60_%`gYe`Nn+a`;BdhoBc0;Fu(8DTTbn>dmsIN zL%uiv7=J&S@5?_<8;>#OC;0dA{8vvsk^kDMujaF~{v>1m8s9v{H&0$@>^hV?!+^5q z3Z)C_`O@6Xd?j5uT}aOqDyQf2>2hHYm0nwkv(|NY6<6<-rr_-kyPU?~$H?!&zRkP@2q56jI~U za}#G7`gCEUQpl&rU5|#{x5M*=iQVP&Y;LBINe|40IXhioXu8NC)1`&+QgMR!rOOj@ zT%V&e=`#!YsX`|8*CLXx&jy4bgnd8D3>4aer&>1rjeocUSlGW{Ah>|Zzd|ycV|4c+FkyacI8sy zUGCS<8NYAcU+#CH+6^(70p;~HeG<0 zSdMuOk=51`W)@q%J^P&}_Fnv?A(dK5rn(we8cKzgMiz6WndL0bGrT6zSXuo|fAajk z6J-{6-xImQzRY~CT$tN;ey()3QYsY6`yM;DP&)U@V_)BQ{;}@DD}{-L${e?b`Piqa z-{(lWoSDC{(mYxz<}Z{wY4`#DEAQghrNmF$*MINH*PeX6>#Y|rk6dm4%C+`Kue3jU zwf*t;My|AfW4Y-YA8}hE_dE0_MmBwe>4CwK^pTzyvqtBlYlvblTKcI{E-%uXC>5d3 zVh&=MEuV)f&lWE1O_vJLF=WsFn44bE1c^o*vE*hioHuGJ&ZaNSEtFCda}f9|6}6D( z6_wL!moJNzboun$!ZZ|GNHZhsg}g|7ZWhYyo71iQc{V*s(~$kl-gMqwtgzK9b7@Uv zYHp#D-g*9X&h2{jnm(S-%~r}g{R?elpVu%T;`!4h79l-dJX=Wb)L!yW6kcIl@C^N; zWrkqAOwoUBko>{~U3y*{!0m7dBzk9YmW5xKu#FJ)S3`~8#7;lM3)s$VFh+eoll$E- zSLEltBGHfmHvU@k)ad4 zBmIK|!>QDuQI%Y3p3+|e4f3@p4C1?Y}lT1EK6%x zhD*#>_NL~i7pAz0*8Q8D>I$BaqVndFm5<+Y$89G zvXu0^P!|9%(Y`SM)aNy-*gC~a5g9GW~Cx(s>4m(zd z0C$2sR~j!e!!Q#~WBK&L$gqzcJZC<^j-<~_E8Mwow zrU(vW!epV$tclB<7&tcA*MF!#+ZPtey;G0aO}Tt-UhEIv2N+R*L`i}h;)0bqnB@7> z#fj76((XfdclvmL?+g6{hZ$;m@WjaR6C>$ELxackXY74PdWHtF!^0xxo^ zzZ|CvCQNwo1h+CxHmzlDnD(ej(n#=#huInnd~rehn+=}HJ!Sj)?*fqkxT^b=LCyn2 zVG&c3d`u;@5>0$;PV3F((*V3rnaeL^?tbfUohJg8ONCzsr2`?fnbtCIKXK7kjPb|% zd@yh)@xjy=&j(}tFIAWS1^)drFu+W1a(fMstKxsBuXq`t4U`k8GL{54o@e_3iZbbJ z4lZ&U&XtQcPDR|RQiK;x%$3U-N-d@5bCuKSrHCu@4*kK_Wbg9_%-M^$ z^ipc6ySqF3k7viq7s?eNwP{3|nfw^fmgc4rAH0sM6NUT&w7-<@IeN0^W#ihgIFY!( z-dSFvQ4}g{^7BQyAxc0so2{VC%$<*T^0+`+Mek3KO)ca~`BJVpUH0!onL}2>oJtE4 z%WK|MM*DFyU{4>No1GddlxB`)j|~pJ>|3CZxp6w3pPNCTrkxbNbvu>0x#`h-!O2Kz zOZ)7~)B-{qp5+^sWPzU|3~Xe&<Kj9YD=d_)qXNm3 z(=@&WzXcrUB@X+t$6)+>9hR+Gz;n!W-pw95&8SUq^opLt*@2Ph6?yc_DZ}jCX*X&` zr9&|>MZn46$$>x*(ikcWCB8HL+ve9zNN5}dM-&0wvBagB;%u>ulAS(Xn3gJR4U6mN z3j#3dbH!YGJXeMowayN9_=(6=P`A~P2P4tq3}p}Z507MrvJBivTmTAG2iS|ITrz2% zMosmvoYwYn5=D|MS2ndnB0uFZM(ql2&*gwXil-zbow(n?JIUACV_Ki>cGw+kSL@o2{}EzKgHKG3+#8aFZDCvJH9-hyHM_yjE&SbG@`H; zJcjWD>a>ReQ0EHMZc7#`7){1vbaS(5nVHKM_R}{3%wok{oDMn4nTl zX{S#zANGi5Q(Z}ybLgbbf}?A;HQ_|?jXIyL&I7+bsyDr7-Fo>Gd zxq0KDFetYD%zo$!U<#aPSr*2-h1cPLbL`sw0Rb?$;B>Bpny4YUFM0-sPiBYsCS`EN zMrPc39PrEZ0cUIrs^=b9>TB#uMNpI=4Z8U(cvF+>N!=#a9*`aqv3F0bB5d}cz84>H zL=6TLxj@eBPWPqb#Wp!9eOX}mj{TS!3n~yxD$H`(G=16B#68upmT+2FM);InFrP^u z#~P*unH5EqD@MFXZnGmi&BUD+-g8edlwLX#)M1m6(4nmO~<>VP$ za<8pgp~ARQw;|6Mx+u!cJbPAxnw6hKxJhbiub!s=;`SPcOnADKt7rkxr=){ciqqDB z7m?gH#rUjUC_y$3n*c3wCf$=aLCF`a=}hh{-x()F7G}+SGOaB)S4kmAK@~j+;Ms1^ zovZYyZbps9x-i3v&1y%hv&C7HI!rVha^6f!dmV}LC39IQ?N6oCOhDRXoM{=sLgN^Z zA>nb6nXq5k6|uCp-axKNuzq&Zfs^)!(QiRZh4 zLnxrOAXDrr6EY?b(C=k_JU@LQYW>7Plg7UNz>}CtU~&<$EJpA6kMR!mel27220gxU zjLCQW_|X%G`v)9Pip|GfOzPTiX8DFJ#0~=^sd5<_8!u?*0WoLu8Vy4ARQhEW+@t1y)j=nuRarO;^aZwW>7VH_j8=80{Mw!ywd1qpE_R*)U{Tw3SVO02`B#Ep7Y)NGHq^1?_ zS}2uxL78dEfvJ4I&lU=_8By=k%-{UHdZb$`1~?Gb0b26IadY3 z5;!8>3s*1}22>O>(NxNv1HyxwJ%&JdF+23KXIlf=Y@ci}BSZa%4+98}_77ymqC}T7 z(z_5j(}t&Qrfl2D5dn*1d*Hl62FB+G`+$so3|VWBepO!`fR)06tDsX*&p|gWW3H5@ ziwAo~`j4@hHRU(zX}n>LD&^U8AwAr4ESr|;2P`b&;^(CF!Dlq6h=DRhF&XvGvxV}3 zy#{4%@MBJc!GEPYm^fNNo-E8dRO+6i0SEA4p0jKTR)@)E=kcOxh)E32U_`Xv3$LI; z(AwkJD+kjfSu9@|2h#_KdU~M>I9x61ipxvZ30)#U*;hC?kUiSLtnR|p>Gbo(*)v*> z%4v0Rm`y)XIl(p?OiINqO!sNuS5R#H-5eTDF%pY0V3S$cOnvM1)Qr z$!9pARXcWKcmxQ8N`&-g)y;&9Hm30(Fm24sW$86Dydo%=qBvbrKWn$9BdhZ1;(QPc zrc%RF=jY01%5zKXHFmGtfl+8FB41nXZY}+ReK&SW9#&K+uNfgP7`;ccj!vqC4nYha zI;0)1_OpHcBRvP%_roWK4nYCNqaw{JUo0aHK=q;m!70?eS0)PP`V(aZ6Su9#(1`)I z3oT3w++I-Y4DlShG=kk)q4GS-%L)i8_GOQP6Nl7+-NWhOBR$6<#O%?*fy3}8h?84} zvfn(B9UgJc2ClJCT9{zZaZtCMJ~DXHI$;WT%HH#|n2;%BZijeOY%XH=D!vD;qX(8C z7&fR04Y;PX7tM4TsxM6Eu{=-U&YPL#L-}-}P$}VS+ zLx}>sw*O=lr(MV%Zmbbh(V#u${3O~XZV0gM984Yu98TCA9x~}Oz7r+DjCh=^0vJji zs)^x+U`UgB!T1_1nk7L@^6nJQXoMIrm9Bf&IJ6=l=46`p{R?jh=bS4R}yssC47mb(?&lp09eS)(2l*IR~E6kl_aw9m`amyie>Pz^BkL^XmguM zANIx|H?aY37g9DwY!_Mu#7i=tE0&8gKuO`Bor(x9kcpXBc)8}cs!j#v*zc4uZJAV$ zn*(>dsDb|=1{w&K0Z%t9XODyzUV{@^@iy+6S(uhvIL_ION>a|CfzzlPb*+1078a-5 zX#;J2f~O3fP8Wbz4paL{gA`z;Ma(+tzG$26v&%@H4RNgJ1-v>w*x?3aCqoHGa{WN5oO`IT;9z=}TYmDDR_);>3PYl)K$zt<5F9DFc?{=uLj`59w``oQSDt8?HYJiLh!7cRfc{U7-7ma9I^iljrU=`ENK`87(z7Uprac$%*QRa79O)u`h{%v_~^kNe1B;P z3cr|Gk;UnGyD@j@Z<=-$-(X6u=-X-5kh(l`Rc-CYo|PBowb5zcX(mD2R}aP)*!q4+ zBnpI$+&Y-fKL`Q@{mjvw17k#?C~QZMg+&y$-EZ4bCV0Pg;=V$RPJ;$?rm7p=&FIWr zMa+@~0}Pu$vsZXfgVo7kCA$)y;>T+^+eqY{a zP$pQ$Z^N=L+4o4yj11AjX(~nB&+?}h(gz3oUKXi~(N-M*&U#+|r2%?<#TnB*R+GwH z)ZpcT5d(9e%bDkEjq))wu=h~h%za>kcD|scG^pvk&@_8h?SG07A?kZTuZ9~xk1ruJ zfnoENeX>=SGiTT!8vk^0vM_OBg7voRT$x10tLr=fZ3HVHjM`iMrEr^}`ZIVIqMXp#ye~urEwdUIkfbgTEpuR8j1kn35p^ z8l(^m44C(1*FDe z8(FYYb{4bjNh(U15}hY)FxZT^JTHq#USxj0Afam-Gw|J%Pv^JDr5rVT80$&_%6IIW3tnetVA)VT9r@tb);_B*Wpcy;I-TLauD33s2Y!|9iFl=3-_|L3$wX% zNM)ILqfkJHJ^)}y*ReW#m&PPypC86}?-+RRbZ#ME=5NdyFsh<&-RM=cK9A=GgI4VT4_c@(suwJC6A#_UAAradf6(Y3R;VGe;9J z0|ng|b^?{;{@mFu+jVG)?dE*? za<4E=5IZCd!*Ms`0GKEcWWpSZh};k$M4J+LRk?x@GQDR^R>4uxrUqnx*m8_LosNx8 zs#-bazD7ACsDKHe0~iouca4R2gJ%(tdUejdZ>Kzznj5B;@fhOS>7+?6wbN-iJ3+gO zQ8RGpE3W_;GR3u7Tf3zPW><2G|-1T|gb-u!H>YQ&%7 zDD$A)no7yiM~6O_bIimsvvY%U>;)P0{Q9AeVN><6aJJhNSUQd}|1``_E={oO^xPEA za@G-M6bTM56UTz%R+A#SP8l0y!}L0x$2t21hQ(mNbiv&uq{Qe6qFO|GK6Ls0!u_lYJ+~7&G#{U+*Y4-j&kOm@P_{n$6K^UO3Io!++Th(`6 z@XsxsZPvCq+});(wPrspx7%v zAwD^Sn@-np`A9b1n90`YG(pHay$t zXmqTKX3Hs4RRxd%dQZ!(-7G=(bauOdNL_2@Y;t?f83~0)BI{r9<*l*6f9Xq69G;fM zsgrE4FMTOr`cmWogA~5>rQl&hxB%SQB64IncKxL<o*W!{VfZ)!344850N)Y7YQE<0^bH>C=^q&7p1#o&$B$=+F4p=z<*o}} zKnq>Mo@1@bvrk}CcF5(N5%4gA_x2zOUTfZJo<~X5V}rvZDS3A-NP>T>TO}MB(G!bYW781Rzoo)}wd|nS`AK)`j<`eA=EV&vOKm&8CTN!*Qxu zdRfiQA$pz|Cn6La(1@IEiUSzO3>JX4<;the%mGLUH#}b`%%&=eiC65-ZaLhsp7^V(yf%X)=kJ58N{x5 zKsDb?R@XAk$OxyDC2*6xbbHe$hx!RU(mZP^SpVVd(L=x~riw|#M2FdtoMqX3T%wL9 zd#AxIm{t@`g6~RBrYT_~BwxOztQs0;8z~AjbUs z(mPm61tiX^)6t{OW*(kk`U}&O%!-`Ea_I6}JkRvU45iLtx97&`$z(D+yFxgJg<&X| z!%{i*jEh=|2Iy#0Rttq|%V7}_bwqbOs=-5Qk(3JjcOM$H^e zd>C2ba!1I3Pq<-kw(rCc`yB79e8eQ8LmBu&d#e85V#tF&!1Y1>Y*K+eZ%oJ1uvYzoD9u>ND)mj8rdV>QQ$nO@TdHiDf5|1+D4fwm2^ z`^eDl8+Qs<)5nN8B2;|$)9KyeBRfSn!E{?09O6H6H@ThdcH7SQm(|Zu2$Nid6A-&= zYGl-J{Yldi6^4f}enS)!Ffi9;X~e1tLVWB*QK9}`a}>{GFFwtxvXK^Moev>TB*bJz z9Jjoa)-}cWm*~QTYkw!preY!rW4uY9?M6Bt0z_aH;{})oc@QOZYai9H&l3|l>_pM_ z4W|$F5A+Wo!CmL`j0vr@B;;(*(9wRh?~$J27lthe!3hpeU1kap;b~>bAt12Uwr7oy z85kUiV#~B6Jo!hd(TbsAF5E`uh9Yrak^m;5#MfAVh1cM<68y8+kI;t1Wr(J=dJzzF zmHS%ABinc{8PmsKmb^Az9nFGog<0tC#xbO500_fH&+Ntd+HIm2RxJ4zVgUn%Pmwvu z+G#`)qY=+GXDEV(W-w^sZ6Y(eSSY2mv)IOgI~sey`Jz}-=kGavVVYnSg)!K12@0y9c)|+3%6%Y@VL@9QB49` zidM(nq~PdOJn4){qJObKX~)uH+P@agmlEK+A1%>)@Xp{=>t6a_kR}{qf-PJ$skiyO*1~OPl$+rsEAe%EgOfq?S`0p)=DoQi?vb(1J(;5e6aO zr!tSl$+WkK)|2d{32_qkjrW3EHHGwu;?|A&v`k7a-V5J-Ma(Yccoy7kZk)PD=a7GS zg&iY|CSay&-$jsEI7G-^^GFagy0U31=>j6(Je<}U*zm0K3%ZGv9z2@P4h;k%qcW>59|aLE3ORIi@Z|n99aB1a=ofeOA08m{57l+x z#h#)5o`I454l_r+jYOHTZr26l$6{>oV>ugGiCOB1v2v4g)vJmK$zSILh(hFluPMK5Nj<F!WRx4>S&kJIBurG2*Z`%iu;r}U z%o~^M8R$9s@<@L#CYcurwKvz8dJc*BtWjn~T-q!Yk=j6Cd~m(Gr8<0UGI+a~HQ`bv z()S=|XxcI@KrXZ*cVLZC1NRf%%@i@0@`0oYwpwvI6pT<4y&#?tAc4m$p`^&5V&Ykz@uwn(!U#){9UbLz zPsF)rFa=?>ppjr5v7FMqveiirLOaHg)(pi5X3NOo6eE$F_FkFR>tb~63MJ`Q9pWl3 zz=lE#(x*`S%uFu4jUG9Mh=S&NkW z@CusP1*)`xPmChDX=;BUmz-I^n1zBe0eJ=fdscs=*d z#5-rd|HNC1fBf8X`-|7wzkQ|s+gICvX}Rf_e!O_?xxp*X4K5!azWUtAnlDYw%YCmy z2r0^84K@Az;2jcDtuu!OEt&VS!#s&4R*`A zU=vTqv;^a}qQ)7-*Sbt?u5sj-KygjJ zkFP>jFSBRjDc*w?Qol@C?Kl)tFYGopPq-?KF(fv*q$HOrbh9F26JJ*-zDk->P|Uyx z^EV~DkG};q>{b=Tc46>h3gwopR>pA(7&}jZ3zPH>HN~*B-zsqmbYf;=uY~z3W4BXI zOMMa8bt%QBE}GBB3zE^{rb<^Ts?y~$ zSJHdTE`)0tB_e&i?~oJW(%tk{mkzA_I_$;h(?WdGkgX69rNYE zJZ-BI9@8O+gc2(MhWQjDK^JDT5(+ z5kvLG(1a-pA>AEtq;e5PQNJLohU_cdx#B+8HR@m>o6*WCTrq1tbhC$;UXij=F6OF1 zp4jH{_{|KIR+6B4t(r~Wvx!EorCm+dr=K2yi6mL*2 zdM%^0nbZVD_1kQh`bMN2QppHS0*H(wu##~_J2TOs!f?$X%+0O>UNEc)?_x)Vyx6`l z5LseVse`DEF83X(XAn4}l{jmQ=b^J)KY6kZc@hLSnWJ#Tkeq{#QEFRFQKgM%OdXcS zJWkAq>;XU#<-BV<<>YkDV*n7*t56(vMq-U+*Tp;PJ(rmX?MTxm$o=b+N3 z)@H0Ko~pW$zLb1NE0?OzoP-c;d&cb)IKY(bLA=c2nesjYMPgaiY}&Zc)t5n(jRl}g z)8<;X8n#ZxL`xnPkjf`v864MU2VkVpItGa)de#h#1r+iX10ZVJIsDBmg(IOnivfCc zLcvvH2Xh{2zo)63Z!AN>4ntZ{mr46JOm2kcRoOu_%DzZRWw`W0>Fj z)Z9K>gVr_wb@B}afqjxd-O>WLO4bjhnUt%36O1qN`kO2f;4=n3wLpNu@z1JCxQ%R~ zJYQ&qfX|~xtWC3At54KPMrLuo{)N^O`1$D=&uICMo30=ZW6RK*Eh zMyH+8X4#OzTrEO2{jREj2veIs084@0OY>3#%}5gJ-t)Q7wZ2eRC?5xgDK5NhSbQn;drLO#yOKbxfO)g0L1|{Ea9IHJX8p z7YZY_gO#MYcjH@-g2hcj4 zbHWFn)dqU_>SQI=epTFBusu+nE6MpG_H;V+Pf)R5d&20ap{<;;6(Rqu{!5 zf#D;ALqz;?+K2)!10Q#Xf~fI{*-OJwX4VhI@cpxDe3@W+E@UXXpa+c1cI^%rDSV0-#aSoKSO@vL%#RvMCDa*{r zL}8^tgN>3mKv>eF6fdAcQ|36=9(00ka$yF0Ezne!WKA;*m*OIkHtiES&u`gBU>G`A-tVD6N{3QsXf?s1V=4Wf{8;g7#XrYVsB_7MsdMi zA!C+})R|?g)Wm@YvZ?D#AgB%SBs!E~QnC4!ImI;?wbW8cWKGrbueB2U1=Twu=H#rW z={=!ep4rm6o1I|%Mv5pb!TN5@(XgTfC(ZfeJ^T=5V{$>6)wsZ1LrXp0?L(efeIJYj zy;eg8J)2_}l08>1GB4sc1nv5gi&aXEbF1_o6XDKjN}Q7m%ZzhZqgcy_aHC{%1n$~p z1m@Y;o*3K1`_CqbI$^g(tW~wMV8OO_)6*mX=+qurhh3RD+b8nH$cF6awc2;9wF?{& zrxRgCFO@Y|p{LML7SSG8QOjf0^deVV`pR76v35K=eewIU|b+D=xGQh)d zo&TvE1z`_I_(M@5xK@~L28pyXUhIOQEVx1JAr=aR8&`i(RNdBxt%n#V8>9f^@Y^Dc zq&CLXftm0QaYLp5_=k9hA%G~oD+-Bn-RChEJXOg1cB9ym3UWHZ z{X&Ew!BlWMOQ7`uZ1mrSd$Af4gjUMgxFL`mo2CMuY1GR|p&sYy@PT;7G0pO?*D6vL zY$JGJwQFfyl2`NZd5;5;0xHH{EsU&)&yG6+MaxK!d)ijbhg`XED}y4nbLy}giK#ll z#gE+%2#k;kZad+Xn4!qR@?5BP%-9`>$AAPgLU@g?!0#r<7%X zeoAB_kD~b84t7TkyWUHtvjkNNpT1xt<=bjSOZpPfny<yP%I%I+oW#So?)g~x^3EOdx8n0;yXyu4-5^&r{KL$2+$%sQmB0wy2;J#M+I z?vKWr6 z)qB_Zx5y*aBL}Cswe)f+=yEzbL(UpU8YWR?n|E}m{%AEUh(w%8!i@z8v9Kw6;wHdB z|8TzzrZ%Bhq!rH{69ckiPsTjrTbnAn*XkNL1u32#8kIt*Q|^E&NXnT%ar9`&FXcD| zj&nJwV)T6+yOp&gU52bU0ScJmX&8}MF_?}6FHMxqQD{JlZG>wzeX<;DkDNb!A(K7` zVkLijtbd>{`;rxoCK%NpI7$qJ9QNXLwCVR8$V!>hE^mK7pImqf+sX=t+vbydO<~(n zZzNA8k$^!u@zr)UD}~th=iy8uzZ3;p2~nqx1@dm5^u!crbc!)G{alXql$1agP${o! z8}-0wB``0cLj->m-K;qGqJC3nbSvGv-gY)Q$~NT#C02MB&biU5VRP zSys7RuCS|EzZ83pJ|O@?^jS>+q(}tXNVu&LgFKAf~M<0`%@!sl~|l zLS6xTUsfRY)gA#xx3Sea=f%;q(k@_lB<|}QMf*khLS87)aNz%bp$Ne0vaZ{3J*7l; zfI>djWm&lFyf4*|lWl^yC?p4e1uY*MX2ICw;o%cZmg+k}=Mkzh(y?su$GkDZF?Lb$Y1hW8yOx=z}kvX63;Dg(ZYY?Yg z8zGGz1js~cz_4I>IB(sIIsH7GYwVBAzWoE1Lo;DS@dyR&yPHQ)3q({8TCNT)v}i_x zo1KmjAC2it(Rj+73<;O4o~r=Hy8TlWAVu)XXE{Nkc74YrH9z|a3g#0QZ{Zwgvj-ve zZtGkGe+q(#&RCP!PArDPq@dK~>PZOZeTdala>qFnQy3y@YaW-;QZSDf7$=QDsv~g* z3)Iuw<4^SuTl-$?jAfTbs;52Tnm~+mgeM@O)}9rL_ZR~Xn9!WMDwS#?$3q+^ehefc z6h8Ag(QAWFs)*f8tPITUB$h}RT|zoH?W)t4!)on;HP7U_!vRWCO%an?ka8FQs0@+! z9RL@VLIbS5MKnsvpq7@>&8&4m;DGip2bofVS|+w>cLWPEe}DRs3_qXar$5V)3@|3E zi=coK$CKh)9}=sHDEZVg6tqrm8{_T$l+Y0{sZJvJtY;};*tD}0Qa*$P$-Xm-I=pmJ z4uCJ{Yz6YeEw4}5@8@u~LU@}Qdnknqgf`y`3O(3i461dE{ucPb5*~_O>+TE=jT>y>b4JR z@WC&~%`hWJom?bdGm`xb$t77}<9CqrGeyc^c|x10IjAAXC!gYU!$A$^DlBJm-@q+1 zjJ^7A_GoO0lVAt;h~Bc+v2^<*vqcP0{Mz%eb=g)$%VW4tXTa%%9?-Y)eMe-y@S1W+ ziXTvbf6tE4{<41mXFWh0<<;F;k>Ta z>orXT;aqA;eQH zy~Gz8QW=B4tEXW*o2x4|WJO);9RM5ze55T1>C`z0c&w^svhl?viQn+~@_f66+WRs4iU8H;@CZYg)+jtR2xkCbfQ|fMQo&>7zngavtDZ96uFQLBI zZ!J=HTe=!8a|U+Y9X0%yZT!12d4WJE%)pzCBKa1DxYFU2c)!jcmHwPR`FMld$b< z2hv@QdmqXWYk^nd&5YEcswo($5k-oq3jeV8@5@yf5FZP+x*r z_g4sLpiK)6GLJBEg3iLN8W(20f^Mv!HovG1p}EJwq0Tv%n!J&6jD|8()O#Eq4al65 zcbvRs_GR;V(iyy-c2EV;d|Eq;mr}3}L0)0O#DE#Kg|z7#d#$utA3zf7Ibof zA!e(Ada(G_FX-fB4_qve3SH3=OU*DD9OYn^Q|91s3LB1XM2o@C|3vSo+XGXe` zD=loKMPh&A<<%P9AIXngo$5m3R5G79)o_AR+ou}y4TYwBqe^evr>5vrbM&djKCL^| zYwu@-^Yu)H521Y(?938drd)r&((a}p#8w2Y0`F z?>igb8()6#8$bTq)h#`*wXQZMwmE%Oi&s zX}VJEwgebxHN>(Tmx+lwrp|)uUH~ciPpCw`mvvUL%#7{p4Ly@@0-GkPc^>lYam{{*_&EjJ2fp~z;QqM z=DKgHeqZlR$ylGNcUGye;U;6&%-B!e=UejYqA{zAzx^yVOf^L#Y`jUkqAvTXw!74r zZ>={nuGbq;(@k2|K>+qcKcc#l7n>`0gzMEctLSh6`>AXXpYm-Loit!S`P2{F-_?fT zCwA|>?g~7)|RsxBpdxFNc?`mSiMW6xLLwK4XtO`r0v&e*$>`OQDv@~)@C>d2NF z>%L`c{*L^epVIoaSnEys?R8skuG{)u`MdM?d`j#0##(Q#&QpHJ_nY8*_YIUZO^e${ zjtph9R_}S3kY6Hv`}&7_dxrWJwtJVa)D9H*}Na;+DoX zU5jO>zAsinKmo#s!3C*>#QC5;Oe2(pE%ThB`~VsT}hU7rSLl6 z&*J=NnyGj?I@mN<)v<1Sz7{vG$>i>8t~XhRbu?pZ3IQ_=tXr+`->A!6Y*MW4oVd@klxKx&9jsTDL{-n_d^L@#W}pg| zs#1AvOcb-AEm2i_?JrZAeFRj7Rjgf@#ji!twbEE*Xi|8&o(xJ}q`-hsP+)}8_AdUB zT;mYZEpU#r)tVlt=3<&WtLIEu$_A20)T&y8-cby7YtWctFSI2k(cwm*q#m=msZOfo z26pyh)_GmolaZQ?f@ErNjoN#YQ5I(>rit@h{0A>^)Hk7w1Vag(IgNJBNEip}qte*Y zzoeDLMpg^xWZRCiYC~tMONpv-6GjEz)zGAyuQcoTD=RH-5m(mvh?=gAD@{63ZRPIK zk&)q1Uru@S2sr?wgU7RW?4{LqUg?a!4A)n--S~`-$$V*LLv*kEdu#P?i_o?9SMFSU z_4rWsP(&Ar!~n zI%>|!pdF41mzPT~bJzbPJ>svyz&g%p%lE$Z+P8k%@$d&7cfEi2Bg@@~m;1-Bc1(mn zvsXI~t*&coOMTSXvZ3YDq16qE4O`tKySV53ndPrMx7>5;YR9+yUk9&t^u{0fcthfj zo&QN@`SBO7ZasSG*!4|!zS;gp`%gR4{{H^u7e=pkjD66t^Loc!t8LAC)4I-)Rd(EsU%-H0;>5+LE|;BKh(9 zH4XeywF|p=|JKxp#bh?w(XyJzCYxJK>8sJZrQtM6<9Zc3cr#J)8c_mGxTm`;Uj=R7{n*J~4f%%3`f#P%QqW&G`76?8 zel=Oy817t3D&KOZ(|r^FonQCsRL<{91WQ18$Hk_~ zmT=z^eLS-@{8jz;Oms)}TE2BD(u;UX2aK)x`Lw5g*cOd|_kzSz1B-X6a=tTmdTWA# z*TuG{T}aLT^vZ@H+(xISIpM4HRbE>SGt;?N8>i+fE6p~qE9(>u0N^d_ z88KFWW?Un~ElqQ6OauHc`EV)m!JX;L+kUv~{W~+SJ@wwGfK`nvX4FZ|`Y?SFXU`sO>|?0Tc?ie~ZZJx^ZU{N$_aKG<-_ z^^T3#x842bci;H#wQb#3wspTV`{Svr+YVjde$St5{KJjcw(q;LecyW<{(Rll?MFUd z*R(P9Y9k!3W8VC?V5|HZOlCKU(E4nd0(E+fpqRzHmRY%~j@5&&Zb7_K8d~ z;i*I-Fp`VSOU-qeMB+@Os8^qY;Y8+-*wxUl4!)0!9@X!SQAOsB>h*e?F|dLkMy8{S zE%~O4>o9pVpY4NCI`S<`P3#h`@VoWx=B0IuG!zZZh;qbZo?^NR=qU2(0MNGne z@|(^)5~S0o`7%a$z#Xh792sae zu71$C(jH6ETxoIjR90G??e>#D@dwhJHaK3Hv*dh?ujFDV1=;d*F_fYO*r7Y#Ei$gM z!GXsoIN6;%Erj*>m36u#%!%!SNstTaumD&Pc^Qq<2)SQk1@~hu!-~-d<_D zK;;Rf=ET{R7Rx@!ue7@&tfPl8F0OQjjPB8cgC_?1I5pDDR3+gx6VP34?x@yAXl2 z?&}*nzxUnOzI*+F2j6M_(|ccUcys;lt$*+FKYR8^&tBVj_{zq^S2y;ru1l=H_oGD9 z`j$)C)sAG_OG$h#&8gMaL~GltkNxVxPd214-*I`}<;puf?>w=ZNbY-V@y>}qoqppxKW<&VvxgfFCbwVTe)pej_`?m$+jg%e8uol+wXv~n z|HrL~whgb2|5n=vt?TvA_15(tv~IiBdf%1S`~3Ys-uW*w|2%VT|G<^~16TJSr!#H$ zf0SrwOMTpuXzS#G)w|YPU++z{rha4TSC+1=yX(rjyWU@S58r*SDf!!585}q zey0?+w@zH`Y$f; zQaDJ_(tRf5KI2o8aZ}0JKHqRmcVYy#l-^0 zMVwSXzy;WFK_Ux*VQdimO~E%;D8y=oV$f1Bg_9|cH%mrfK%-n2rea*IJTAL7WfbZ$ zSZC*x>y-pQc)Pb=DLN+WV+s(8X6m%yMLx#mTHp(cI+V>)Jx0<>9GZRG*KUbI$+59o zwSU_Gq%#%^bEM}**zYH;L8zD@T}hTpQhFAja$y?Gf=Q~Xq*<8739nNE^EU_^O~YbO zRnB%>c_q10gG}}!2njCpj!K8Pahk{tm|^ibcU^hC{-oe)o9R84C!|*~5&kr%3hNC9KK>Vl+c^#rtoeYnqs=c%qxRcHO zrr%A#k2|iWx+(UMjYJJ0=-!nO>Pn>sWm>P!Vi{B#n=}>Ta&cYsDOxA8KE&eU+(6ze z(X8y`x~?&}p&6%}qO_R6c_w#)t>bI&K0kOcm3l0be%{!nEn+muu0B)*nZpUX``KqM zZ!}DhK(qLvg}Et2Y@y_e`uIX`R-;D>09mLJ_4p!{`C$%dngWlqlJ&EVY$F@7iyvwomSv; zJb?A?dn1GAf9>4tKIMPBm2_MSBzrstiW5O3snQR(mI}JI!D%JJS!Id zq`l5bz`S%Xhg!+^RO9(2%5j#{W3^lCTOSvP*^2VSzCI<(SK||QzAIRywtz;H!*^!7 zVU@xx-jQQ@@d0s$4;726)vmZuRskQ>PBA={#Zw})jxBg0n?26i%U1O3V9#(iePY0Q z_50HO6#PoRNIAN`^x%n+-oay5@(a0*(X#s5ntH6Rur8xuiNsf3BA0}-=cQU_+d&$> zoMd|csJ%FFd=&~qSNvq-%ZLf7sNz^q+TuS;$KZzuk%D8I1t!C-y5C+JEiqQrHKt`& z6aHCHay2g&M?bINFY$v=;Wa3!UVqOO-Qn@-I*Y2h%4|7bfba%QJ;)EW%-rBM_des!W0U55K(&0IiBr0Z#PuuTLpW8(QVd72sOG=-+ub29qI(%I3!q{y zic~zHXxY#gdq6BQ+De#RmEdXq#a#71b;9a2ikCXUqiNSM@Rf|yW>YbpGc_8KMCtD{ zdv;E2={NN0RsGtlUsVON#T3ZWZ|drA>6g^UmE>&cclq}QH`xZ$6eCz%dW)O>AsoEC zlO@t2vTd*Syw-fFXSJ!Z?amwYuGEg}?H#Y~diABZzInOxk50TX@lM0?9gm>6?cI2N zBOea*B|@M1vJWy_q~;SYZ$%nfvw8_m|SBH~dU*EENDw)`{`Fl&R zEnVCAz?F>;yuWc5I^L$I-Yfmt(vOy|J=K5Zss5`^y|Db`(dG5Wu5=9iq;t#aFY3XM z$C8P*t*bfvJkIC4S0}8&JfGXYx8b!7s|EXeGTGGjt>o&I{M`DRqMNP!Ah z6A4;uQIzK^$#O#y^R3jre5?I#yXg}t`Vc8#6)CFrlW(7@XE5woeAj6!x6q`d?YJVa zRNMnM5|)d~Qp`!lAg~J{kulc@SzTCMa``b=&MbCSUVm9`u%wL{tBQYjx-fB;^Cjcr zW;%C)SD1Ly)}B#LG@c7h(kFKZ21>0@$0!|t5HBTA2S(@&f6ihvETf|<>1SEJPLF{* zEKp;bx(pF=3&u%}2{w<|A*e-IK}Ijr-xQ{uS!tf3NPIqbzcIfkqo8LfYc?&(9Pz<5 zDg(UR5Quob3&T*rc)%9^6@~OL(Yn?_Gv0DqI&2FySk7B7@S>=_#AKh)DZGVE-ix<~xf>YocdDg>7$7ByHg#%S__112w&Axeft za5F_dA@b+az4Vt#m$HAv$B%#1Y9xZQxRr2~*f+Qr-$s;BA}C2oTUF{%32VA$o*7XaX;tLlL@yM%LnndzaJHY*ldb~9Qrb-td>R$El%33$p9eo(4weQPIqAyDHp*=&_Ar_Zo#$G zZZo4_i69-{;5Gd1wlC+shND&7)pNNb_9Ir?eecRkt0BwGEb!eN;$9P1$mF(|C`A`L zeqtn2Bo3))vdud9+GSRn8_TT(qQ5AooFhczmeZ>Cf}Eb6LY+WmxNsoVqvRv5e$4{= znAZX6PwIxpFf6svv@4INd2sV#w@kx?co@o|Xh!z}C0xQ;CShl}*SRKC22}HCv&Cd& zQqSzOPNc@h#$0z(rnsQLIx%*mstaQ!vkI&80X@{!TGa`@rv%4WNCsDGg6C{c%Yt#k>Lw6ZzounISCzrkZYqbveWm}D=iUbwm;W&zE+x<~I`GbH_+I;K?LXPP_nmLP(~Iix zXm66=oqfrlZo1?3@!xN|{!r$<-GBMeH~wnV1Mffg&DDmc&Tl61AK|Xr^6-@{4=-=t zeR*lMy}9#iT>M5~@-M&M|L3I-`TGU#Y8*&@EBRr2YbV%#bF#DRdeIf6X^&wSL+0y#NL}Y6J^L$pR%BpdU0n_hD`O|;ClLDl>1`=E3Fo=(nes40#;HM zu+si^GmfneLOPmDZA-1VyPEk9cb5y4m`da~{%P_e-<-L{drEgKwGrBK%X_98ti?9n znco!t%6C@e39+AQfJ-F`N~vC3T2~E1xyfADo7KdbI#QVS=NfGK{UD&Wb;7XU^dt&kx}()Y6m z_=l?ab_SagSHT@8%<32*)8G)6C{U3hHNAKj#f@K)a0OiK z=B>N++TA?1d!-SF{tp_e%*L85&C{-V-AW7kbdF!a_l2m9TK+8IDjDa!+oa<)XoLUT zIMa*&wyIspp5|e(gQEtxod64G(7UIqP+zD>5-vhQ-auFtn~~Yf2k{4GCOAE;MU)&# zUb^K1dd4E#*ckR|Fav4b1Qu}%fHMTa7-V*R#PMVq+<6X;2?rcVAp#Ql5Tmsn%yhLZ z-oHi-bM*j7`|?G(7yozXch$tW>SMT4KpAF0@TJB9QOey3x#Hkm!B081L}RE&vPTyA zQ~JGp8$yL{c3dGl8@0mNovSc-1*k6nxr>ksj9bgHSg=yULLf6k%4SfoQvvrcav~iH zXO6@;R8FE`a9?_j-E`)lut=YC6;+(Y0-XkvzpR}cAAouw@RxTrI^AhkoVHT-3o!#?wPaFcmn3JeEZDe zH_~Y|4qDSO6&`KtmiP+cVTM8=+Y9V(IuH8Bi?{f@-W*QBJ_p>@R~vp9A~CcyJaszK zv8@%TOge7$nP4FgkD2x~0r{4uFFegLW&V5zu%%9STQG$j!k9*3cHmVlW=aZAUECKe zTh)U<7|cRtZgy^FPF8vyf{hQwzcW+1Ab!!VUoyx$qw}^>L7%7jwbIVcMNt@4_@4z3 z#S-iOp6;)d%raZ)Y<*-Y5~KzCM6*b+qsS-aXDoP13ePU}}> z!Y=)1`tvvS>$mmmclfo^h#AEhwlx-EO#frK`YY=gg<={@|AcQ_08iz=0!>{?tZq%T zw!faavMz1$$WOjkes5~|*%z1ZI(c=&OUtbdD`__tL?itZVy?@BYen-@4;3 z*WGvdsmsrOu<@R^4qk4&JoL_E?`7ZLxPQ5IKk>(J6@G8&)uHdb_?s`jv-zLi`GY&J zbv$*Y`tgHYeJ5ymjKO!i%hA10DmtXmQr zHm$Z1U-e-}qIDw?OVJ`Spv}8G6I(**9N!=ZNAoe*Ok`0E|34`v;S!JFB<-*4gdXyYe&9)<;b_M9{Hu^*1N8@j$Uu= z_+HyHS>d+<@L{8X?o@sp;o2;f_0u~;Fo^%ufS%b%<&ue zD{hXDC1x}MPQo*K&j0xM?~QT8{1Q=DON~Elc-I5*VNCZvf~&a-zVl7>Vy5!V)#Q&V z^sAVeBTi68-75({?MXMd_D7A#T$-bhxWgc4%G zn$AQ?>tS^g31Y$Sj)7_BY7r|xDg1sd8T~yrrhDIRej^zmUicTNw`1`Tj<0S6uk)N1 zV*YaToMIVtQ6~{Jk=)YN$In)wAQ8krpjSqFr8o46ZX|M#!f66mcI^u?DU-a`BQKFF z1TbHu*qIc3a~AwN9!n;#uMAa^jb>g^%$)->Ys1WZbeY`XnbPlR`hH)(Ra%CP{l8W6d-y}Bqt(jh9r1&emPnK~mZAK~Kk9$Dda+2It;JQewEk^5La*2TUR z@{c`a<~oIcVXp%Svv?&-PAY%ifG>BXUCaV}UirKL8lqCL$`<6aw?it0Osp7-b9tk0>E*nTpL4@B7cWVZkC5!(U5n&bw&xV|Q~Yq8nv zmWz$_a1kI>%5ge^?MD^*2V)Wl!UV`7%hC!}stYqR>ET%{fV%X4{#yKxqd7xc0M|4A zlvOYbkkPaRhbVd29cm zGEF+KTEPOrEd1EM-2VZm(ko`?fR=7|$nYv9f?x}l936(w+@Z)C&Si627wOXQ3r7~J z$hO8~sAGRLQ$7aDus9O~_ra=akvwpmab-I%RA5YIl|5g`ovjj20dplZO_f5r>^lT3 zMY*!5FPYkbv6#{+=Xo5z)S>>~^f97g21ahc&-M}z!TwB z2HW3EuQ@Q?TBCd!Rj}R)+tcFMTH5&@wlwy&qR|w@3-ot~U>VntI)T~>aK?d!@~_HU z&Vyo&Ytq$(df$JDGNCW0hX#)x#eSXc86F<&?I&ZpFMYCqiH99@5Gll(0 ztv(bi5T_H4%#6w>+1YFoZDArpq@dItUDw%$x#O_YcvN7o#VR?88$9LtumvrPY-*ux`~Paw~cx7$l%fJaL>`KLcXJ&8TD1ff0N*}?rL-x zoUHT@5s!;MLU_5L_Bq1Et4QPTcnsL$CtpYan9o&X%BXJxm$kY9KTA|tB*7#wO+Ffz zEEhPor-CWZiW<5t75DkvMvxd%Im1TUEv5rcXSc2-_bras$h`9IzkyJ)t_`Nkmjuu) zuxpK1$Woe0f1qzoDL3`KRw-X;oXNe?a54n~mY<~yB#dg2c!-fLLi@x<~IFD{>aY5B#MQJGIA)7R4vyj5P_8HJ)X zznFYxwRv6Vv#XnuTee-_w*B>qw|2buzIt@a5cPJ=XOM zL2GwBOVHZg_pY8u?q9!Q_1nqB_PbTN?%KAlE8Du>-}VSWYulfzn`m@p>x)0R>z>t7 zJ@_$!YMu8IsHV>Zs&(!pNKFk+@_Eaf>)%*UoSObVoowvPX_~ffd;OQ+dhCsbw{pM# z?Cbk3cV2$%@-9a8@I;a#6P^4n>i&-!pOH)JfLvM!LY;)53hUn}nk~Wd-Bf)>B|iIY zJXFn9zZKDL%{;&6M+wrXWkja&yNRUWn7=A0W)*qN>?fuWH>0RJ&s9Ty9ja}JWos@e z{kR#mSnYP;+2P6jdi|~@7TL88cCAi)8_H!JMYnFu+s4IVjQnwhF=kor9R*ilG=oRL zjm%h$!7bwI;7bIH3(t{>uL!_$7Hv}(7kAvKE5;gkru_#Tyoxy*%-rWGOdv;q+Dk?qEw7uD8Gk!e7o6W$vub9Bs(k60iv?Q`1Wj zjKV4=E<~fX3@wLRBb*UtCup?-20}JbyoRR6;^K6qLtU%17wZUrA{w=C^YodnoH83z^XH zbc{HZH*O)1)}p~>Z=*+hSPQ{~^+nL(wG0{;Td6oHZ(U9m-EETHi!I%+meee~C0aru z!pL<6mRtOvQ>i1;_HT4(Sx$szyN){xFfkS)!910cvsCNkiuSGi1tgRhx!vI9l3y)~ zxo18UnN^%rY?Aq;EFN*UX)jF8Lg}1>7BwC2doxsoz zzEh?Y$uGY*vHXow%bULCQHB3!wV}DQYjv|Ed(ZEk`>;8AM+U?D-S>WYM|`ji)j>Z>p@57H=8?*U+73 z(632tw8%MApG2G{iP(?k#qv6HW^s>C7mY<>qM;FBq-ZVIF!}y3pv%b0N2Q%j)p&5@ zI%VfnQp`k6`JHv6{_B6vH~)=R{1HBou+YB#TKj!h+V8v8{@|7N2d}n2e5vnW9XzzW z{;3}~y?1W8{lIe5z+W8Xm!)Rs#+geC5*?sFlr!sfQM4Fc1_zG5>@>}4?iZ60!ZbDH zm|xBpxU^_lBI73JrWa=Hp!?GWT+48N3}qOs5JnUjrtWab;uKvSa!f*43v5@0Nk(c7 zAL%)sb)r%4GQCE!LRnZrYxiNsti>w`!|Ge{5%VWUY1q4S`8*toeIl(UC$5i4#^S4q zX@gs9w{~Lz(OpX?v}wvWV%LrL5F6XHG`OJH?mjEm?@9j=nz>`Dqd6~SoH!qVW#yv{Z4g6 z9h6e4lg+7{Bsi-x`<->kOd@B=YmqT|(I=5}ZjNKCruhtZ6o#y5U)kI2P4dA)+&ttg za3Sb57GfKPHbNJIeY{cUtCpD>vu3C8MfUj6X{})t6)q!@HkO@JMnE)G_l9LW>+$(z zC!oR}tZP12)HQ8!4Z(6_c|+J>sVs^i1&^?Li2!BQ>0mBzYp-`u#5yO`Xl9E*HzAZ5 z8+J8pRwiBgq99$1y~b3j!ms0nd^gtCZYH-I1E`u5HRYzGqs)1&OpcoUSW1Lu!@{f) zZn=A4VNYe0I(LGMuO2cU+(!|B3oZW-dVVJ0U(t`WaO3dL!|UPjl*YaxV6fqj*8~jS z#DnQV12@!oFsV2}`FH=u;Y3vh%h#<M61+N^)-^T z6e(GR=+zAZ$EQ;8-l>F`p{KB?eZcn6j#X2=HhbJ`e(zXbl`(p-3@g4=wmlroj z(Q~0g>BalvL35U2<-+FNBfFh%Rvy{XAMtB37Q=@mE(2V_vp;iAmvwLP{a8^9F*lVc z3h7*$n}E>kmS~7^zQ0pBSf%f4&IHR!Z)@Vxkst3SsYr{Bk5bU;P6EVysdO?q`Vp`Y z*@g7t86yS~=UmElObV-xKuAe)-b61bku%s`PkR$@BtHc{kU~B|*EKI#pu0Djse3o{g8g@VA|d>XEeBQVs-N$fnR}ldgH)*% zSxm0qy>n;g+%sp+?~}&P=%bm@Xid~xPsx%ka1@JIV%-v;^r#{}*jHo~E5jm*g{g`l z;!#G12OV7 z&@y7A&JwhZnfkhTp+0bHWc=7^X*rI-c@B?_ievleVJazc#v!)LZoA$gx(ohlo~r90v;eMQtd^1i*lu9R6OFST2&i|MW?*qKVqY)*02}B4fPW<)|v*b88_d21gGtP@4wqX5mABInJGsl1w z4lujrV1H)U$c{!M9@+EG^1}2euW!KfKax2c2JRr3O+q$f@&Wj>!#isD#*4mpH9ugTDHT8KvpV6cpK<7PJA2xtqXADm-P{D{IpSk8$UarSEXc^0C7=% zv_VD%@``oTr}8%BRD%>mB;Yqg9&RNk5h*SsU>y#J`Zpdjw_L06=d17ix1n=~#Ro|0 z)CRSh4Z3i^0gf^+UVQ6O6U0&Y8SB9JB0Q08MQZ=oKo@Nr({Pt9#1T01wP%Ri#Jzd$ zf&}88vJw<;SN@RySIk=fHxaWIT+k~PN(x?7A7ktgXmrr9I2Q(YwEFO3r!NuSc_jQF z#K5&UZ)P)oekd-csOC8}iuQ5F&tK~5S85h$mYY)E*s>9r?rC{z_9MH(@EsbkT&!Bs z#4{&Hk33KMWeK2zHTU0yMC*bUdj&~2(EhOQxtq2q62=o(xM-KKiQ(2>Rc;GMyt+k-DG4Zd)D zaBOLC?AG9!cgC)_-0s@D)V23^*Z!rh{WrUM7dsDp)VI`nXz}c;EBUB65q;6N_eSsS zeM3w8hRCAN6u%pdE6%U=Otey$FP_nKx*xdR_4rcP;~#zhv#!JM_TGB#r4{BtmXNr$ z^%!A%oyB8O(wjZ@;%8k?zT5jr_sW4T#qIS)tpxiWyq}5QJw&?s>ovdF{MY*)vcSGt zJ#aSamD06OO4oiVUHdmux+?lDFQuz|<*TirUzO9NTHsFYnj6?UmBk&m;09i6|1z89 zxx#B{ELL~}irV_CC&#e_>EI*JsLxM=zX+YFQjizQZBN#l6xMmemyy0zhyjBrZ1| z7~(uTRT0bPn9BJp3Knr5Ut*JV2D55Au*oXuYWl!rxZ0~)gA9nanx68fM!aQU^9dbG zN2q}@d=$qmWrxB^%Ond^9%oLDW2wX~klL|XRZVKl9T@^jw)hLPXl-GCd>oCBS~lhS z8b}zMVWuTm5(x@JD6ow4kbLPRhh@w+Oi{2wnzqrpIF^@EeKtumRKiONILZ*f z=gIm@HA|*jSVpvln!Iqs`iID*sTpz_&O+2Bis%U!UhY^z9gKW&DbcYt+3k?dZtWPB z7g6kpd3`bkvAUio!?n!#LDu3sOQi9P7ijM4a-xoP{FUYa6y(p^Q{Kyt_Hz5N!$*!g zcRN0193Lgt{T^_I{T1+d(m13yuRJEqpYV4do2&}!(zgaCFi}cCk}b-E2kFw#RbWHG zK42h7`$+cWqiMzlrj$QUdmG57kPlTcGlb{kn927__m+UOb{myb0L1Z&4itv}q&!p^ zSrYJYMTR~7Pg(`)E4Tq>2>+iZ)=^@QmVRCaK_2fZonh}*|KfOKSbUebgBgIC5;cT? zpwc~n;LPU^k&x-)nUVZuS&W8b8dVEIgu?FYTrZt%8R-}NGGPYs49qZ#1?7(5xwW~$ zP(w>hyE55=mC}LPclzhyb4=HWsUz0j7Tt!%HdKcw5}93s1AEElu2)*%3+Hq~Z91ijA0 zz7&{3h_Fz|1BHy%FkrCn$a!WQ!9iM#t&2Q#p@{)`ddj_UDBC@8L&HTnmC+IYq)OiS zY3zAa4H}E%5#Z*qnh$6R6olR9<|~%bfvr&SdHx(_aTTKszhz>Alw0aSuoeMmoRv&Y z&y)2j7$O{e%`jFPeYP|aEb(Q&` z0z=$CB;}6AQTLmDAX|ugxKn}ae!AV^+CtOB83FE1&T5gJ}NeUgQEl zpf!HPgX`Uh<>`a;+ZY!qThQKAq(a#?=bV$&#Kq$gDJT0RCiY8AJR~vkP{{vMTTxLi ztIM=pVDHUWF54fJBW>3W34N76g-7;8Bjo1!cKEaMPPJ%2d^CE%thk z29o2IEA97lu2gsZiYrF(XWhtV5LfkH+oxhr%5dqfHtr)Qa(DkXy0Wo<37Jn2hOID^>Ic_=i&67N-xi<68(7- zM5!>JS7$jUQx63L5qeduK64fLEmy8@oF4JJ*y2h(bGxXUOcd{CE357K!t@ZcTYz7 zXab_Le_O|5t{YGNp8J~?a~;W4)X~_u(2(?X+JF7U)&~}vlikgYo&30CXXjIkxwhm~ zw6*&PBey5}_fRlVlX)p>?wa6g>&lK?&E|J@+=@E>RZ}08k{r!yK3CJ3=+D)*+Nh1d zhPtP;n60agGf`AnIhD_~JbbgbYoTeiuqA3U~zn&Av#DO%t^mtn_g#>6?C;{5gP$+F_OK3Y%z+8Oi_=4l}o;#D+ z7|B$XAcGB~B9RVI({)qniZniSQuk#qT0cyhG&waY?lqH;+I>LUhji(tickChpY!?B z6uN1T<>%#||MUL;|Nbw3X=(}*NY}Yfhu>`>0y#qnHRrbBX4+wHp_Cpzx+vEV0 zP4YrH2xYVEhM#c76(I9}@*+9(hIb$=x64gH*#dOUa(F(a{D9m7R1xTnT;b#nc>%N* zP={w zq&k$NlA&suCxXGJw2^FD!G?%qm4rrRMHZE@Y+6mIMmleeRK$cPD=>v*h}xj8(BnX4 zq!f|nGBC=>(M*q+rHZa(5{jrLMc6?$X9Qu+jAn>RM#~MQI>iGpVnm|H6e>z-UBf*N zDKtyfjDfpX#FRucijH+yC`&atm#`PIjMY#j_OcE{}Q%_zf zftR@1ze42_F_3_zBKe?RAFlJ7-Gk)^^~On;;jzCLUBf=R?vzFD`^MXA(3LPc;Eb|s z|J!JLmVQ!nIrSJYC&>Z%Sm54WLGX3>| zvW{0OisIe9T`vRg>0OUVN>^uA(iN?1RHMfXswjHb#^X79d~D;EuF;L_l`$oeGc+Kr zbJV6T;B2J}*cJJ&Gn+S?8Vlo+6cB@~g1B;>hb|wf`q$06`H**p@E-4s zmozV&_1Vo?nEl%<#lr{2Qp=!ni4?heF;zhzq#qc!uw}m@fi!DI3V51bwKSLpF=T?P zGf__~LsB9yN`r9gIYY4?vyuG#1%)nEe5}?vzE&p2IGO7dyRn&EBI;16+8|)nt!D5x5Tx>|?1tuT%a%E1bK| z;~LAIS?roU8`vhk4v>nK#6SdLX%(T-`|jWC?EfG+w{w(?xet<2F6J-%yjRo>h9|N* zq=3N86C}jhd8)agihIPP`?QR5R2)&H3;>Zh2=E*P(6)882qp%qcr}w!sA_;(x0OB0 zr?ns`evJ|7&<7O9rF2daSKw4DI@x6VOdeaN7vK%_y~(YeWk{qWu5HU@D8M70V8oqQ{Ko&QGPk{&fnq40^cl15%`FJe{IVdc|P}8G~8B}U09RNgv z#uEd^6fnTOO|g@Fk-wKY1tP;;cYRP-LL4y~$_BJ=!zlYch%_94Y?x58`wCxlg8~DG zLM6ZCPI`*G!CVs9DY;B!-e^GMy+F_wXrT05!gKqXtzqH7Q81|Ds(iHJZFGuH zN$E7mI(QCJ5@d#nGf-ui6^@Pq$Tg_R5g^NV9*DV2A1GH4W!>~EV_07H9++GwXlY|4dk3Eortqp6bcDoKk z)V_<|m$qKmIx%!j`l$P6>ytP6UJ&hz{P>>Iu4&$Tdhe;dXY}%}n|w!2XemASUCRWG zdnSNqpmGVkT*oT8mwo5h9*(bMG^`M0$9s7+wouqi%=BDCs6fxTIdYKXNkfZq`(j?Y z6Lw^J@HuqG2G&~#7+ACC4eT0AC|PuSV;tQD)N>6viw+-(;@c#!dn;rB#4fidIvs93 zUl=c3>7Mjl?ztYFTD<;N$Cjy%E!B>#)o@S6-}BA15Cxd?L_Z6+0=jvkcXfpL*Ec;% zeiP{B+p0CsfUjTCh-shNha-@`gFmq}wN;OMum=P3I0S~6)_1GF2?`t$(` zX9BN)y_V?)X{OuZ1Cw(2iqC7w>Wfe_xsmyr7u3G(@zVo9G7NwHDab$|LnN@Eyt?Xd zFZF&Vgv#6{&jn9et+t52*!@Y%!?keqGoif}ZmkKS^N)@{T7IA^EWUE$T2J-C%~QhW z3cs0&4e%IhvBuM)#nOjq?Bk;j>fZpi&tJ^dsfhy@F~dqp(f zSga_iK`cm2YeQmM%_uDJBthf~f(xUme1nnj(J2e(fD4E0gf?Dh!JW~>!VYo$`g)iL zP`4iQ0&2Sf90-eG1gjD1!eE&l>tZ}q2-i8Xu6D!%FD~CMGR7)I>nu4pTqPD_xNIXt z1@X1}hM_Q!VeJ?~B8Vsqj_7`@IgIL=!>IkxW%8pEs5JMGX6X3XbCie=t|ku7EIkO{ z$KbEO0T~1>{1Jy&Ymw;11LdbH9Y3su4_5pKYd-(!+^O7|gH_+6E1N&@t*AxB$;f+c z@3c+ktIIc5#7#F=eK>kUt%RSf_@A6^Sv>Je{VmR)W##6VB+b zpQuGvPbMZ`zP9R`aV=TtK3oYOsrZl7gf@%ay)4RbH~1n1R>UIs6qe;%x5ce+Av2}{ zs#!b&V=l}%w$S=DY~qcA!pdUdjz< ze%sex;=#ffLM7j<*T;Ehqb_dgY|zE^bF;xHw`^tu+1|@lUVZH=cZ7?8S2DxOB@bF< z=i^rS%mG&3_5@}W~MXl^dEQ9X(h`o*DaYkZ8KB<(HT+EN$cq! z{XOsQ-iIVevl>0O@8`ZB&-*;T=Xu_r#A0Cq&v(UtxbOp65dM`C?&G&##MRTHAY2zz zAtk7y>dA?DPs$_GyEo^}ODT!JOF3WOpYro}UoMajrh@rUDulFO4dlZ4NGgK&pc=|W z^RZM+6tt#PTn(q1J%T2wk?%&|##iX^2*(69c2!WD-uBozZ`(gYs%68vanx;ISGQGd zNwsZQzZLb{*41yv@gly7l%yQr~Qb1GYyl$E-|e916nqf{*B zW_QU-L6r--T`8|KEg24}9pre#XZ$m)e% za*^rzqG{|xy_}XYd0s9%&J;GAKxS#&FnQ|%b4m*Ow-U#FoKbwguA_E2>uXY9(mJMmBYsX$n?XFKBWe zv&ikjD^J~bLz0#kY-35&*bXc3EHxTdEgx@fNmJR$Da)@-Oz6z~=n`gRd9wx6@)@R9 zG_uW2$OzEFgP|;xjb{rZ<+!`#@|-+^kyt@|obN%#kr;dxNm^kU#nnH>%XMK+xWKa; zi7HMH2mW>**k+z|CYNRH=#E7}%FU(1bU~jfxGXP!-1?ZTT>`(En#wl=+@A39 z7M5QhzkmWOl1>{M_}ygbv=uyZ{LskA(UW6V_<1^fM;X)ETVQ;grU`=>1Q9L4ue%s!l1apc%#^Lm7ylAp-+S z7H6$kI*p@hrn7~tnNF8C)p1p_p=dY0Ym9=-3zbOxYx`f`e(yxDcX0# zy!rgC&wUX6lvDcp@g_ z=u&vgWBHk8mRP}(h-nfLE0|6fAic02;dC1OSjy3RG@brJNy#}Sfpl8cGsupo({&Of zoo3zWlx-%`OQes;RwADSc}rjq(|dwQt4sRvVmuA<72#*nkq^VME5nsY>y?p8Yxn%q z*N!Yp-IbR1`J>nNE=%o|wyybO*T$Bmu1dUR{>ZhqE6-Ns{`uqA^ku34o+SCsiuNa{ z_l-rzJ1BXO=x_vvD?a>n;=>PtK|&_Tg@WG^7$KetLts!@#3>8&lr38PsR&QSoU$lS zxq>FfQ_Z}rCDp`JtvuDCwn6N~XWJ8}c<^2unaL|}iiePe6yJT`bfHXrI z0)T2m4QNH|Ii-tAX1YPHkzC<&O~paIzg&Y1>NXuU0!87upA34#Q(=-=|1V98I{Gib7In`Dm1#x;2--@=_TV98Fj*ttud)J-|zXP-tV z>;RF+h!F3yCy9I(B;m6{Rgh$bHNr(4pjiG%rT}6P*=;>#65iFFmI39UTF-lC!6CxUz&;icm@^&DtF<`@)DaAfgMWz7z5ikOnD`=W( zBokqFlt%dpA`hH{<;$8{-mrW*eFh-T&!AnQ!Y^ixES;Y4L{`hGTFIlO=+z)xfUdm(Ob z%a1O}kKT<1+QQcYE6u_~i9g=*hg-h0bt$^#+KC&!N^95khrj;tHwLdA`KPYY1#hLT z^TzhyK5(a}ztXewR`}-JiXiSwR+4*ey|B3Ng%z*3=|yqX>)qVBB6vGG@A`!9QzGa? zd$=SYVY@JwL^B)04ny)nIBz% zqS{jF?Q=5wDm}gPBMZ=Iy_Ib{7o|-%_AX01D_wnd=Y9924qvA|BpOJF0*?MIcdFTpc@}toms|()&RM~XZJinHGw97`WMa#5+$|E<$(%lJJZ(wiI*hM4KBmv4VSn1JQEsvgCg53Wc+P5O8ldV$O-+-D2poX#=|BPn zymeq`+)_-}DQ?4R!4ypJR!y^KPyH1}*25=@kEcz{IMbkbNj`yj#MPS*P`oumf*S!< zV?q_7?$E*72TI3NYtv{1pv4A?SM_q$UhA36iw)|%=BZchst-yKAeB@r>TjUBFN#o% zt!AxX|D4#^heU?Y2I^f6OhXm3O?7R98`p%Q4ZY#5saP}Su7?`FO1*CY2%f5{eYf3Z z&zy%f)qO2OD!hKSP}O=F&^7#6K4NjBe587oNT09gmzrjG8Ov7g zytS*|;V^I;TR|J!>8!l9*0p;GTs=LJ#R1JW@5DI>S6S|f@^RaF!Sy#&3X>Y%pqnQY z(*59C1Lu4KHl6X9TrxEI;1lwm-Ra%Ccgq9FS4ufkew3#Z4ctq>4q8N9J&BiR1d&6> zbG^h7sxT)47PSI&fwR{>s_3pe6JOahC;pyzT@-~^`T&hQ-}BCh39ls%mAx?aM&Qf{ z!r@}fWV!#uliY2Ih5_DyUgtX93Ox?D5o}d8r|!ZfJ$-p=R#WOGd{c!4cehmfxV-RzHxu;)zBhW9F zKU@3ES}`G;$~0DU943T};md~)jU631^z6|wTOg7oG!Dp#AbSZNTCw4gu``EGo=iVG z`qa@e8w*XKfSo5YMOFL;+!&h5R3s8!8~aG`hiPmKIR!y0^%s{b#D z18sVjBE8>!b}6!lBb%MKq}v1gmj?DP?tGjq-plXljR40lM4N9%b}U79EDj!CiX7&x zo`XqwyKQi(ZE&%5H^4~CBfugAiM+9<)u7NZc-}k++ z5bs!OKYLRGoO)k=bh-WPa?{ys-hYfV@z$po$DUt2{la4N=jtFD?Dlv)Dr)*As7S$D zu?&b6swxM*4UBKmBo)&ZaqE07B05V2M6&odz5qgqC}kVy@wO|YmDY}BsiP9xv@C6^ zH1)#*uhf9jL;V0samwr61|)Q5S=#oqox8vFMZ!XbW$BU1wuc=A^l+tb`=ZpbU@lAB zD;=9{EVcQb)a7fk!4QpsCQ*HeWAKQp1<(gz6+$QyM+iQ=t;H&YT3p-Rr$&&nu}T+pcrj|#ahD+8~&YDWQ#~;f# zk<%5E)H@tXB}29RGlWD8+m`of%y#<@RUDGt{6?tn&b$PGkcQB%x7fs z2V!%lw7eoeOmGrT5OM~}@Ml18!3|#>S8~K&Z|FPN;2@4p2*4&Kk9oPHVVX%N`z_R5 zB^~l5+lnD$d=N<%0fB{i-@X0{Ra&#dS16CjxA1Q?gU}8?$B+6wN%W06UNx$c!GHDf z+HU}VczM9K)g1EeYKQ8->Vuc9Qw?A%{rC;yH=uT@A^Zl_uo`(?3<+!h)NVC;)ej3{ z6QW*l#;ZMQ6MiG=W;KrAsM@PG<2Qx~QVV{Y)IRRCkE>g`7p-|VoRF<%u30FAbjS{< zl!u4&fZE*lM+61Hfp~*rDKk^hP;NYkn^-u$P9SMY1MgrHopYblC$pI(mz8i9!}Q>5 zX@#nLoS}`Xf3R*1Of-B(F{b6LYBLD9wwMDK$}nJvso<7_)IfB~2|H2P3gIf(ufj9{ zEVYy&UxLF;V1@yF4=xevr~zyEOnk~HcdlKDB+vf=TCv|CLZ*O5FW8LT9tBJz-|XEGC}|}ND^xJF#yzb<_z{NXRrYp0YqQJa6^tCQJ-*} z9(9|Std{-oR~z;S4V%6uD|X<_4%;EzxNvV5~Wt zeJT1lgXyWHiIhqBc@4}8;)neF_Z@PW51G3M%9}SHF)#d!h5-ZOa({n@5GzvGAuCgMFNi_rAOE0@ z*8_HRM=N))TOz0AU26LOKS%qY%97N9NO2d&KGJ05&_Ob@PzAOl(qY?a%nHinNqY{? z5n^AY!PZ$8>@qSoIyf$fcyHrz-f#bgPs35qWfEfE6hYJ3EW!(Xlupp$?4cuPhDS%v z4V@i5Ja+W-^M{<&(bK0#PoIOZP~d`ckG^9L@Nsh_+F%V>;o4U=;>PL_ezNf_kjtO7 zTsT@gKE%vquF8W45iG>#c(f3L%b6weD3}WkUobJ%BIIy&H=Hf1p8wy+$Lbf~YN>Oi z*eKbcGzX8Ta}c^nv+`(sQ3XHuU!H$F>Po} z%+H<0S`ZbbgP(TJba~6V3Dg&V&^Y*Crn0|N?QD=#KIBZsl`uFF7Zpfic-oTkX(bEU zQwu0Vi6el9i2|ZigbqqOPa(RhR3h!E7x*DVv7sH{O9Fq4flVaTF~J2|W?F_VF%HFw z-G$uHh(9L~m>5TBY@LjARIJ1U)pY{GiaFp^gKJOI%6-*YIRGiyP=32{jj*8&&)N-s z`$S2_nCbjLbmQ*IS`mGLmrp1-g=*a$6^hI#a#>?KSG<^WfefeW5@m4P0*ioYlP;5w zU@4I#b|x)xo}hh_cYU4?FLIR$sK(j_9)SORK5N1$v2LMk|AZ}&NFy`|Bt~AM4WJV@ zMk6pG{cyd0wa=N?*=(HClE{B2$C#}T@Cw< zglj7P4N_NSgl|o!fo&9!{zAv39=Ks?=Z;gZzbB`6~7Jbt2y&LiQ=UL4?#Ed!5K1 z5V=X@n?(MI2<_rp6X+V{5&0hejjJGJ0*!qbCWAKKi-GDhm9Cx@{+rlm{~kPK|Bk-Me?JVxR(+_q+AR8x^Y(U+FY)&gP7w|BdwtP+ zVG!F|reQTe@oKOe_XuHjnN>lHD>=A&=w8XW?5PEG=c)m17W4>np1G?3vd{q!vkTnt zCHL9Ldw$tlyTCK&HLG54_adNipysH(9v2B$QN>%aEgh;3REH-@~-iY{&vFPq*sq%Yx*QA#vi~ zF)-4=S0m+^6U*kwfpR3_;8h5ma@Vff4I?f%Pw@{KiI5eTwC@$cD+4h=(wx1FIkPkf zu2&e``HqVXcBm+HX0~YGH?kxXubky^PcGn$&<3ZTi2`qL&~6bNwb5uC9$dO)=-3hl z`!f{(3;vB+ka^+XLd^?ZHwJIDFNc!zhwp6JadZ6j&%Zpr(Dlmnt(iZWdu#6Y(2=E~ zBlE}ZL^>8GU!VIRvg=M$*PVFV!m;1Hd^^5lDZXQ|=@U2e?;LyQu^&yZ&0qY)6L6cx zo`l;p9){aA6vlyww#|=l-i5p(_Z^?5edUKAv@4Xy{YoA5MgwL$h!`Lv5Dnj_1h3jC z8hA&E9v=Rs3rZ6uTwcNjB}S?UcT(K@ikiV_kr0D}>?E_91d-*(+Cud*KlL7fbq%;c)SEI1s7*L~&#R@)d-=#Xp zevE2#9E|?}!P1H1-9Dcv{a2G9ZvVNk?WaQbKMTp93-V8eEpWih2R@AKoe$pi3!cbA z{|92%FFbMathgc&UG)l{?iHSKx4%}QUqmb`oah!|+`%*XDw*J=%j@N$DVwJ7<# zh~i&(!eaZ1074~^6`t%7JZ<+pesNS>IKC>-FBP_}@T`?qUb;t&E?im_@Z*%x+msj* Q+y1jV67hT~z&Yao02Sk#)Bpeg literal 0 HcmV?d00001 diff --git a/harness2/core/__pycache__/tree.cpython-312.pyc b/harness2/core/__pycache__/tree.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cfab2c3c8ab15525e22fb6d0c83f46f41575edeb GIT binary patch literal 25937 zcmdUY3v^Uxe&4e<) zXb`czjJvoVubr%@owM>;CrjFMQquHvoi!as`g9CwQoxlT^x zMO`o7r|Z=56xa9aJN4{s=rrJM=r#74I!!Fi*lX^ybXr*4)NAdtb=p|m+-vW1bUJv> z@ASL;u5-E)G}GzUaemJQPPDwuzl+xB>C6$Wow@jK5^e9<)vr3PjT0R&a-#EHos#ce z<;8X8A+Jqzz2{c*TDhw zx?Ef*F2`3vTrUcEmx~+3a=a_P+D9uVJ|$Kltuk90eG@l|l}N8byLx|)San;UJ*T)y zT!FS%h?~V~+KPJ0e_t^EcMV!>^XN%`17$>g+ZU>>+=Vs9@#GnXGesNA?cj2+ut$h?{2W! zc3kmGBY_J&{pW?i1z&&_^GmXD)*l%5`}^_HF9e1M1mAgT@f=k(JRn_?g&tWzdp=>H zUpSLd%P$Kll_#|D-CsZ z`_Om%FdW2GJgrA9Uz!Jf-Ip2$N4~TU^{Y`kEnSy5(2eiG{FA==;OhF#r&I5)Q}w4; z9Y!T4OkvR9-<>)>>_t`L;UE1i!dsl5>*R6lbw_a6anN;sgQ(w%6w!c~k;P1inOTfd zEn3W~#cW#4uEiW$%*kRdly$S12eBM2mP;`lYom5xO=2E$?`*GMP`A>+ zIjpUJ@$PNHp`bk~&>j_h{o;b{$F^xhQZEOjq@f=hKIje%4fgs^;zOO#(9m!?Y4!CE z`$puhf&Qevr$3N%YQMYsd{>kC>7Qt!3(G7cr&Yex)7#r6_DD&$5~a$!sJWyA=kxq| zztn}37)W}2LxBPH%ekJ+^xmF+f6{PvV4ycyC=Z=w#}W-(9_o?&a#!Dg=>O7&ehML> z9Rp9=aJ|AwGeP6niO(pvSG^*1{JEe8`ui%i&JCg+jW00*&>89K*^$Z5& zWTE80=#lp7B?(Q?B!jib5Bo$vO#0^5i zQyMpvD&a3GYTvFM+dGMB>Jt?;QA0_1ecVu!s9YH}l!ja5hLwrJM@UgisYV z6ovEShN?vM%CYvz14#bJTRXNRYOkI#)JlzNPd~9xclo3lJ6i4u`nwJItv+@{0EGBQ zf6i!*K#CUNxQi)luV$ooNTU@-4PTQ}uhImg#*CbqEvG6D8GZm@7ci;q;Fscw6O76LsZK4hEXjg+HV_2O(ES28=l-+ zMryY8(0cKeIdr7wA!HI6&<~k9IIpWcsB2sqw0j#n_U_)d?^HvdnB*ISC5@@0-*~dQ zzEeS9T@AoowLu4`t#bMMyAjf85zj=DFh1H*K%mB2q-0Dt~Xs)W+!Az0t~j z@#6h+`GvRk+}snXdv9p!{PfA_^5%H{_PN~rTdQuaiWEhxQEzj!YB!UwL5@rsYR8k!-BF{`Y!3J)P~>3@Fx?* z9^>Zh?n(14$4y80{Oq#E*s{jyqS-Yqu{ACCj?L~k7Ta+wZa*G19RKRzsVi*os}@1Z z5T~B%i%4JmsD#wT6nB@IPTZZ!RgLl?{vvI+MGtbMmO#!E^~hsglmS1}d9DCC1qi0I zM>K@EAL!Ds>bs9?LxW3FQHU2!qPdiNY=6>u`RmA?ZOrNZ0ODAJ2ipgvKCm@1=r?dV zy&%A^_!;F^ho3}=5Z9s{S7KPs0pWsvC$j(I0LUa!2!L+^M zCy=xq_4N+sjWU56-Z7XNLGM*r4K7X%t2uUaLCyyZ*<36_7$6Kyh zZn!2~aZB;6MTl91NPa|)TWZIe6K2axp%+4vEi>lgaBHObM{RGlO|`teFSc~`oU1sx zWW#j#^s#8kwz%u*sO9N|&3>b9qHZ!UV_TYVILCeCXUBJp^YePn?objQ*p_~T_{NEi zlk2W+1wo+f+e}3q1XITiCA3Fc7SML(IDZa1_C-tf zoUNR?*VP@{!A(%DSq?V#DLBR4(9DOQg@p@mwHB?IOxszP@~VPAj>9 zw!T%Ir02O4+z3C+UDco9hIy|wxcArvKhA*!J{w1f9S`E56&Zj^wX2*}gU}2n8eh91 zbZDYu*AMOiJiZU-Rc_e100GcmT1V94Q;59A@8x))fe?2ZAbFI}*i|}Eim5(+pTEWP z-1S@oH^Tj0nup3>Kw# zVAwAsSI6wt5qZX5OP@L?w$0ir zV)ly2x*2;7eK|g{W!5gl>_Q}O#=hbsG-%J8yga_`zJ1&6!;$jHuD5DuD>lX|Hs008 zE4IeVwnYuwKL4zoa}<8c@y3E*xbx@rcz-F=9DdKbbym>rP|EM)e_GHXIw$==!Vts6i&2Wq2~4 z{WaZnuK~4=Y9%}Hi)UYkvFPn=7kxrl`Hn%^7$$i@f-Tck<)8 zd+y{%DyOBNhTad|Yl&~%69 zmhuBc9li8z-#`7w?(1f5I;nx$0fOeF9aM&nhq}WvnvgEx`#+%!MF@x-&yVGAhz$Q|^oOJI{0(vY zhNxi!BVr)2uqf1_U(~iN)oQ{Yo98k7%ADs6@dV#P03j=JsuC%(E~M|^0(2VLlR-)t z(j?Dg2yTsRQUO#eclrvJ!x%DN)W~nOW=#^G z1R4)QCNG>(m;Veg0Nn$(>AW(I5F$sKC}=Dx9M zV$<~|NFK*+bB?00Iqnd~cFZ~RCYx`y-E0fDyt41-&hl|S;r84ZnHahL?Qq}JSb^AK~M<6_8(y>mdx7Trshg65S=?8c+!CnuNP3>aI+9if_4Z zx+D4-PZg3AdHJ`t+}tu{zH6P_63^QXSwhRpmiYqCRWV=4Iozs9Artp&KywP?A?_rD ziqsC-%lyO>k1{Cg4WOtqj&e2;gSyqP^71m|*7#_~)$oj~$-;B|gFtjE29ZG)<1~%( z69CXbz=VsblZGtXqr##T054epz`eXQgq$f5p5%Q}5I+~t$qEQB#a3kU>93N=FWPi{f*IUu~nk=mfj-n`cIDW`Lo=<~5zXJyP;8Mjp3 zx14xSAFn-eZ+N!t`2Duyh{ji*nAhu!rW7(@N2HsOAw4vg8XjoJ*UXI*(nKc26lN7r z@FM@7CVE-8b<}MF5jPz~_MjGRyQtqFcF_n#)Mij2Brrky*JA<@qWKwYtT7wYnjVMN zCsPL7mWG8%<8P7-v`id{vt_g7d>iS=s)%x zBa2L*u5O$P0G0#G(3mGr;bFbDJq(KU~?_}9Xyxt?iC;_Nit5#L_5*}xTq$_@LOFBK(LN6i25h{v&lox7! z1l(uN(!}8~ZLTK=o1pi3|oa z>@E${PX((F^+SJqi9tlD9awXq9GDeD(#8xox(L@Njj&VblNH$l;drl20iQm^lLpZb zaU?_i7Dj|aiz|Jb8etNJ=P3`g7ySVYhE&Dkh)f8I5e6(!Ag|gU0pCfa-@%_uECeGD z9G*$TwWhJwgu(LCo)`8^1|s6v9_R<>+@;}-ad+j|E}{)uN!<{ZUpX*!c-FZ#gF0A1 z5*XsSE8?!|as8aVVDfwz!h_0)JayrR&qFwQW7ov4f4XPh&bijjJ2+RttaI6na~Wzv z2i8pNLd0UdVVPiZ(Qsf&9CyuF)`47^Y<+oKB1~ z;ifWQ_I_2|su46qSrS1b^4TzdhA1_Y96}j-vJxG7sP1gG1-v0q zJd8Kojz|XK0iaEUN4AC`-{(}Inj$-jL3Z#Yn*(zH6;wo~G5YiVfXY15MwDpBW}!XG zHR}Pk;j?g%|DFnx3@t^16mE~LZk}2*8Mt-z=G6!vS$pewd`M9sOP8Y)Ndt-f7mypQ z^-OZZwXj|3H8k)CT1#VG!k%+u%fyy=;dT}2Z0wH4xnFAVAdeQUM9PeUY z=mA=_NDU!NHWEWLt7J+xR#QdNSy@brTC(3LTG=RUZ|Fy@7-_N6Zf2N`SOEEBh0S7W zE=O&Tow=o%H7W7*Y>nE}Bt!NURQs9E7qY3}h$I#r6o&MdzzKk~`KyrqNpm3;B0USs ztK;QcQdp^g&stGV{$!NoJNfJTTM?E3F~< z0z>?S)Pw=jHxL<+rX--bC-onC0QOl#qjCTkUqSKhO!xPtRk2`z9RdSdpB%^H2MeA9 zPGwe3AU~Q;$*Wd2)OxF=Qy6}dZwMNwm0*rQwm;{fQV_tM_xAW@06yCE3?p|-dsuE8 z$buD!NdzYte33+oy+Q%WQY4&KF!-|HCv{&)^82MO`UuJnA0?(|l!z#U-5;q70}^fJ zq`ueRFI`0p0tUuN5oBXcB=s1ybb~S(&-Gk|G?{ftcdk$Brab?QQVh&&Oy4~u$#h=R zq<|_auxD7DNxy+C{}F$(k0)w?a8@1!|M%_H33tx8HBq@bTDfVwX4ba+zHRy4zOj}$ zJ2Cj{Y_DH}EV8w?z)mdRC1a640ElXL5jqGv~(fiR0m-*^*VUl2y^-+Nm{D zho^Qr*q@3^+ce!u74+dJ`#w#>*DtM5A@Tm zKe5K_o9{Nq?Az|?@7r5QUa@=Me=7Ndl0RHBRr%vpcUHYycQ-$_YTG@-{Z*~e&gaGr z#GkH+;h&{?&Rv$O4J}%njJP=bb4%%b371nlU&=Xh#`Y<+KnhS|);W%r2*05&16A-i zgfa?K;Ln+i59^#f08XEh@T8PXtI1$1rAXi)YP0$FXya3rrR+$kyKRlTOrVS2Bfsa#Z{El%%r@V*L>ipc` z2QWxsE&&WjGTUE8vhF+YBgDu)r;O|gziGDpBo>EW2PW+D7QrXL1cxvY&?yXxZG<`$ zP5(KdY8m=|;-*oT54=8!z#$B;6XXG<_6xA2BZTblM-|<3M|p(4I81{s%XrZFmi!SAl`S`d-i*y~4LESXxCBVq15R1@qI0 zpCmi&(NSUGEEx&~Jx#(;|E2zcVf2!+2*D+aKS8!p!vLsG2s(w8P$$4{poJ{^ztq7l zs9NY7!o>A~%3!8N`*wF6hTZ@lXttoffa3wkf+T-3Sa@3N>KWF*GoWT<@Z*7zlyPAn z?9PWs8F`vHJP4Ga)PL45RKWU>GE^`Z4xc>I-`$8AgW+|f;)$WGC&@5WV#g;7H8gSZ z(BL3bD+?q3Kn){ug2iY9>9j^8rhkTlmY0!g6YRSKA{1LOvVYW4RI_4%I8cTve- z;!pm2g@A#bWzMX1NzA%rZ2P>67|y#kFs_FgW_j(@x(^I9!iMpd8*LM9 zlWjAOrFYkZj)B0!fBW!^z2eS_gvD{gK4G7fCdFv!k!b$W8OyPR#g!;s{$}m#wQtr> zxWW}ZDVv-T*Y z7q{#;?O`b_Mk*plB5feTo}FrmHtvWn-8tiFgLWoS03pw-&rR}kMdgulR8+KT%J*ac z9sl&scl+*++;7;G$S=ON_vYTjk}|b7k;_v#Q~G(1uRF>kEIG#iQm-#4#N?Kig`a)% z$m>TUo2Smk%htq8*3On}ij{17P|}pDhKx`nXQv9L%u|=AbEfstb$g@2z6bgHAG*21 zs!y$4N$tY|Zpq5eKXXHTGHYKRvoD7>!02AECJj^als}rke#Ww4&f@yJc_T7>$@Im) zTF|zF``L=TUB&vJZ7STAr~fd|h`6Q}z*b3HlWfPHS!nG>{FBJ5yevo;E)$doK%^

- Harness²
- Recursive Agent Harnessing for an Open World -

- -

- Paper - Website - Benchmark guides - License -

- -

- - Install, set up, and run JobBench: uv pip install -e '.[render]', harness2-setup jb, then harness2 --bench jb --k 1 --mode parallel. Click for prerequisites and commands. - -

- -A **harness** is the system around a model that enables it to act as an agent: -prompts, tools, memory, skills, orchestration, and more. **Harnessing** is the -process of optimizing that system. - -Harness² is a plug-and-play framework for **recursive agent harnessing** in an -open world, where agents encounter tasks unseen during harness design. It -connects two levels of recursion while keeping model weights fixed: - -* **Task-level recursion** refines the execution harness for the current task - through a contrastive **Propose–Probe–Compose** step: propose edits, probe - their effects on the same task, and compose a refined harness from behavioral - contrasts between executions, without ground-truth feedback. -* **Harnessing-level recursion** updates a persistent *improvement harness* - with adopted harnesses and evidence-supported lessons from each task to guide - future refinement. - -

- - Harness² pipeline. Task-level recursion: an agent executor runs the initial harness, a harness editor proposes invariant and domain edits, candidate probes execute them, and a harness composer combines them into the final harness. Harnessing-level recursion: an experience reflector reflects on the step and updates the improvement harness used for the next task. - -

- -Across three professional benchmarks and four model–harness configurations, a -single Harness² step improves GLM-5 and Gemini 3.5 Flash over their base -harnesses, with gains of up to 11.5 points on WorkBuddy-Bench, 4.5 points on -JobBench, and 11.8 percentage points on LAB's partial pass rate. Increasing the -task-level recursion budget brings further gains. - -The release supports **parallel and sequential recursion**, **OpenCode and -Codex executors**, and **JobBench, LAB, and WorkBuddy-Bench**. - -## Video overview - -https://github.com/user-attachments/assets/76b10548-b14b-4c8f-9aa5-a93662df4d56 - -## Contents - -- [Video overview](#video-overview) -- [Harness components](#harness-components) -- [Install](#install) -- [Run a benchmark](#run-a-benchmark) -- [Experiments](#experiments) -- [Web access](#web-access) -- [Results and resume](#results-and-resume) -- [Citation](#citation) -- [Contributing](#contributing) -- [License](#license) -- [Disclaimer](#disclaimer) - -## Harness components - -Harness² exposes eight editable components around the model: **system prompt, -guardrails, memory, tool guidance, subagents, skills, scripts, and plugins**. -They control context construction, the tool interface, delegation, and behavior -inside the tool loop. - -

- - Eight harness components grouped by runtime role: system prompt, guardrails, memory, and skills construct context; tool guidance and scripts define the tool interface; subagents orchestrate delegated work; plugins control middleware. - -

- -## Install - -Use **Linux, Python 3.12, and [uv](https://docs.astral.sh/uv/getting-started/installation/)**. -From the repository root: - -```bash -uv venv --python 3.12 .venv -source .venv/bin/activate -uv pip install -e '.[render]' -harness2-demo --output runs/demo --k 2 -``` - -The local demo runs both recursion modes with deterministic editors and real -Python executions. It requires no model credentials. - -For model-backed benchmarks, install **Node.js 20+, the Google Cloud CLI**, and -the [benchmark dependencies](docs/SETUP.md#requirements). LAB needs Pandoc and -Poppler; WorkBuddy-Bench needs Docker. Then configure your models: - -```bash -cp .env.example .env -# Edit .env with your Vertex AI project and model IDs. -gcloud auth application-default login -``` - -See the [setup guide](docs/SETUP.md) for model settings, authentication -alternatives, and the [Codex executor](docs/SETUP.md#codex-executor). - -## Run a benchmark - -The following example installs LAB and runs one task, then evaluates and reports -its result. Model-backed runs incur API costs; start with a small task selection. - -```bash -harness2-setup lab -harness2-check --bench lab -harness2 --bench lab --k 1 --mode parallel \ - --only-domains antitrust-competition --max-tasks 1 -harness2-judge --bench lab --k 1 --mode parallel -harness2-report --bench lab --k 1 --mode parallel -``` - -Sequential recursion runs the same commands with `--mode sequential`. `--k` is -the task-level recursion budget; see [Experiments](#experiments). Judging and -reporting must use the same benchmark, executor, subset, `--k`, `--mode`, and -seed as the refinement run. Add `--dry-run` to `harness2` to preview task -selection without model calls. - -| Benchmark | Tasks | Install | Guide | -|---|---|---|---| -| JobBench | Professional work | `harness2-setup jb` | [JobBench](docs/BENCHMARKS.md#jobbench) | -| LAB | Legal work | `harness2-setup lab` | [LAB](docs/BENCHMARKS.md#lab) | -| WorkBuddy-Bench | Office, web, and code | `harness2-setup wb` | [WorkBuddy](docs/BENCHMARKS.md#workbuddy-bench) | - -Choose one benchmark per command. WorkBuddy also takes `--subset office`, -`--subset web`, or `--subset code` on every command after setup. The -[benchmark guides](docs/BENCHMARKS.md) cover each one's data, domains, and -grading. - -## Experiments - -Scale task-level recursion in two ways: **parallel recursion** explores multiple -candidate harnesses and composes refinements from their execution evidence; -**sequential recursion** carries the refined harness and accumulated evidence -through successive Propose–Probe–Compose steps. - -

- - Two ways to scale task-level recursion. Parallel: probe k general and k domain-specific candidate harnesses, then produce k compositions from the shared evidence. Sequential: carry the refined harness and accumulated evidence through k Propose–Probe–Compose steps. - -

-

Parallel recursion explores wider; sequential recursion refines deeper.

- -| Option | Behavior | -|---|---| -| `--k 1` | One Propose–Probe–Compose step | -| `--k K --mode parallel` | Probe `K` general and `K` domain-specific candidates, then produce `K` compositions | -| `--k K --mode sequential` | Refine the harness through `K` successive steps | -| `--substrate cx` | Use Codex instead of the default OpenCode executor | - -After each task, harnessing-level recursion retains the adopted harness and -evidence-supported lessons to guide future refinement. See the -[experiment guide](docs/EXPERIMENTS.md) for task selection, concurrency, -reproducibility, and baseline comparisons. - -## Web access - -Executor network access is unrestricted, and Codex runs with its sandbox -disabled. Public benchmark rubrics may be accessible online. Before evaluation, -configure [network restrictions](docs/EXPERIMENTS.md#web-access) to prevent -access to benchmark answers and grading criteria. - -## Results and resume - -Results are saved under `runs/`, organized by benchmark and subset where -applicable. They include task summaries, deliverables, adopted harnesses, -reflection notes, and evaluation results. - -Repeat the same command to resume. Use a new `HARNESS2_RUNS_DIR` when changing -models or task selection. See [results and resume](docs/EXPERIMENTS.md#results-and-resume) -for details; each command also supports `--help`. - -## Citation - -```bibtex -@article{xu2026harness2, - title = {Harness$^2$: Recursive Agent Harnessing for an Open World}, - author = {Xu, Ruiyao and Chen, Yanfei and CuiZhu, Zhongying and Dalvi Mishra, Bhavana and Ming, Yifei and Yu, Han and Han, Rujun and Lee, Chen-Yu and Pfister, Tomas}, - journal = {arXiv preprint arXiv:2610.14224}, - year = {2026}, - url = {https://arxiv.org/abs/2610.14224} -} -``` - -## Contributing - -See [CONTRIBUTING.md](CONTRIBUTING.md). - -## License - -Harness² is licensed under [Apache 2.0](LICENSE). Third-party code under -[`third_party/`](third_party/README.md), benchmarks, and datasets retain their -upstream licenses; see -[third-party notices](dependencies/README.md). - -## Disclaimer - -This is not an officially supported Google product. This project is not eligible -for the [Google Open Source Software Vulnerability Rewards Program](https://bughunters.google.com/open-source-security). diff --git a/harness2.egg-info/SOURCES.txt b/harness2.egg-info/SOURCES.txt deleted file mode 100644 index af94ef2d..00000000 --- a/harness2.egg-info/SOURCES.txt +++ /dev/null @@ -1,77 +0,0 @@ -LICENSE -MANIFEST.in -README.md -pyproject.toml -assets/Harness2-wordmark.png -assets/components.png -assets/harness2_pipeline.png -assets/harness2_recursion_modes.png -assets/logo.png -harness2/__init__.py -harness2/__main__.py -harness2/demo.py -harness2/providers.py -harness2/registry.py -harness2/settings.py -harness2/suite.py -harness2/workspace.py -harness2.egg-info/PKG-INFO -harness2.egg-info/SOURCES.txt -harness2.egg-info/dependency_links.txt -harness2.egg-info/entry_points.txt -harness2.egg-info/requires.txt -harness2.egg-info/top_level.txt -harness2/cli/__init__.py -harness2/cli/_adopt.py -harness2/cli/_common.py -harness2/cli/apply.py -harness2/cli/bridge.py -harness2/cli/chat.py -harness2/cli/completion.py -harness2/cli/config.py -harness2/cli/demo.py -harness2/cli/diff.py -harness2/cli/doctor.py -harness2/cli/human.py -harness2/cli/init.py -harness2/cli/judge.py -harness2/cli/main.py -harness2/cli/report.py -harness2/cli/run.py -harness2/cli/runs.py -harness2/cli/setup.py -harness2/cli/status.py -harness2/cli/task.py -harness2/core/__init__.py -harness2/core/context.py -harness2/core/events.py -harness2/core/judge.py -harness2/core/ledger.py -harness2/core/library.py -harness2/core/prompts.py -harness2/core/proposer.py -harness2/core/recursion.py -harness2/core/report.py -harness2/core/runmark.py -harness2/core/schedule.py -harness2/core/selection.py -harness2/core/spec.py -harness2/core/tree.py -harness2/substrate/__init__.py -harness2/substrate/_native.py -harness2/substrate/base.py -harness2/substrate/claude.py -harness2/substrate/codex.py -harness2/substrate/command.py -harness2/substrate/opencode.py -scripts/render_readme_typing.py -tests/test_bridge.py -tests/test_cli.py -tests/test_executors.py -tests/test_providers.py -tests/test_registry.py -tests/test_runtime.py -tests/test_settings.py -tests/test_suite.py -tests/test_workflow.py -tests/test_workspace.py \ No newline at end of file diff --git a/harness2.egg-info/dependency_links.txt b/harness2.egg-info/dependency_links.txt deleted file mode 100644 index 8b137891..00000000 --- a/harness2.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/harness2.egg-info/entry_points.txt b/harness2.egg-info/entry_points.txt deleted file mode 100644 index 6ac95293..00000000 --- a/harness2.egg-info/entry_points.txt +++ /dev/null @@ -1,8 +0,0 @@ -[console_scripts] -harness2 = harness2.cli.main:main -harness2-bridge = harness2.cli.main:main -harness2-check = harness2.cli.main:main -harness2-demo = harness2.cli.main:main -harness2-judge = harness2.cli.main:main -harness2-report = harness2.cli.main:main -harness2-setup = harness2.cli.main:main diff --git a/harness2.egg-info/requires.txt b/harness2.egg-info/requires.txt deleted file mode 100644 index 293682b6..00000000 --- a/harness2.egg-info/requires.txt +++ /dev/null @@ -1,5 +0,0 @@ - -[bridge] -litellm[proxy]==1.95.0 -fastapi==0.139.2 -google-auth==2.56.3 diff --git a/harness2.egg-info/top_level.txt b/harness2.egg-info/top_level.txt deleted file mode 100644 index 12ad3db7..00000000 --- a/harness2.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -harness2 diff --git a/harness2/__pycache__/__init__.cpython-312.pyc b/harness2/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 359a090d946dc56a32bc95526ed3e1805ecc111a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 243 zcmX@j%ge<81jqhe&8lZ&V0aATzyKqZ@tFh2n9h*G5XDf*sL51n0pxll7UiWD7a!VW zr4W>wTv}9|S(d8cn4X$fq5zf3%u82D%P&$$%v10$NX=6S&o9bJ(eu+}yu}_LpOT*( zAHR~}GsuQtZu+6csYS*5#)+x=dIgEasrmZl`9;|!MX9O9`i5ntMP(I+7W(Cex~UaF zt4i{LN;4qV8|lZ#XXa&=#K-FuRNmsS$<0qG%}KQ@Vgnira(yw#b03%)85uvxF)-Q} Hu>iRME5t@7 diff --git a/harness2/__pycache__/providers.cpython-312.pyc b/harness2/__pycache__/providers.cpython-312.pyc deleted file mode 100644 index e20f1ffaebe0a5f52283c0e31fa72613226c3960..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7462 zcmb7JeQX=ab>HPK$>owFC0f+C?Ug@8*^=z2=d4sYJbA5&4k9GV+I|o)4B=6qa!Z*!k;RAV4m~p}96GLP;)4Z(YDekUxU{BS!&R z9EzfEW|tHt(@ruHXJ_8Lc{@A%-tWCP`}2|#o`Uj^|M#zBe^o(Izrvkdu^J6}{c~vC zp(H9qNwj2%(&Ltpg~n|r%7hqlWkW1n*{F5g7P65sYt%l@g*a&2Bzx2`&WCttb0MeX z2)QhjY?JuUouARhJuFlgCAmJLq>|4p2IVv3LWN4Ca>@M>8*)oN$pcr9R3VkZwN$E< z%HUciRY_jBdZlV<8(g>DVEi?yjzJWTsnM_|3kejTP$U`Q#;_8VRY8eqaX}78MuqXX zBu9lvRFPxaepQIXW5dcw5`{G-9&7#$UV@+2o#9w4t`VBr3w?(a*20l!SXH6JB~esO zL>28f_)#!-2N4oLD31KB?g+=UQ4~)o5#1I~$g!}Z+a_eB$y2(0NS07Ma?Q`{W$nF# z-TlG7o({3SuSYx^xU82KkS+{%i-WSG@LT5ni05ReP5N}S`E4B_8b9GP0 zx!{FP@%%ZlKhV_^?A1NNzCdq#PyT*`up_o%6i>;qi1|jzg)utx`UlXsL&;Q#2K};J z0IdRzvdFB&oPZJ1VoA#yvXY+7?Ab`qZuab?$C*8j^c=W%gZDdiuFuqgRG?jmg~w${ zkYiFJ4vHX5D%z+JL9zrYpoF8UfaGBr!E%9V=mzjjC@3Bqhf!f7jFj+DRBqN8RYN+9 zCF!hoBO&XQ&JBfCSp;nWeMu)0{s(3v>l!b?ZDMeTnSJ;q}HLPzUgYYpvp*XX${LQ z%?Vgfyk#u#SdNOjgxzwdBl1ZBE=kRUnO2 zerTv~L-m)`f3m?Be92$*WJy(CG1EO8S!FA-mDMw6=eDh~)!A*nnXcJ^Ro0jFmd~7? z?Rd(Tqy0Ha5!2M6Pf-|JKm~HIKTVAljuIAl`E(8*-Ur@kdfGD0q%DX|)5wx$RF7sV z5mu8l^@&AdK4o$(AleMO|n%?qTbVnkLGzEhMvcqD*%o)%coy+Z^?oef53BwNS zRw7nqflq*f8la&&31r}RFGtZKpu7b?wG*ltDqFwj!S?&xm(x-zxzXIxj;8HP*D)13k zv{?f36sN;10t;z+;B`-9B~P0H#hea>Qi=O zw!gF&aWI7HaD}%8;ngF&^*Vi6w~a*OL*b~OLEA7HuK<;rE?t z?I0#1bbvMsFCCqF*VE3M zckh4qarlezAB}(6zq-GBwj)!zd)c)IGqru(mTmAq;ve#ux@Xz7H|z1v@;QN3EB5La z`~2Ah2Ok}Kcr3e9xHq*hwX(By>F7&4TMA~(f*JD^=ODMQUa{9?OZ`i&8Bh4@>%W}( z=c&i%zO4UQ!%rJBt!IIj;li1#qR}Y413*NkCb;VE%W!?qd0(cgdFlG2iH8$QV^2!H zie&;9G8GqB`S&vHdj?G8K@=Xo6n+FIYTgVJ;1H0r5PmllOkzHwrWuI^)yzW$NHq&6 zqh*?vtQIQGr1S9p3B-tml2=rq*a^jwlMEE@6&?qn6uncyJl^Cg0$2rHPLm5;$CgsS z<@(eEE~TW`(EJ6g@>3f+AA|s}y+`Pm)kHj|%4#vyKQ4fao-j0=#2(+txnN-kDbk26 zG&LE%>O{_+oN75iD5t&yh9wC`+yW+M#FI_LaB{ptSQ8Yjc@z5M7Xju>ulGUY4yEON zZ83~bgWvr;kIrLe5>U7!61?6Vdc$T|drw1<=aOS;63Jp%jVOvve=lWwKY8fT+rxfF zXUD*K>Q*T^o&WF~4U(t1)Rv4#R4d*oY_(566)vz}PmPzp{CT+MIG+<#DKn$6b6b_QO3V51@kM zKisL}Cl$Oa2?ct)P);#+@qVj;Y&d>EHMq*hbue@s6I!6sZN@UtY20BFilpKF@g?eCc0DkHhP;Foth8_ThH49xL(d!I@g9gUI|CXn%ee@bfl~l}y?+o7_p1YLc zcg%FW;Jvrsn)m(i#9Z?lMc3CotE*q!{mGg66VPke^K6fQY2V+TU;H-oS~rh*pLr|i z>3g;X+Yg)OcH+4e&njx?+wYxOII~)@KjUivn&0=#S2KV3-pPfNt3H3Gyb)*zcCVG# z9lO^Uht+1_QepUkH;Ld<1yFxa__7O-r*JMz#l>{&mT8NE-EI+VN?XzpLf$1oV{ykS zv8H3j>mfz5=CA<*&f?D8HbJAE+QtlfaXxU`oR-I)%}yHvG6rT936E499_1EOaFvV@ z#V7H_=cRyGa^gD8NG>z{DiS399{Kq0MP`A_5>sZunsQBZZ5GKLqa_b`gHn)EB!o(1 zA5x4Co1LjXZdCK7-OkQ>Hhf;I{;4v>#6P>75|nkGk^{TAXIj3^6- z_2>w$NTg)_Gk}IRou1ZhfCy*h8*l~ySsEnqsp>xsD^03&t;GcOji4!2mK(|^C8!BM zBxD`2#5e&D-5yWibVSvSJV*6&2D}LP-XuaIK+}f1BzGiegK?)2Z4hDa$4u`+wH0bS zxra5O84Uu{HO#mZKzuXWWXrsFs%}@UlciCByLT?$zBoU% z;@h9yzT;kKAq3-&Wv(XcZ+ukou;TI2760*Nes89I5O9(6W?dz>PTxG8t*psbRb!yb z*6zyI3X8ku54?0(eeN~J?&AT*T(&hY#qn#~94>fJhn0U7)$&Q%{2QaK2@0JDOMq?O zS|BE}q-nqkM#A_|1od#dC9!t_^6-rBHt^y^74yFl1lEv3N@U%kRrV)Vans!X6%K82`m~5 zH_W;bS}A8-Z3aum2SY6|N*uM_JoO=LI){7Mwh1Vuc;m=V3@a@rKpB(VCLJOXa!BMc z`#9tw#|wDvDb747%-_oso>WESJc*nnp*+s`?HdVGvGfRG1c2|$;FXOZS#{WUzh!I z`#){}+LJkcX}Rn&*uRk-&6nM)Sg2S$x>9{0Tfgf;&Hb9$({pXht{qvAZ?>c`i+NF5 zzqog?GUM}O@ydeo2c=83yB7nCZ)d6xd`bU|`zg2V>B(?Ce@GI9W6)g5pG`HAU}MBM z4atobT+x7*H@RYFGM0Dx82|8`Jb_&QOXte=z^o1RWR>ax!UbqIu=sSczmI zCe#R{MVzd<$^fYi%vzyYevm|8?g{*K!WSwR%Q$J{#q^96HN&D@+ zDO(#PcbZ_>e2LL9O&iW3*ilVO=vG{-r&IN9LOyHA9Z5n?*F8Ac*D&B0ASUp;4Ofki zU(r!q9fJyo4iIxlM$$k|M~HtX4jt>RZc`(pGCr9eipQh64UR1VPIUev&bG|Y62$(= zc_p06jH^0>&#iGj2KfaBd;o%6N(Qnb*U%^s{~CU37AgP$PuWbj>1*%yW3PGdgM|;4 z+8>>Nc>d}3wk%hA%X8BMG5(Q-Bl8W5`ngt$ycwLoes5x7VtzEUuPam6{nXWyb#9vp{?G3mRB6p`DcVt!EwBDK zu*N{^8x@~#{X;`rCrf?BIy=40S7kKReij{s0aOhYKE~IQD3kiWLmk~IMXHp1S*6P=F~a$EIHFThpOC(qs-~d@4F8V z0BKrDPy5e2iT7RH?cMv`?|%2&?)~ecA_Ir#um0(N-w(EN+~3g;`LRn8{^4Kn95>Dh zTsJ51f})=vP;@JJiYxn--AeXWb*tE0-K}PCO}7SbRljyX*R5k^)&2SbL$`s&HT}i` zQ@3fLsJn>$*7lnREZr6s*Y#TmY~8j2d$*nEyf#rI=mkT6@qnY-fqbK25{hms1hZff ztb*;fQm`Y%Boqq{!E}xjoVT^PloFv7Ic2xCK92sOopM2mu!8-o6spj}>e~jP2Bnq= zF0{2&s1@p1tRCN3c3a==RB+ysx3UoFE>&>*IHBQ1PFU_OdrK)Lcb5yBg%vNVx+{b& zLL=Ul!d77=-c|VBgm<;DO=!luM%XUw5L#YTcP|lk3av=xF&sOhc~o)p%jje22^umSI7LYwde-VH*#uo3U&!d_t$ z-Yc#s8~24a9CBarieAArBo6j@JwaE%8w~pV=K{@?a?$H|`#s)fm)kG6P8>OW$mJD$ z!9eTJs7H-_#_0C@2ZQdQZ_pp;#Bc3k_s|e(BdQa;-XYK6(3M6-MlZO7Zco2E5Xh)| zecpZn`KqJt;CbW~oj858y=Tw<_B{tXI`?IiyZu*w2K3_Y@EJYzz$bV`R13NT7Xld$ z6;yurfH$MMG$>xk@U0v^~6_XYxMSFti7b3Q>F_6Hi38Ev+NXhypk6{Ti0=+3Z! zcFz0Go%iXq7>-u7Qoed%Q1JF=lxKbZjK&9u&u~D<=m*@Fdpzg|O1S;O z^JvcJ$=G&xp4fl<$kC2HJ-d%~^c-wI4JfLDSBAV9-C6qju-K2eAM*8F@LuWh`Y&cQ zgF`?SU#R?iFgO(0vT7AFTcxyCfNa&pH5v1fqwSr$J8~^5#1;%zT#KMl52y_Lg5HdJ z2v`vSbT4~7!@)s-$v4385V3%_9|(#xz3A=DM%8^ZmW&bu6N}K2hrur!5E|j(AJV*z zb6&2S2hCCle7Ew70wXf+8T9x1&b5o;pcvZS;lJqa_X#dQ5{bhi!|JUruiJCpH30l_ zpYyuAAx_r0zZ1yXqunFtBQ!+qC-@r{a*KKEs!Y#Lvn2OR+YQM9hk z@IwJwwJw*~hF3OU)9#~okYg~vcdj~AX@ks zXb~ug*B=a^cc3YNS%79R+a*(BKg1)*IDx+aExD`QM<&~i!=s0<9l6>*YqTV-$73fG zRqNs>-#qjBnR`|1Qr6>%6Hlj%rxU8v40}ANm3wlC^D>ggxnK@$g4ulv@LmKl`aohA zJ_7$%w)SN{P|6py)kkAr@W{w0C}>XQnu3yk0lie68Ozgc`+D{q>D=3~uV?>}!|fq0 zh!3sikR!KZq=eSs;6VQ!wMd`ICAQ*?%$Ib>W_FaXv=OusgZZ}Si! znBqb*0g#+pRp~;^D7?502j`j zo)x0?D?W>CfdH|Z0RrfMQ!#EBGmM+Z%!wtdQr6Y?tOwq0`%cF@9e;T6?uz9411aOd z5BieErxU8DKRanEw8ADWjL10HvHWuA%Pj|FWgh`lGlX~!c@@MSba8cDP%g6Q;t6Wx z>|RAZC#rd%hevT5H7{|Om8UrnEM;i(QMVZIy4(_@){pHJ3`Vo77rQ%=)w6ET1y^to zB-b_Ue{Og%2&OL(6p_^$QeAXycV+mC*uk{4GWvaIGwS|9w-Cr!i0y0hdcYL1jWVM< z=)J;XKs=@BzEp^GMvx@12iQo5y<#u=d>(&+9SE*+b1J2}c-B&KV`OwB_Eg++>zR~g z<;1$AWmQ7A>eE?M@r4dTMA2?oJoVM6sHq0pzdlaBG@j^v4_l?S}%0K3} zDm_GnJd#?C2GT}SFzT@>Q}jMDE)YkX!Dz0 zSQDfvW{a$I=6heitCaef6!`ip9v})-F7A$^QzS91QIk;v0R{pYgD>FoV|&0xn^6fqPf$FG z{EVgt1n!(yr0+E9kOK}qqhS*VVmK)J&Y>3AIu^-jhQM<9E@zbB_cCf$Dx(nw!CVS# zmvUtvwjGN7Xp;p)hy<=60L3Y(8t)(L|GIzF5K%@>e%R0)-5cE$TNCSwwI^-0lXVfz ztkD_u-aavHteq{cjtQ@wzjZzycy-|I6Vt_O(^WN5RkUkNKWnl_*TDC-qy;u_iZiO7q2ZwTWuz0~3p|AY zb8NEQ=osyYdaiX6V_}XAU*9roE>Bdgo-Cf!CN_2?Dh{N~2NSx3=_32py|dP`nDgtF zh$?L_iD{xWqnbG-XLEh5GDOx#OJb$*hWo0<&yrkPGi(C408B4wU(!JnL0SlIfFI#` z2vIL7`4t2n`eezgB<^dndz z_o!EKQSQ?-m{tDzn#cCZI|eY+`D@sFRdO7?`;v+q z)ru{_O8N7GbI}!*TOobba>@LAR4cc*@cjZ0!dlWP(jpa!wU{SR67>Zl^AG<9k*_HV zMVv5~7jon!B(5W_?jk4JMiq-qC1l_%o6m?(ZdX9#*{6W`K>0etgJdbgInfB-+K=FC=~V&51jRiOf42Npi`16wN+e2wVbV_&dXKJ4kE)F z8ZDxQ+NO<2^dZRT+S>Q-K6&Uwm-qtxRP_z|{P_uI>?CV5qeS*FgXTBcNgs((-a7b{Ym4kk7p`ruH) zdio!giX!tT8qQu7YkyJf^u$~dS5`k~=8TpbD@IpDH^gdF zhU%+(X0^6x-Ob>%w(7^lUySTYJ4(m*jqQu=dF4=~eb!POZNK)VH|mm>WfRWHrEgVa zQ8`{855&ERro9Q*zWdhwb6U>f<{#=f$Ewe1Vcpy6cyrRyI_aHy?hpGGw9=7q9k_2j zNUf}z({c8a@v^b9SW{d~*;ZaXKpk9pTbS0?r7M@sR5m3mn|@T;GI|ia^)lkhy>EaU zUnw`aa^l3q-o)A`6Rth?t!-&r$$0r#dF+Xlt#+buX62K~l~3NYJt;595?9<2*CZOY zPPP4M$9o-jeKUKXP40a*wXG+y^@|CI8*s&P)pJ#?56qm-3dS*3|61d%#y8sDJpB6M z$rY*kt*MG_Df9M(ZaY?}&ML8nnDpEOwvZnFVH9Cr;d>GcBFs_>uS!sos00&^T2Q^6 zW4vIR(jbqCQJ=}v3OeNJsh(HY%fpytfWbx&1CtR(6a$%=@HPrgp$Kmi-e$augc8Am zw;3iUE8dnXw#KrIt}AO?2|aOKQsqGe&n7WKORCYp;IIfv3zEWA&x}k9AjlA@TOS$8 zYT^pJIT2ve|hN3ER$EY+I_lNCe(hBe#;m?LubirjYVsKTA#ikAyyU)P#mCE9CcNM3iSCU(@zu@;Ms7(Ho8mh>V;LHGwg4NI z5+V@;C{niq1Vop-VXLTXp`v*`B}O~oLV#U6Z7Ysw{CF-0M&D@Pw5~jD zaYl+Bvnc9O{qv?gzbHT}UPD22l@MRXdE!6{ah`Fe#N>snxqfEZz6=$m^5`kB{qpGL znCjY>6T0fpc8y3NTd&)Ih3FbD@F0JdJRiTP=7?LA*yeKR=c2MY71Lfv8cK-ho_KG;H1IIx<04uqW#MQIKM;%{fLUSMtueT3x*Jg zH_}vX+zquy;e!VIj1DR4Fw`btHTClq#dU)U=_BecgglE~;u?)Gt%}gQa3WSLCvK)- z3xZIQ%#@3R{a#n7S>niLlQA~GtjV|;hT5#jm^gH1G?p4UhzFF!6L^gF8`kHf8rDOrEsOIbJnZ73)jc z8Y3E%jc&bXtRY7C%Gi~7MXIzpvOjGpjp>t?+W5J9mX@@&WV~psD0Y6@x}1t{xo50S zo9tv01kcrct2rJ_RWwhVTGE#NP|xK`SfdA$rkY6Ghs~=~mNj?U;$5H~PsR3(eIaRe zO|&H~YZAIOzqn^v6QJS!nZD@Ba@9N%4e?Z35+5T{ptr+Re-1L|P?Pu&#y{Bd1Xcio zLEok(!kyP2yub^Z5tV`i0t3AkK4HF>hXx+45yCB|Rmg7;H`F&Y9{Ab-QGvQSn*osd zB5AOUKwJ$_)&i7Oj%Y^IZ}7j3eTgA1`wi697HAnHQOqXetSpkbLM2;sUI$+0cEg-r zS|KTttR}9eU=0PQ5o9!?_Z(;wa4avV5ejBytfN^;k8Lktit>oaq^&X51oOfPNP_4W zk&@p~WScgL=}ck_*>W^cPWpSQL1H+BEZvfioaN)sj6D-?N;z922Evw5(zq0unsBX| zte-4PY(A1Wdg6nNiJGTU_EQPdskE&FXlhv+xiVu}N+P?-`ar3$8qy7`CmqR#CuSP9 zB^$O)H|&^pbS4{i+!A8$xb5!JyBGhgDS9l%kDW*~?6~LXWOe2^h2HRsA3HjkiRgWY zX?L^v{iSMz^AL~dB%{Ssh|JgMEJklXOV1Xgv@)WACUbp+@e3XpwH?2%Q9I0x+K)p= zdF#?Jqcz*VG6pyuz#9oB08zpXrRS0$9z;LnU6|N8(&EFbhHb;y=@s8adI7zm0zXGm zU>5?YRUnJDjqe}ZAM1MMaLQ6UV`)iRT2hu(pav4Lxo2y7!|`U>>t)}nmt-z5 zl9p{L%Z`L@2ippXWV4SeA%PIi28TtzNCe=Hlp%f_aq*8RpcwXIiiZkB7r26*K`XS| z78Jffg%_rV>@sc!TIB_h?La|QA_t9DaR)UBl{P2tqF^_IjHRciz?ZG3M{SXFvphtxHz;6>jtI9nNdbX$f#q4eNof?&*aAlo{3iEP)vk|<%2KMb5ACI~lhgLP ztL>5XDOKHvOX??Y*zuKlOIF_0DX*GyJa&HWD-~h&Vpv}w=;e$zdEO$dhS2#W zcn6IhCeuEmgA1NQ`F=zTyF5{Q_VmI0u<(JwH7hBM=nF>^)`wL=+VRgF4K{`#=?vKuHUz<2Nl#8D z!Il^{fX}oAOXWPl$hJw-!fmp!C22eSYFYtxX3O31K3>&2OyQ0XO_^+P_4k`K` zKVr-RV?c^nXdbc5|5n%x^@^pS&2lJ9NT14{yg3g0(e_*_Q?AHv%ghUfXf1o-WuoG_ z(_ECl{_UV$PR$$hm>){%h^_!TG+_;}LmSo=GGk%g3lQr^bYIq8;$>eIc)OIzE%sH( zvHTm~p5L>`8YxA@59VLJ#!4YvSuz!ujdt8BAp zuG45TW8J^|cxQW8*V-QF$M<#|Y8PKZYkA{BmVmoL&IK$X8%1k~T%x6@I7$_XOQYYe zj6TWZWL5;d(!N9(k1RoYP!Q$&_uZJQ<6hMQ%vjq#3YRqJd~ z*|jqf_+X~BW~#3%x?vhM-Fzm#Jmp+Dt!w(R)frLEDb)Jie7dA!+&|`@I2ZM&N}h=9 zn=>f&?fk5>YW&RDnfS((vnjF{CJDVI5`;W<>J86C{kMGgb*tw}Ia|qG8CS71QQkay z?%mq&tbAwXcUs?RP3-PUtvzvd--lXry0~2~*pw&KwEed^&bMekobAD&<@~xqRyByXWqnPHa6gtvfpFEFJG2>z;8gOFEbR z$hmw}54(&9V#JM|qdQ|;?;Dp-v`x1C-XVC2%-HLa_PY4NlzsJ#{fVUgiIjcQjD1(q zzU!{zZXjhp7}+yxvPK=z!0m$*%2ZYJ#JMTQlrZ(wy{t6 z)+BXR5#?x4#2bBTR%eQwh^~vNW0xj0_jRl0l-$y_AFGNY!iz^{jhM{X34qzylu$K& z`pFv3x)dI2dS}Mun6;MPu1Z)M==W}(#&05{$Yky0vB^D&noTMD=Beirrd_{;BO40* z@)I3rEuFD6B&8~tM3R#r_e^Rg&raEtut81n?Zq9-)!(z)J8IS6t5qYOWB<@X-sSr?Q%L0% zQHz?jm*=8B>u1`UwT2W9CC?VuT~dqZ|WDJqsY6q%Nx2_!~A`6?4{fW zDcu&Y5ke6`Pa(cvDVG-va*HqUP?~hDC3WJK*v8GW_!)%`?NRS8we7%%p;6 z;UAJzlXHJ}f;Z+1CYb-M--3zDI=z>YPMVzFCDo`xBGbQt=I$uO{dkFAL{O+76%QaG zRN0gDG0j`(b8hadfnfm}o8;!+(D^;}-8FA-nRI{Gk!sp8y?n>_)W2Y6{}(E&`3(OB zF@m!5#svvQa_e7|_fI|XsoNY8qp#L@+hM7G8Z*;XVJTMdxzt2*r5s`G|l3M7uDE8YF&|HaED7a1m zF?di>2Jk9y&K1*?M?oWv>syFG(1*Qu+3n(KZ8b#x8==uqY}tf&+Ol2}=ck^!dpcz~ zlF%KI$~mUBHM16bbi=jKjAcpEvLwaqJhNJ3r2RL(bTe?fJ=&kt)@F+&bmjSKZ>&oa z5~LbBr|UZ5*&97EetPV5l3<#u*^+W@Nm{l{!EUrGq1!cUa3)HgNg2*0RA(f$Iy#(t z2<%wP(s_?URnB6@E^^Jut!Aicu;TKKPl^RWKIg@MIi;$zVhwkSC(DU@_ELX}i>21#-n(uhO~hsg5IWD6agqol)2 z88thKkl}q9CEUgqnBK9rn8V1*KJB$ZP8<6nkWya6f)mEKl*%ZIPRXdtU%7q zu9)+c?lDW0pDiwl4!^P`Zu{!abVcQDPrUwB-|w`54Yl3bH#t0Y?48T^ns=tl%45fF zX<|Hh#mcJ13Z)_D#9p zIe6DLUBBnnb064}j)Rf*w5jN3arERX6{ANYyWwFZH=um^Sun!mjbn|mw)oPNy&gV5 zOKRh1f9J~0x~Oooal(TH$%zRCZ1HumgELjF$*R_gV5(~Uv}r@yT8svD`pB_SO@yCo z;*2H>Jf=B6_SIT-tY}*P+zJbTjpYx^}I_6bb`71dELXW@|w>ZcEY*k zHKKxsZ*eqN3F@~sGI`p-?1+nL5@m*Q1a8^%Pp-k!X`Nyq6?^(}%%7aXanI*0saQao zzQtHP z5tCpWDT3~(TJA?!CokhHxqmlYB-n|495K`J@5hfpQ1|8ZmvW0Bak_9(*qrZ={Dlz< zpp880Ti?=rc{wJ-vae1~$-f1s+=_S;TbDI#8LcQpT1ht7 z#xQ1{jdm(*h9=Vxwg7+r;9ouifoL$N<@{%2K(YLFL0d5DEKbM}`)p3%W3WZ86@u7T z!Hqb=j#s%LWESZO=L8>U4@>AU);Clm&Tz3<5_Sru;o`-#t}-6dDqtJ1Gwd9#kYeY! zNCo`#%3joki}T{zH~Fx0jF<4flJma|ct2*9iL?#A`?x-iRIWp*8qMjOWoUu{G|OKK zs7EWpC5%#(;G3KCW#m}c6fTihBQDeU99ki_H&XVis7<&`E=$xkT-KMm%FTf(I9U7~ihW_Z2mFFDQba;qv_VuW0MIH4uO(%}c00c~YE zb@3u*PP|0HcPStp6inBDf>%cC_g5GqY7I3 zS;dHPo?=8gIId*coeb|0AEIE^Kj5n5^LhX&p{f9`36VUHKYNqGOFZfjK7T6dc$T%j zw`tNHcTX(;w-?^qBx!ek!(!*>b(w^B*5!YZR)@*ifI3r7jZohSg0gGFukQM9QP=Z| zo$6ihEt{;r?VC`&Hu&DMUx*)~F)@Q6WXzg$00&p7F{|UrDs?0!51AH#Jf!&X39hVF z1YJ9f>U50>R6IR{7aEsJZpl}uWm3;DT|!1hH>8mJHmnN&nNmNX;CmFfDL6}k2SLWb zPINM)#9cVUL;YUyXOv0$o{XlS30F+$DzLE}qC6EH1D8BaU!V#kMKSPYltY6P>VXS5H=i+H646r6*(*VBps`Z&br7ko00mz{kkMw< zff&LUPhc=FK__W4p44 zN^*EFmY{P_5xNWkbw8!6Gk)DOuCrzck=F*V?weC+)Y~Dy-8emZIIB>LK&ZJbsf}pe9C>kw_3e&nuoLxdK97lRma6QFTH+gVtDf8)Vjo~ovG!!Qsuj&nzYq9 zXW&e>sB?7NjIkzZtVtPNGsdQ*v1ww@v~e}8TM$=^%i;8nBQYo5SU+uF@pjjw2K(phfF;ncsw6*pVHHWh$^Q*q~EDT!9 z*GxLz4Nf;~yXlR!-|W9zAK8biE27?e#>%v_EV7s6{EMSsj(aECU+hFR zPo0|TNYw0&8b2(qOgomu*G-iDhpIbOZ`CC0HYOb#qX*O0%Gjl(wIOOu*Bwt0c|?$? zJAU^9f~Y#`jqOQTU4R3LF3iFBo^ccq-W01Vvfa%l8WY+r11e0RJz{`ADLiAxi{DZ^wqQQigv z*iS9am^Z$6;@Im4V2GQk*_5o=H07AC*^#cUkB5@gYoZ3Zjgol9w5d5=WVvx_^i=fZ zwP&yHol_{)`*Qf%JFTmp(V|q z?yz@Sl|Qhm5&uD1@ezgc2Tk_Q?aCi)S0nyI^`@ho@`pQlgb8)=(Y4A%ne%9iGSQ-@ z_y+wki!!mpbj+Yk8q|o-+s=u#0~f+SjAH|yfBi|`%C?whnb{6OR_P&LWicJ6EabI8 z$9;wjKMLry!P_i+d9nNs#gTGZmARz{mJrEd_ zB=}uX$|6gnIdt?895`IK&u;ruyAPdg&vNH4_ul95>cpMqICXF%4EwVyfpt6^$k2fk zFC)0ifamdfEKaZ{(E9{>QGbS z=&Sd<7noWabHC=h<$LwQy_yXZ$0qCUJe{i9@LnKmy!ZqEbjk7j4myMNkHOS88iD&l&Ys2M{wGC&@WjuFH@!eNkJC{eN z9B$pjsf492W!V5f91Ul6kiB5W;sWbrvER5ddga=eBB~F;f&|B(AA3F?m@aLa2u#)A zd7kw3S^t&I)5c}+u(_>_b-Zd$l&webGv6}D1F5p6(IQ-o za}3r3v+ag&)OYPdMERk`e!DJ7(~zp(GhIQ7ePWx8W5!pCB5HQS%C#*q+siv)#5i{A z^2CNo`<>1AmaLz0PCfTt+1>8s<|A~#aB1wxD2^DC3E|}E8KjIK96K0S#oZ~#iV2t- znr0kpla941$9kMX)){UXMh!E%>ZGnZ=813kk*@K<5;(kk3MS1|OuLokL!Fs6V9TR` z4FG(hvYqGtlGnEvDgV;MBZS{Ip4r+Nj;|#F|V@_{MSVJK<*zSHB_Cw164YH%Vf1Lz>{4%iF-x@!c7#!2l8EE zA7I`4j+DjlIX9q?o_qzU#WT3Phao(J-D0E4_f=I;GN0NgIlSY3%HCo45}rav*dx(XwuA8a7$KBiirbflFny8LZfBt(v?jQw5npm z1GUz!dC& HY>@vCmPIpc diff --git a/harness2/cli/apply.py b/harness2/cli/apply.py index e4bd38e1..94d55392 100644 --- a/harness2/cli/apply.py +++ b/harness2/cli/apply.py @@ -12,17 +12,153 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Apply command placeholder.""" +"""Write a harness a run adopted back into the project, after showing what will change. + +Only what the run changed relative to the harness it started from is applied, so edits made +to the project since then are kept. A file the run changed that has also changed in the +project is skipped unless --force. Only the executor's harness files are touched (for +Claude Code: CLAUDE.md, .claude/skills, .claude/agents, .claude/scripts and the hooks in +.claude/settings.json), and they must have no uncommitted changes, so git can show and undo +the result. +""" from __future__ import annotations +import argparse import sys +from . import _adopt as A + -def add_parser(subparsers): - return subparsers.add_parser("apply") +def add_parser(sub) -> argparse.ArgumentParser: + p = sub.add_parser( + "apply", + help="write an adopted harness into the project", + description="Show what the harness a run adopted changes, then write those changes " + "into the project. NAME defaults to the most recent run; the task defaults to the " + "last one the run finished.", + ) + p.add_argument("name", nargs="?", help="run name (default: the most recent run)") + p.add_argument("--task", help="take the harness adopted for this task") + p.add_argument( + "--mode", + choices=("parallel", "sequential"), + default="parallel", + help="take the harness adopted in this recursion mode", + ) + p.add_argument( + "-i", "--interactive", action="store_true", help="review and accept each file separately" + ) + p.add_argument("-y", "--yes", action="store_true", help="apply without asking") + p.add_argument("--branch", help="create and switch to this git branch first") + p.add_argument("--dry-run", action="store_true", help="show the changes, write nothing") + p.add_argument( + "--force", + action="store_true", + help="also overwrite files changed in the project since the run started, and apply " + "over uncommitted changes", + ) + return p def run(args, ctx) -> int: - print("not implemented yet", file=sys.stderr) - return 1 + adapter, run_dir = A.open_run(ctx, args.name) + task, adopted = A.adopted_tree(adapter, run_dir, args.task, args.mode) + base = A.base_tree(adapter) + current = A.project_tree(adapter, ctx.settings) + where = ctx.settings.path("paths.harness") + + delta = A.changes(base, adopted) + if not delta: + ctx.log(f"The harness adopted for {task} is the one the run started from; nothing to do.") + return 0 + moved = {rel for rel, _ in delta if current.get(rel) != base.get(rel)} + todo = [(rel, kind) for rel, kind in delta if args.force or rel not in moved] + + print(f"Harness adopted for {task} ({args.mode}, run {run_dir.name}) -> {where}\n") + for rel, kind in delta: + mark = "!" if rel in moved else kind + print(f" {mark} {rel:<48} {_delta(rel, current, adopted)}") + if moved: + print( + "\n ! also changed in the project since the run started: " + + ("will be overwritten (--force)." if args.force else "skipped; --force overwrites.") + ) + print() + if not todo: + ctx.log("Nothing left to apply.") + return 0 + + dirty = A.dirty_paths(ctx.settings, adapter) + if dirty and not args.force: + print( + "These harness files have uncommitted changes:\n " + + "\n ".join(dirty) + + "\nCommit or stash them first so the result can be reviewed and undone with git, " + "or pass --force.", + file=sys.stderr, + ) + return 1 + if args.dry_run: + return 0 + + if args.interactive: + chosen = _review(todo, current, adopted) + elif args.yes: + chosen = todo + elif not sys.stdin.isatty(): + print("Not a terminal: pass --yes to apply, or --dry-run to only look.", file=sys.stderr) + return 1 + else: + chosen = todo if _confirm(f"Apply {len(todo)} change(s)?") else [] + if not chosen: + ctx.log("Nothing applied.") + return 0 + + final = dict(current) + for rel, kind in chosen: + if kind == "D": + final.pop(rel, None) + else: + final[rel] = adopted[rel] + done = A.perform(ctx.settings, adapter, final, base=current, branch=args.branch) + print(f"Wrote {len(done.get('written', []))} file(s), deleted {len(done.get('deleted', []))}.") + if args.branch: + print(f"On branch {args.branch}.") + print("Review with `git diff`; undo with `git restore` on the files listed above.") + return 0 + + +def _delta(rel: str, before: dict, after: dict) -> str: + body = A.unified(rel, before, after).splitlines()[2:] # skip the ---/+++ header + added = sum(line.startswith("+") for line in body) + removed = sum(line.startswith("-") for line in body) + return " ".join( + s for s in (f"+{added}" if added else "", f"-{removed}" if removed else "") if s + ) + + +def _review(changes, current, adopted): + chosen = [] + for i, (rel, kind) in enumerate(changes, 1): + print(A.unified(rel, current, adopted) or f"(no difference in {rel})") + answer = _ask(f"[{i}/{len(changes)}] Apply {rel}? [y,n,q]", ["y", "n", "q"]) + if answer == "q": + break + if answer == "y": + chosen.append((rel, kind)) + return chosen + + +def _confirm(question: str) -> bool: + return _ask(f"{question} [y/N]", ["y", "n", ""]) == "y" + + +def _ask(prompt: str, allowed: list[str]) -> str: + while True: + try: + answer = input(prompt + " ").strip().lower() + except EOFError: + return "q" if "q" in allowed else "n" + if answer in allowed: + return answer diff --git a/harness2/cli/bridge.py b/harness2/cli/bridge.py index ce364f59..7a0879f7 100644 --- a/harness2/cli/bridge.py +++ b/harness2/cli/bridge.py @@ -51,14 +51,26 @@ def _endpoint(path: Path, host: str, port: int) -> dict: def _arguments(p): - p.add_argument("--model", required=True) - p.add_argument("--host", default="127.0.0.1") - p.add_argument("--port", type=int, default=0) + p.add_argument("--model", required=True, help="provider/model to serve") + p.add_argument( + "--host", + default="127.0.0.1", + help="interface to listen on; use 0.0.0.0 when containers must reach it", + ) + p.add_argument("--port", type=int, default=0, help="port (default: the saved one, or free)") return p def add_parser(subparsers): - return _arguments(subparsers.add_parser("bridge")) + return _arguments( + subparsers.add_parser( + "bridge", + help="serve a model as a Responses API endpoint for Codex", + description="Serve any configured provider/model as an OpenAI Responses API " + "endpoint through LiteLLM, so Codex can run on it. Needs the bridge extra: " + "pip install 'harness2[bridge]'.", + ) + ) def run(args, ctx): @@ -97,10 +109,12 @@ def run(args, ctx): ctx.out( {"provider": snippet, "environment": {"HARNESS2_BRIDGE_KEY": endpoint["key"]}}, lambda data: ( - "Provider configuration:\n" + f"Serving {args.model} on port {port}. Add this provider to .harness2/config.toml " + f"and use the model as bridge/{model}:\n\n" + data["provider"] - + "\n\nEnvironment:\nHARNESS2_BRIDGE_KEY=" + + "\n\nand put its key in .harness2/.env:\n\nHARNESS2_BRIDGE_KEY=" + data["environment"]["HARNESS2_BRIDGE_KEY"] + + "\n" ), ) sys.stdout.flush() diff --git a/harness2/cli/chat.py b/harness2/cli/chat.py index 239d2d76..f7e8a23a 100644 --- a/harness2/cli/chat.py +++ b/harness2/cli/chat.py @@ -12,17 +12,109 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Chat command placeholder.""" +"""Talk to your agent CLI about this project's harness2 setup: tasks, runs and results. + +Starts Claude Code or Codex interactively in the project with a guide to harness2 and the +project's current state added to its instructions. It is your own session, with your own +CLI login and settings; harness2 only adds the guide. +""" from __future__ import annotations +import argparse +import json +import os +import shutil import sys +GUIDE = """\ +You are helping a person use Harness² (the `harness2` command) in the project at {root}. + +Harness² improves the harness of a coding agent -- the files the agent reads on every run: +CLAUDE.md or AGENTS.md, skills, subagents, helper scripts and hooks. It runs the agent on the +person's own tasks, has an improver propose harness edits from what those runs show, probes +the edits on the same tasks, and composes the best of them. A run never changes the project's +files; `harness2 apply` writes an adopted harness back once the person has reviewed it. + +How you help: +- Writing tasks. A task is a directory .harness2/tasks// holding task.md -- the + instruction the agent receives, written the way the person would really ask -- and an + optional task.toml with: domain, split ("train" or "test"), workspace ("project": a clean + checkout of the repository at `ref`; "files": a scratch directory seeded from files/), + ref, deliverable ("diff", "files" or "message"), check (a shell command that grades the + result), timeout. check/ and rubric.md hold grading material that neither the agent nor + the improver ever sees. Good tasks are real, varied work the person wants done better; a + check command or a rubric.md makes the results measurable. +- Reading results. A run lives in .harness2/runs//: run.json (its settings), + events.jsonl (progress and cost), adapt/// (summary.json, the candidate + harnesses under harnesses/, and the selection/ and reflection/ notes) and library/ (what + carries over between tasks in the same domain). `harness2 status`, `harness2 report` and + `harness2 diff` summarise them. +- Settings. `harness2 config show --origin` prints every setting and where it came from; + `harness2 config set KEY VALUE` changes one. + +Rules: +- `harness2 run`, `harness2 judge` and `harness2 doctor --live` call models and cost money. + Propose the exact command and let the person run or approve it. +- Never edit anything under .harness2/runs/. +- Do not edit the project's harness files to apply a result; `harness2 apply` does that + after review. + +Current state of this project: +{state} +""" -def add_parser(subparsers): - return subparsers.add_parser("chat") + +def add_parser(sub) -> argparse.ArgumentParser: + p = sub.add_parser( + "chat", + help="talk to your agent CLI about tasks, runs and results", + description="Start Claude Code or Codex in this project with a guide to harness2 " + "added to its instructions.", + ) + p.add_argument("--with", dest="tool", choices=("claude", "codex"), help="which CLI to start") + return p def run(args, ctx) -> int: - print("not implemented yet", file=sys.stderr) - return 1 + tool = args.tool or _default_tool(ctx.settings) + binary = shutil.which(tool) if tool else None + if not binary: + print("Neither `claude` nor `codex` is on PATH.", file=sys.stderr) + return 1 + guide = GUIDE.format(root=ctx.project, state=_state(ctx)) + if tool == "claude": + argv = [binary, "--append-system-prompt", guide] + else: + argv = [binary, "-c", f"developer_instructions={json.dumps(guide)}"] + os.chdir(ctx.project) + os.execv(binary, argv) + return 1 # unreachable + + +def _default_tool(settings) -> str | None: + for tool in (settings.get("improver.channel"), settings.get("executor.name"), "claude"): + if tool in ("claude", "codex") and shutil.which(tool): + return tool + return "codex" if shutil.which("codex") else None + + +def _state(ctx) -> str: + s = ctx.settings + lines = [ + f"- executor: {s.get('executor.name')} on {s.get('executor.model') or '(model not set)'}", + f"- improver: {s.get('improver.channel')} on {s.get('improver.model') or '(not set)'}", + f"- harness directory: {s.get('paths.harness')}", + ] + tasks_dir = s.path("paths.tasks") + tasks = sorted(p.parent.name for p in tasks_dir.glob("*/task.md")) + shown = ", ".join(tasks[:30]) + (" ..." if len(tasks) > 30 else "") + lines.append(f"- tasks ({len(tasks)}): {shown or 'none yet'}") + runs_dir = s.path("paths.runs") + runs = sorted( + (p for p in runs_dir.glob("*/run.json")), key=lambda p: p.stat().st_mtime, reverse=True + ) + lines.append( + "- runs, newest first: " + (", ".join(p.parent.name for p in runs[:10]) or "none yet") + ) + return "\n".join(lines) diff --git a/harness2/cli/completion.py b/harness2/cli/completion.py index cd942a9d..a2a3c023 100644 --- a/harness2/cli/completion.py +++ b/harness2/cli/completion.py @@ -21,8 +21,14 @@ def add_parser(subparsers): - p = subparsers.add_parser("completion") - p.add_argument("shell", choices=("bash", "zsh")) + p = subparsers.add_parser( + "completion", + help="print a shell completion script", + description="Print a completion script. zsh: harness2 completion zsh > " + '"${fpath[1]}/_harness2" bash: harness2 completion bash > ' + "~/.local/share/bash-completion/completions/harness2", + ) + p.add_argument("shell", choices=("bash", "zsh"), help="your shell") return p diff --git a/harness2/cli/config.py b/harness2/cli/config.py index 539fc2da..a2b164ab 100644 --- a/harness2/cli/config.py +++ b/harness2/cli/config.py @@ -20,19 +20,24 @@ def add_parser(subparsers): - p = subparsers.add_parser("config") + p = subparsers.add_parser("config", help="read and change settings") commands = p.add_subparsers(dest="action", required=True) - show = commands.add_parser("show") - show.add_argument("--origin", action="store_true", help="Include setting origins.") - get = commands.add_parser("get") - get.add_argument("key") - set_parser = commands.add_parser("set") - set_parser.add_argument("key") - set_parser.add_argument("value") + show = commands.add_parser("show", help="print the resolved settings") + show.add_argument( + "--origin", action="store_true", help="also print where each value comes from" + ) + get = commands.add_parser("get", help="print one setting") + get.add_argument("key", help="dotted key, e.g. executor.model") + set_parser = commands.add_parser("set", help="change a setting in the project config file") + set_parser.add_argument("key", help="dotted key, e.g. run.workers") + set_parser.add_argument("value", help="new value, read as a TOML literal when it parses") set_parser.add_argument( - "--global", dest="global_", action="store_true", help="Edit global settings." + "--global", + dest="global_", + action="store_true", + help="change ~/.config/harness2/config.toml instead", ) - commands.add_parser("path") + commands.add_parser("path", help="print where the config files are") return p @@ -44,10 +49,7 @@ def run(args, ctx) -> int: values = { key: {"value": value, "origin": config.origin(key)} for key, value in values.items() } - ctx.out( - values, - lambda data: "\n".join(f"{key} = {value!r}" for key, value in sorted(data.items())), - ) + ctx.out(values, _show) elif args.action == "get": value = config.get(args.key) ctx.out(value, str) @@ -60,7 +62,10 @@ def run(args, ctx) -> int: overrides={**ctx.overrides, args.key: (value, "config set")}, ) settings.set_value(path, args.key, value) - ctx.out({"path": path, "key": args.key, "value": value}, lambda data: str(data["path"])) + ctx.out( + {"path": path, "key": args.key, "value": value}, + lambda data: f"Set {data['key']} = {settings._literal(data['value'])} in {path}", + ) else: ctx.out( { @@ -72,3 +77,15 @@ def run(args, ctx) -> int: lambda data: "\n".join(f"{key}: {value}" for key, value in data.items()), ) return 0 + + +def _show(values: dict) -> str: + """`key = value` lines in TOML syntax; with --origin, where each value comes from.""" + lines = [] + for key, entry in sorted(values.items()): + value, origin = ( + (entry["value"], entry["origin"]) if isinstance(entry, dict) else (entry, "") + ) + literal = "(unset)" if value is None else settings._literal(value) + lines.append(f"{key} = {literal}" + (f" # {origin}" if origin else "")) + return "\n".join(lines) diff --git a/harness2/cli/demo.py b/harness2/cli/demo.py index e1a6fed9..d2b0b6ef 100644 --- a/harness2/cli/demo.py +++ b/harness2/cli/demo.py @@ -24,10 +24,22 @@ def add_parser(subparsers): - parser = subparsers.add_parser("demo") - parser.add_argument("--output", type=Path, default=Path("runs/demo")) - parser.add_argument("--k", type=int, default=2) - parser.add_argument("--mode", choices=("parallel", "sequential", "both"), default="both") + parser = subparsers.add_parser( + "demo", + help="run the method locally, with scripted editors and no model calls", + description="Run the real recursion and library workflow on two toy tasks, with " + "deterministic editors instead of models. Takes a few seconds and costs nothing.", + ) + parser.add_argument( + "--output", type=Path, default=Path("runs/demo"), help="where to write the demo run" + ) + parser.add_argument("--k", type=int, default=2, help="recursion budget per task") + parser.add_argument( + "--mode", + choices=("parallel", "sequential", "both"), + default="both", + help="recursion mode", + ) return parser diff --git a/harness2/cli/diff.py b/harness2/cli/diff.py index bc3d4738..98997cbd 100644 --- a/harness2/cli/diff.py +++ b/harness2/cli/diff.py @@ -21,13 +21,22 @@ def add_parser(subparsers): - p = subparsers.add_parser("diff") - p.add_argument("name", nargs="?") - p.add_argument("--task", help="Task ID.") + p = subparsers.add_parser( + "diff", + help="show what an adopted harness changes", + description="Compare the harness a run adopted for a task with the harness the run " + "started from. NAME defaults to the most recent run, the task to the last one the run " + "finished.", + ) + p.add_argument("name", nargs="?", help="run name (default: the most recent run)") + p.add_argument("--task", help="show the harness adopted for this task") p.add_argument( - "--mode", choices=("parallel", "sequential"), default="parallel", help="Recursion mode." + "--mode", + choices=("parallel", "sequential"), + default="parallel", + help="take the harness adopted in this recursion mode", ) - p.add_argument("--stat", action="store_true", help="Print change counts.") + p.add_argument("--stat", action="store_true", help="list the changed files only") return p diff --git a/harness2/cli/doctor.py b/harness2/cli/doctor.py index 88abea01..dbd20bf2 100644 --- a/harness2/cli/doctor.py +++ b/harness2/cli/doctor.py @@ -26,8 +26,13 @@ def add_parser(subparsers): - p = subparsers.add_parser("doctor") - p.add_argument("--live", action="store_true", help="Call each model; costs money.") + p = subparsers.add_parser( + "doctor", + help="check that a run has what it needs", + description="Check the agent CLIs, provider keys, settings and tasks without calling a " + "model. --live also sends one tiny request per configured model.", + ) + p.add_argument("--live", action="store_true", help="also call each model once (costs a little)") return p @@ -52,11 +57,12 @@ def suite_check(): if problems: raise RuntimeError("; ".join(problems)) - check("suite", suite_check) - roles = [config.role(name) for name in ("improver", "selector", "reflector", "judge")] + check(f"suite {config.get('suite.name')}", suite_check) + names = ("executor", "improver", "selector", "reflector", "judge") + roles = [config.role(name) for name in names[1:]] roles.insert(0, {**config.section("executor"), "channel": config.get("executor.name")}) binaries, models, referenced = set(), {}, set() - for role in roles: + for role_name, role in zip(names, roles): channel = role["channel"] if channel == "human": continue @@ -66,9 +72,16 @@ def suite_check(): if model: def resolve(model=model, channel=channel, role=role): - provider_name, _ = providers.split_model(model) + provider_name, bare = providers.split_model(model) provider = config.provider(provider_name) referenced.add(provider_name) + # each raises ConfigError when this CLI cannot use this kind of provider + if channel == "claude": + providers.claude_env(provider) + elif channel == "codex": + providers.codex_config(provider, bare, role["effort"]) + elif channel == "opencode": + providers.opencode_config(provider, bare) if channel != "command": models.setdefault( (channel, model), @@ -83,9 +96,17 @@ def resolve(model=model, channel=channel, role=role): ), ) - check(f"model {channel}/{model}", resolve) + return f"{channel} · {model}" + + check(role_name, resolve) elif channel != "command": - rows.append({"check": f"model {channel}", "ok": False, "detail": "model is empty"}) + rows.append( + { + "check": role_name, + "ok": False, + "detail": f"set {role_name}.model (provider/model)", + } + ) for binary, channel in sorted(binaries): def binary_check(binary=binary, channel=channel): @@ -95,11 +116,17 @@ def binary_check(binary=binary, channel=channel): ) return (result.stdout or result.stderr).strip() - check(f"binary {binary or channel}", binary_check) + check(f"{binary or channel} CLI", binary_check) for name in sorted(referenced): - problems = providers.check(config.provider(name)) + provider = config.provider(name) + problems = providers.check(provider) + detail = f"key in {provider.api_key_env}" if provider.api_key_env else "ok" rows.append( - {"check": f"provider {name}", "ok": not problems, "detail": "; ".join(problems) or "ok"} + { + "check": f"provider {name}", + "ok": not problems, + "detail": "; ".join(problems) or detail, + } ) if args.live: for (channel, model), spec in sorted(models.items()): @@ -111,11 +138,18 @@ def live_check(spec=spec): raise RuntimeError(reply.error or "expected reply ok") return reply.text - check(f"live {channel}/{model}", live_check) - ctx.out( - rows, - lambda data: "\n".join( - f"{'ok' if row['ok'] else 'FAIL'} {row['check']}: {row['detail']}" for row in data - ), - ) + check(f"live {channel} · {model}", live_check) + ctx.out(rows, _render) return int(any(not row["ok"] for row in rows)) + + +def _render(rows: list[dict]) -> str: + width = max(len(row["check"]) for row in rows) + lines = [ + f"{'ok' if row['ok'] else 'FAIL':<4} {row['check']:<{width}} {row['detail']}" + for row in rows + ] + failed = sum(not row["ok"] for row in rows) + lines.append("") + lines.append("All checks passed." if not failed else f"{failed} problem(s) to fix.") + return "\n".join(lines) diff --git a/harness2/cli/human.py b/harness2/cli/human.py index dcd7ea03..87fce0b9 100644 --- a/harness2/cli/human.py +++ b/harness2/cli/human.py @@ -12,12 +12,76 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Human selector placeholder.""" +"""`--selector human`: a person picks the final output in the terminal. + +Only the blind final selection is handed to a person. The candidates are the same anonymised +files an LLM selector would read; the reply uses the selector's format so the method records +the choice exactly as it records a model's. +""" from __future__ import annotations -from ..core.proposer import Proposer +import pydoc +import sys +import threading +from pathlib import Path + +from ..core.proposer import Proposer, ProposerError, Reply + +_TERMINAL = threading.Lock() class HumanSelector(Proposer): - pass + """A Proposer whose select sessions are answered by the person at the terminal. + + Domains run in parallel threads; the lock keeps two selections from sharing the prompt. + """ + + def call(self, mission, context_dir, label, **kwargs) -> Reply: + if not label.startswith("select."): + raise ProposerError(f"[{label}] the human selector only answers selections") + if not sys.stdin.isatty(): + raise ProposerError(f"[{label}] --selector human needs an interactive terminal") + root = Path(context_dir) + files = sorted((root / "candidates").glob("*.md")) + ids = [p.stem for p in files] + with _TERMINAL: + print(_header(root, label, len(ids)), file=sys.stderr) + for p in files: + print(f" {p.stem} {_gist(p)}", file=sys.stderr) + print(f"\nRead them in full: {root / 'candidates'}", file=sys.stderr) + choice = _ask(ids, files) + text = f"Chosen by a person.\n```choice\n{choice}\n```" + reply = Reply(label=label, text=text, ok=True, attempts=1, channel="human") + self._calls.append(reply) + return reply + + +def _header(root: Path, label: str, n: int) -> str: + task = (root / "task.md").read_text(encoding="utf-8", errors="replace").strip() + first = task.splitlines()[0][:72] if task else "(no task text)" + mode = label.removeprefix("select.") + return f"\nPick the final output — {mode}, {n} candidates, shown anonymously.\nTask: {first}\n" + + +def _gist(path: Path) -> str: + """One line per candidate: its size and, for a patch, how many files it touches.""" + body = path.read_text(encoding="utf-8", errors="replace") + files = body.count("\ndiff --git ") + body.startswith("diff --git ") + size = f"{body.count(chr(10)) + 1:,} lines" + return f"{files} file(s) changed · {size}" if files else size + + +def _ask(ids: list[str], files: list[Path]) -> str: + choices = "/".join(ids) + while True: + try: + raw = input(f"Choose {choices}, or v to read one here: ").strip() + except EOFError: + raise ProposerError("selection cancelled") from None + if raw in ids: + return raw + if raw[:1] == "v" and raw[1:] in ids: + pydoc.pager(files[ids.index(raw[1:])].read_text(encoding="utf-8", errors="replace")) + continue + print(f"Type one of {choices}, or v followed by an id.", file=sys.stderr) diff --git a/harness2/cli/init.py b/harness2/cli/init.py index 2acb46c0..c3626afe 100644 --- a/harness2/cli/init.py +++ b/harness2/cli/init.py @@ -12,17 +12,223 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Init command placeholder.""" +"""Set up harness2 in a project: pick the agent, its models and where the harness lives.""" from __future__ import annotations +import argparse +import os +import shutil +import subprocess import sys +from pathlib import Path +from .. import settings as S +from ..core import tree as HT -def add_parser(subparsers): - return subparsers.add_parser("init") +DEFAULTS = { + "claude": ("anthropic/claude-sonnet-5-5", "claude", "anthropic/claude-opus-5-5"), + "codex": ("openai/gpt-6.1-sol", "codex", "openai/gpt-6.1-sol"), +} + +CONFIG = """\ +# Harness² project settings. See docs/configuration.md for every key; +# `harness2 config show --origin` prints the resolved values and where each comes from. + +[executor] +# The agent CLI that runs your tasks, and the model it runs on (provider/model). +name = {name} +model = {model} +{command_lines} +[improver] +# The agent that studies the runs and edits the harness. +channel = {channel} +model = {improver} + +# The selector picks the final harness when several candidates compete, and the judge +# grades tasks that have a rubric.md. Both use the improver's settings unless set here. +# [selector] +# model = "anthropic/claude-sonnet-5-5" + +[paths] +# Where the harness lives, relative to the project root. +harness = {harness} +""" + +COMMAND_LINES = """\ +# The command that runs one task. {{workspace}}, {{prompt_file}} and {{output}} are filled in. +command = {command} +# The files the improver may edit, as glob patterns relative to paths.harness. +editable = {editable} +""" + + +def add_parser(sub) -> argparse.ArgumentParser: + p = sub.add_parser( + "init", + help="set up harness2 in this project", + description="Create .harness2/ with a config file, a tasks directory and a .gitignore. " + "Asks a few questions; --yes takes the defaults.", + ) + p.add_argument("--yes", "-y", action="store_true", help="take every default, ask nothing") + p.add_argument("--force", action="store_true", help="overwrite an existing config file") + p.add_argument("--executor", choices=("claude", "codex", "command"), help="skip that question") + return p def run(args, ctx) -> int: - print("not implemented yet", file=sys.stderr) - return 1 + root = Path(args.project or Path.cwd()).resolve() + config = S.project_path(root) + if config.exists() and not args.force: + print(f"{config} already exists; pass --force to overwrite it.", file=sys.stderr) + return 1 + ask = _asker(interactive=not args.yes and sys.stdin.isatty()) + print(f"Setting up Harness² in {root}\n") + + interactive = not args.yes and sys.stdin.isatty() + found = {name: _version(name) for name in ("claude", "codex")} + default = next((name for name in ("claude", "codex") if found[name]), None) + executor = args.executor or (_pick_executor(ask, found, default) if interactive else default) + if executor is None: + print( + "Neither Claude Code (`claude`) nor Codex (`codex`) is on PATH. Install one, or " + "rerun with --executor command to drive your own agent.", + file=sys.stderr, + ) + return 1 + + command_lines = "" + editable: list[str] = [] + if executor == "command": + command = ask("Command that runs one task", "") + editable = [g.strip() for g in ask("Editable files, comma-separated globs", "").split(",")] + editable = [g for g in editable if g] + command_lines = COMMAND_LINES.format(command=_q(command), editable=_q(editable)) + channel = "claude" if found["claude"] else "codex" + model, improver = DEFAULTS[channel][0], DEFAULTS[channel][2] + else: + model, channel, improver = DEFAULTS[executor] + model = ask("Model that runs the tasks (provider/model)", model) + improver = ask("Model that edits the harness (provider/model)", improver) + harness = ask("Where the harness lives, relative to the project root", ".") + + config.parent.mkdir(parents=True, exist_ok=True) + config.write_text( + CONFIG.format( + name=_q(executor), + model=_q(model), + command_lines=command_lines, + channel=_q(channel), + improver=_q(improver), + harness=_q(harness), + ), + encoding="utf-8", + ) + ignore = config.parent / ".gitignore" + if not ignore.exists(): + ignore.write_text("runs/\n.env\n", encoding="utf-8") + (config.parent / "tasks").mkdir(exist_ok=True) + + print() + print(_harness_summary(root / harness, executor, editable)) + print(_key_summary(root, (model, improver))) + print(f"\nWrote {config.relative_to(root)}") + if executor == "command" and not (command_lines and editable): + print("Fill in executor.command and executor.editable before the first run.") + print( + "\nNext:\n" + " harness2 task new describe a task the agent should get better at\n" + " harness2 doctor check the agent CLIs, keys and tasks\n" + " harness2 run --dry-run preview what a run will do and how many calls it makes" + ) + return 0 + + +def _asker(*, interactive: bool): + def ask(question: str, default: str) -> str: + if not interactive: + return default + shown = f" [{default}]" if default else "" + try: + answer = input(f"{question}{shown}: ").strip() + except EOFError: + answer = "" + return answer or default + + return ask + + +def _version(binary: str) -> str: + path = shutil.which(binary) + if not path: + return "" + try: + out = subprocess.run([path, "--version"], capture_output=True, text=True, timeout=15) + except (OSError, subprocess.TimeoutExpired): + return "installed" + return (out.stdout or out.stderr).strip().splitlines()[0] if out.returncode == 0 else "" + + +def _pick_executor(ask, found: dict[str, str], default: str | None) -> str | None: + options = ["claude", "codex", "command"] + if default is None: + return None + print("Which agent should run your tasks?") + for i, name in enumerate(options, 1): + detail = found.get(name) or ("not found" if name != "command" else "your own command") + print(f" {i}) {name:<8} {detail}") + while True: + answer = ask("Choose", str(options.index(default) + 1)) + if answer in options: + return answer + if answer.isdigit() and 1 <= int(answer) <= len(options): + return options[int(answer) - 1] + print(f"Type a number from 1 to {len(options)}.") + + +def _harness_summary(where: Path, executor: str, editable: list[str]) -> str: + """One line on what the harness holds today, so a wrong path shows up immediately.""" + if executor == "command": + found = sorted({p for g in editable for p in where.glob(g) if p.is_file()}) + return f"Harness: {len(found)} editable file(s) under {where}." + from ..substrate.claude import ClaudeSubstrate + from ..substrate.codex import CodexSubstrate + + sub = ClaudeSubstrate() if executor == "claude" else CodexSubstrate() + tree = sub.import_tree(where) + surface = sub.surface() + always = surface.always_on[0] + parts = [always if tree.get(always, "").strip() else f"no {always} yet"] + if skills := HT.skill_names(surface, tree): + parts.append(f"{len(skills)} skill(s)") + if agents := HT.subagent_names(surface, tree): + parts.append(f"{len(agents)} subagent(s)") + if surface.hooks_file in tree: + parts.append("hooks") + return f"Harness found in {where}: {', '.join(parts)}." + + +def _key_summary(root: Path, models: tuple[str, ...]) -> str: + """Whether the API key each chosen model needs is visible to harness2.""" + from .. import providers + + S.load_dotenv(root / ".harness2/.env") + loaded = S.load(None) + lines = [] + for name in dict.fromkeys(providers.split_model(m)[0] for m in models if "/" in m): + try: + env = loaded.provider(name).api_key_env + except S.ConfigError: + lines.append(f"Provider {name}: not configured; add [providers.{name}] to the config.") + continue + if env and not os.environ.get(env): + lines.append( + f"{env} is not set: export it, or put {env}=... in .harness2/.env (ignored by git)." + ) + elif env: + lines.append(f"{env} is set.") + return "\n".join(lines) + + +def _q(value) -> str: + return S._literal(value) diff --git a/harness2/cli/judge.py b/harness2/cli/judge.py index 5074ddac..f45a5666 100644 --- a/harness2/cli/judge.py +++ b/harness2/cli/judge.py @@ -91,9 +91,18 @@ def collect_selections(ns_dir: Path, modes: tuple[str, ...]) -> list[dict]: def add_parser(subparsers): - p = subparsers.add_parser("judge") - p.add_argument("name", nargs="?") - p.add_argument("--bases", action="store_true", help="Judge reference executions.") + p = subparsers.add_parser( + "judge", + help="grade the result each task adopted", + description="Grade each task's adopted result with the task's check command or " + "rubric.md; tasks with neither are skipped. NAME defaults to the most recent run.", + ) + p.add_argument("name", nargs="?", help="run name (default: the most recent run)") + p.add_argument( + "--bases", + action="store_true", + help="grade the runs of the starting harness instead, for comparison", + ) return p @@ -178,6 +187,10 @@ def allowed(task): json.dumps({"judge_model": model, "rows": rows}, indent=2) + "\n", encoding="utf-8" ) ctx.out( - rows, lambda data: f"{path}: {len(data)} rows, {sum(row['graded'] for row in data)} graded" + rows, + lambda data: ( + f"Graded {sum(row['graded'] for row in data)} of {len(data)} result(s); " + f"saved to {path}. Summarise with `harness2 report`." + ), ) return 0 if all(row["graded"] for row in rows) else 1 diff --git a/harness2/cli/main.py b/harness2/cli/main.py index a7cb8641..93dccb3e 100644 --- a/harness2/cli/main.py +++ b/harness2/cli/main.py @@ -46,20 +46,25 @@ def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(prog="harness2") - parser.add_argument("-C", "--project", metavar="DIR", help="Project directory.") - parser.add_argument("--profile", metavar="NAME", help="Settings profile.") + parser = argparse.ArgumentParser( + prog="harness2", + description="Tune your coding agent's harness on your own tasks.", + epilog="Start with `harness2 init` in your project. " + "Documentation: https://github.com/3ae3ae/harness2", + ) + parser.add_argument("-C", "--project", metavar="DIR", help="run as if started in DIR") + parser.add_argument("--profile", metavar="NAME", help="apply the settings profile NAME") parser.add_argument( "-c", "--config", action="append", default=[], metavar="KEY=VALUE", - help="Override a setting.", + help="override a setting, e.g. -c run.k=2 (repeatable)", ) - parser.add_argument("-q", "--quiet", action="store_true", help="Silence logs.") - parser.add_argument("-v", "--verbose", action="store_true", help="Show debug logs.") - parser.add_argument("--json", action="store_true", help="Print JSON.") + parser.add_argument("-q", "--quiet", action="store_true", help="print results only") + parser.add_argument("-v", "--verbose", action="store_true", help="print debug details") + parser.add_argument("--json", action="store_true", help="print results as JSON") parser.add_argument("--version", action="version", version=version("harness2")) subparsers = parser.add_subparsers(dest="command", required=True) for name in COMMANDS: diff --git a/harness2/cli/report.py b/harness2/cli/report.py index 1d725d06..cbc7e01e 100644 --- a/harness2/cli/report.py +++ b/harness2/cli/report.py @@ -47,8 +47,13 @@ def _spend_and_fallbacks(ns_dir: Path, modes: tuple[str, ...], k: int) -> tuple[ def add_parser(subparsers): - p = subparsers.add_parser("report") - p.add_argument("names", nargs="*") + p = subparsers.add_parser( + "report", + help="summarise a run's grades", + description="Average the grades `harness2 judge` wrote, per recursion mode. Name several " + "runs to compare them.", + ) + p.add_argument("names", nargs="*", help="run names (default: the most recent run)") return p diff --git a/harness2/cli/run.py b/harness2/cli/run.py index ce19ae1a..4d578e72 100644 --- a/harness2/cli/run.py +++ b/harness2/cli/run.py @@ -40,25 +40,45 @@ def add_parser(subparsers): - p = subparsers.add_parser("run") - p.add_argument("--suite", help="Suite name.") + p = subparsers.add_parser( + "run", + help="refine the harness on the tasks", + description="Run the agent on every task, propose and probe harness edits, and adopt a " + "refined harness per task. Running the same command again resumes and reuses every " + "finished step. Model calls cost money: preview with --dry-run, cap with --budget-usd " + "or --max-calls.", + ) + p.add_argument("--suite", help="task suite (default: tasks, your own)") p.add_argument("--bench", dest="suite", help=argparse.SUPPRESS) - p.add_argument("--executor", help="Executor name.") - p.add_argument("--name", help="Run name.") - p.add_argument("--k", type=int, help="Recursion scale.") - p.add_argument("--mode", "--modes", dest="modes", help="Comma-separated recursion modes.") - p.add_argument("--seed", type=int, help="Task ordering seed.") - p.add_argument("--split", choices=("all", "train", "test"), default="all", help="Task split.") - p.add_argument("--split-file", type=Path, help="Task split file.") - p.add_argument("--only-domains", default="", help="Comma-separated domains.") - p.add_argument("--only-tasks", default="", help="Comma-separated task IDs.") - p.add_argument("--max-tasks", type=int, help="Tasks per domain.") - p.add_argument("--workers", type=int, help="Concurrent domains.") - p.add_argument("--budget-usd", type=float, help="Spending limit in USD.") - p.add_argument("--max-calls", type=int, help="Model call limit.") - p.add_argument("--selector", choices=("human",), help="Interactive selector.") - p.add_argument("--steer", metavar="TEXT|@FILE", help="Improver guidance.") - p.add_argument("--dry-run", action="store_true", help="Preview tasks and call estimates.") + p.add_argument("--executor", help="agent that runs the tasks: claude, codex or command") + p.add_argument("--name", help="run name (default: -k)") + p.add_argument("--k", type=int, help="recursion budget per task (default 1)") + p.add_argument( + "--mode", + "--modes", + dest="modes", + help="parallel (default), sequential, or both separated by a comma", + ) + p.add_argument("--seed", type=int, help="shuffle the task order within each domain") + p.add_argument( + "--split", choices=("all", "train", "test"), default="all", help="run one split only" + ) + p.add_argument("--split-file", type=Path, help="task partition file (benchmark suites)") + p.add_argument("--only-domains", default="", metavar="D,...", help="run only these domains") + p.add_argument("--only-tasks", default="", metavar="ID,...", help="run only these tasks") + p.add_argument("--max-tasks", type=int, metavar="N", help="run at most N tasks per domain") + p.add_argument("--workers", type=int, metavar="N", help="domains to run at the same time") + p.add_argument( + "--budget-usd", type=float, metavar="USD", help="stop once the run has spent this much" + ) + p.add_argument("--max-calls", type=int, metavar="N", help="stop after N model calls") + p.add_argument( + "--selector", choices=("human",), help="pick the final harness yourself (when k > 1)" + ) + p.add_argument("--steer", metavar="TEXT|@FILE", help="guidance added to every improver session") + p.add_argument( + "--dry-run", action="store_true", help="show the tasks and the model calls; run nothing" + ) return p @@ -87,36 +107,56 @@ def _estimate(tasks: int, k: int, modes: tuple[str, ...], totals: dict) -> dict: } +def _spend(totals: dict) -> str: + return f" · ${totals['spent_usd']:.2f} spent · {totals['calls']} model calls" + + +def _stopped(name: str, reason: str, totals: dict) -> str: + why = { + "interrupted": "interrupted", + "budget_usd": f"reached --budget-usd {totals['budget_usd']:g}", + "max_calls": f"reached --max-calls {totals['max_calls']}", + }.get(reason, reason) + advice = ( + "Run the same command to resume." + if reason == "interrupted" + else "Raise the limit and run the same command to resume." + ) + return f"Run {name} stopped: {why}{_spend(totals)}.\n{advice}" + + def _print_stream(data) -> str: + domains, stream, est = data["domains"], data["stream"], data["estimate"] + lanes = data["workers"] lines = [ - f"# {data['suite']}-{data['substrate']} split={data['split']} " - f"seed={data['seed']} k={data['k']} modes={','.join(data['modes'])}" + f"Run {data['name']} would refine {len(stream)} task(s) in {len(domains)} domain(s) " + f"(k={data['k']}, {', '.join(data['modes'])}), " + + ("one domain at a time." if lanes == 1 else f"{lanes} domains at a time."), + "", ] - lines.extend(f"# namespace[{mode}]: {ns}" for mode, ns in data["namespaces"].items()) - lines.extend( - ( - f"# summaries under: {data['summaries']}", - f"# library root: {data['library']}", - f"# domains parallel: {data['workers']}", - ) - ) + width = max(6, *(len(d) for d in domains)) + lines.append(f" {'#':>4} {'DOMAIN':<{width}} TASK") lines.extend( - f"{i:>4} {domain:<40} {task}" for i, (domain, task) in enumerate(data["stream"], 1) + f" {i:>4} {domain:<{width}} {task}" for i, (domain, task) in enumerate(stream, 1) ) - lines.append("# domains of this split, with task counts (the legal --only-domains names):") - lines.extend(f"{len(tasks):>4} {domain}" for domain, tasks in data["domains"].items()) - cap = f", max {data['max_tasks']} per domain" if data["max_tasks"] is not None else "" - lines.append( - f"# {len(data['stream'])} tasks over {len(data['domains'])} domains " - f"(domains parallel, tasks within a domain sequential{cap})" - ) - lines.append(f"# call estimate: {data['estimate']}") + lines += [ + "", + f"Model calls: {est['rollouts']} agent runs + {est['sessions']} improver, selector and " + f"reflector sessions = {est['calls']} (retries can add more).", + ] + if est["average_cost_per_call"] is not None: + cost = est["average_cost_per_call"] * est["calls"] + lines.append( + f"At this run's average so far, ${est['average_cost_per_call']:.3f} per call, that " + f"is about ${cost:.2f} in total." + ) + lines.append(f"Stored in {data['run_dir']}") return "\n".join(lines) def run(args, ctx) -> int: events.start(None) - status, reason = "error", "" + status, reason, name = "error", "", args.name or "" try: project, config = ctx.project, ctx.settings k, seed = config.get("run.k"), config.get("run.seed") @@ -166,6 +206,7 @@ def run(args, ctx) -> int: "domains": domains, "stream": stream, "estimate": _estimate(len(stream), k, modes, C.event_totals(root)), + "run_dir": root, }, _print_stream, ) @@ -232,23 +273,26 @@ def run(args, ctx) -> int: if errors else (("failed", "failed tasks") if failed else ("ok", "")) ) + done = sum(int(row.get("done", 0)) for row in result["domains"]) ctx.out( {"result": result, "totals": events.totals()}, - lambda data: f"{name}: {status}; {data['totals']}", + lambda data: ( + f"Run {name}: {done} task(s) done, {failed} failed" + + _spend(data["totals"]) + + (f"\nErrors: {reason}" if errors else "") + + f"\nNext: harness2 status {name} · harness2 judge {name} · harness2 diff {name}" + ), ) return 1 if errors else 3 if failed else 0 except events.Stopped as exc: reason = str(exc) status = "interrupted" if reason == "interrupted" else "stopped" - ctx.out(events.totals(), str) - if status == "interrupted": - ctx.log("run the same command to resume") + ctx.out(events.totals(), lambda totals: _stopped(name, reason, totals)) return 130 if status == "interrupted" else 4 except KeyboardInterrupt: status, reason = "interrupted", "interrupted" events.request_stop(reason) - ctx.out(events.totals(), str) - ctx.log("run the same command to resume") + ctx.out(events.totals(), lambda totals: _stopped(name, reason, totals)) return 130 except Exception as exc: reason = str(exc) diff --git a/harness2/cli/runs.py b/harness2/cli/runs.py index 4aac7571..95162298 100644 --- a/harness2/cli/runs.py +++ b/harness2/cli/runs.py @@ -22,11 +22,11 @@ def add_parser(subparsers): - p = subparsers.add_parser("runs") + p = subparsers.add_parser("runs", help="list runs or show one") commands = p.add_subparsers(dest="action", required=True) - commands.add_parser("ls") - show = commands.add_parser("show") - show.add_argument("name") + commands.add_parser("ls", help="list runs with their progress and spend") + show = commands.add_parser("show", help="show a run's settings, totals and tasks") + show.add_argument("name", help="run name") return p diff --git a/harness2/cli/setup.py b/harness2/cli/setup.py index 7833e9fd..01bf4236 100644 --- a/harness2/cli/setup.py +++ b/harness2/cli/setup.py @@ -23,9 +23,14 @@ def add_parser(subparsers): - p = subparsers.add_parser("setup") - p.add_argument("suite") - p.add_argument("args", nargs=argparse.REMAINDER) + p = subparsers.add_parser( + "setup", + help="install what a plugin suite needs", + description="Run a plugin suite's installer, for example `harness2 setup lab`. " + "Arguments after the suite name are passed to the installer.", + ) + p.add_argument("suite", help="suite name") + p.add_argument("args", nargs=argparse.REMAINDER, help="arguments for the installer") return p diff --git a/harness2/cli/status.py b/harness2/cli/status.py index ecf0b9ff..7059e902 100644 --- a/harness2/cli/status.py +++ b/harness2/cli/status.py @@ -12,17 +12,206 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Status command placeholder.""" +"""Show a run's progress from its files on disk; `--watch` keeps the view refreshed. + +Reads run.json and events.jsonl only, so it works while the run is going, after it stopped, +and from another terminal. +""" from __future__ import annotations +import argparse +import json import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +from . import _common as C + +_MODE = {"agg": "parallel", "seq": "sequential"} -def add_parser(subparsers): - return subparsers.add_parser("status") +def add_parser(sub) -> argparse.ArgumentParser: + p = sub.add_parser( + "status", + help="show a run's progress", + description="Show each task's state, the harness picked for it, and the run's spend. " + "NAME defaults to the most recent run.", + ) + p.add_argument("name", nargs="?", help="run name (default: the most recent run)") + p.add_argument( + "--watch", + nargs="?", + type=float, + const=5.0, + metavar="SECONDS", + help="redraw every SECONDS (default 5) until Ctrl-C", + ) + return p def run(args, ctx) -> int: - print("not implemented yet", file=sys.stderr) - return 1 + run_dir = C.resolve_run(ctx, args.name) + if not args.watch: + snap = snapshot(run_dir) + ctx.out(snap, render) + return 0 + try: + while True: + text = render(snapshot(run_dir)) + sys.stdout.write("\033[H\033[2J" + text + "\n") + sys.stdout.flush() + time.sleep(max(args.watch, 0.5)) + except KeyboardInterrupt: + return 0 + + +def snapshot(run_dir: Path) -> dict: + """Everything the view shows, as data (this is also the --json output).""" + meta = C.load_run(run_dir) + events = _events(run_dir / "events.jsonl") + tasks: dict[str, dict] = {} + for domain, task in meta.get("stream", []): + tasks[task] = { + "task": task, + "area": domain, + "state": "pending", + "rollouts": 0, + "picked": {}, + } + spent = calls = 0.0 + ended = None + for ev in events: + kind, row = ev.get("kind"), tasks.get(ev.get("task", "")) + spent += float(ev.get("cost_usd") or 0) + if kind in ("session", "rollout") and not ev.get("cached"): + calls += 1 + if kind == "run_start": + ended = None + elif kind == "run_end": + ended = ev + if row is None: + continue + if kind == "task_start": + row["state"] = "running" + elif kind == "task_done": + row["state"] = "done" if ev.get("status") in ("ok", "skipped") else "failed" + elif kind == "rollout": + row["rollouts"] += 1 + elif kind == "selection" and ev.get("chosen"): + row["picked"][_MODE.get(ev.get("mode"), ev.get("mode"))] = ev["chosen"] + if ended is not None: + for row in tasks.values(): + if row["state"] == "running": + row["state"] = "stopped" + plan = meta.get("schedule", {}) + return { + "run": meta.get("name", run_dir.name), + "suite": meta.get("suite", ""), + "executor": meta.get("executor", ""), + "k": plan.get("k"), + "modes": plan.get("modes", ""), + "created": meta.get("created", ""), + "spent_usd": round(spent, 4), + "calls": int(calls), + "ended": ended, + "tasks": list(tasks.values()), + "last_event": events[-1] if events else None, + } + + +def render(snap: dict) -> str: + rows = snap["tasks"] + head = f"{snap['run']} · {snap['suite']} suite · {snap['executor']}" + if snap["k"]: + head += f" · k={snap['k']} {snap['modes']}" + last = snap["last_event"] + lines = [ + head, + f"started {_when(snap['created'])}" + + (f" · last activity {_ago(last.get('t'))}" if last else "") + + f" · ${snap['spent_usd']:.2f} spent · {snap['calls']} model calls", + "", + ] + if rows: + w = max(4, *(len(r["task"]) for r in rows)) + a = max(4, *(len(r["area"]) for r in rows)) + lines.append(f" {'TASK':<{w}} {'AREA':<{a}} {'STATE':<8} {'ROLLOUTS':>8} PICKED") + for r in rows: + picked = ", ".join(f"{m}: {c}" for m, c in r["picked"].items()) + lines.append( + f" {r['task']:<{w}} {r['area']:<{a}} {r['state']:<8} " + f"{r['rollouts'] or '':>8} {picked}" + ) + lines.append("") + states = ("done", "running", "stopped", "failed", "pending") + counts = [(sum(r["state"] == s for r in rows), s) for s in states] + lines.append(" · ".join(f"{n} {s}" for n, s in counts if n) or "no tasks") + end = snap["ended"] + if end: + status = end.get("status", "") + why = f" ({end['reason']})" if end.get("reason") else "" + tail = "" if status == "ok" else " · run the same command to resume" + lines.append(f"run {'finished' if status == 'ok' else 'ended: ' + status}{why}{tail}") + elif last: + lines.append(f"latest: {_describe(last)}") + return "\n".join(lines) + + +def _events(path: Path) -> list[dict]: + try: + text = path.read_text(encoding="utf-8") + except OSError: + return [] + out = [] + for line in text.splitlines(): + try: + ev = json.loads(line) + except json.JSONDecodeError: + continue # a line being written right now + if isinstance(ev, dict): + out.append(ev) + return out + + +def _describe(ev: dict) -> str: + kind = ev.get("kind", "") + where = f"{ev['task']} · " if ev.get("task") else "" + if kind == "rollout": + what = f"rollout {ev.get('label', '')} " + ( + "delivered" if ev.get("delivered") else "produced nothing" + ) + elif kind == "session": + what = f"{ev.get('label', 'session')} " + ("done" if ev.get("ok") else "failed") + elif kind == "selection": + what = f"picked {ev.get('chosen', '?')} ({_MODE.get(ev.get('mode'), ev.get('mode'))})" + else: + what = kind.replace("_", " ") + return f"{where}{what} · {_ago(ev.get('t'))}" + + +def _parse(ts: str | None) -> datetime | None: + try: + return datetime.fromisoformat(str(ts).replace("Z", "+00:00")) + except ValueError: + return None + + +def _when(ts: str | None) -> str: + t = _parse(ts) + return t.astimezone().strftime("%Y-%m-%d %H:%M") if t else "?" + + +def _ago(ts: str | None) -> str: + t = _parse(ts) + if t is None: + return "?" + s = int((datetime.now(timezone.utc) - t).total_seconds()) + if s < 60: + return f"{s}s ago" + if s < 3600: + return f"{s // 60}m ago" + if s < 86400: + return f"{s // 3600}h ago" + return f"{s // 86400}d ago" diff --git a/harness2/cli/task.py b/harness2/cli/task.py index b3bc31ba..cb47089a 100644 --- a/harness2/cli/task.py +++ b/harness2/cli/task.py @@ -20,17 +20,23 @@ def add_parser(subparsers): - p = subparsers.add_parser("task") + p = subparsers.add_parser("task", help="create, list and import tasks") commands = p.add_subparsers(dest="action", required=True) - new = commands.add_parser("new") - new.add_argument("id") - new.add_argument("--files", action="store_true", help="Create a files workspace.") - new.add_argument("--domain", default="default", help="Task domain.") - new.add_argument("--check", help="Judge command.") - commands.add_parser("ls") - import_parser = commands.add_parser("import") - import_parser.add_argument("file", type=Path) - import_parser.add_argument("--force", action="store_true", help="Replace existing tasks.") + new = commands.add_parser("new", help="create a task directory") + new.add_argument("id", help="task id: lowercase letters, digits, '.', '_' and '-'") + new.add_argument( + "--files", + action="store_true", + help="a files task: inputs in files/, results in output/ (default: a project task)", + ) + new.add_argument("--domain", default="default", help="domain the task belongs to") + new.add_argument("--check", help="shell command that grades a result") + commands.add_parser("ls", help="list tasks") + import_parser = commands.add_parser("import", help="create tasks from a JSONL or CSV file") + import_parser.add_argument("file", type=Path, help="the .jsonl or .csv file") + import_parser.add_argument( + "--force", action="store_true", help="overwrite tasks that already exist" + ) return p @@ -47,13 +53,25 @@ def run(args, ctx) -> int: elif args.action == "ls": ctx.out( suite.list_tasks(tasks_dir), - lambda rows: "\n".join( - f"{row['id']}\t{row['domain']}\t{row['split']}\t{row['workspace']}\t{row['has_judge']}" - for row in rows - ), + _table, ) else: ctx.out( suite.import_tasks(tasks_dir, args.file, force=args.force), lambda rows: "\n".join(rows) ) return 0 + + +def _table(rows: list[dict]) -> str: + if not rows: + return "No tasks yet; create one with `harness2 task new `." + head = ("ID", "DOMAIN", "SPLIT", "WORKSPACE", "GRADED") + body = [ + (r["id"], r["domain"], r["split"] or "-", r["workspace"], "yes" if r["has_judge"] else "no") + for r in rows + ] + widths = [max(len(str(row[i])) for row in (head, *body)) for i in range(len(head))] + return "\n".join( + " ".join(str(cell).ljust(w) for cell, w in zip(row, widths)).rstrip() + for row in (head, *body) + ) diff --git a/harness2/core/__pycache__/__init__.cpython-312.pyc b/harness2/core/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 4117a5dd85b7210f190225498148dc68d5d5a771..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1452 zcmZ8hO-y4&9G~fXZC_iyg2r@RAS1A@jnE2fvTU+37Pi7hC`gMRMDz0K8|Z_+kIb6~ zl)M@+9xz!CUR)E`ghXSqSq~mOn0WE-rDTaV|MB~O zz27=JlL*!i`1kdb7($Qbp}o{I2A^bsxrDJ5hx$G-OvB>_w_Jy~o0JiQRK~K|$^~iP?p8sa__RhV(#yyxRD*c%V!F0rIfQP*!2JbbNe=@b30Y82=iX z(QhJcolmd|NcOryw2ebm*DmCu`URp8@5KSDs-HHQln5hxgJC38^a~XHQd@KV#WweB z4=?9%P>LIFXMeAlQ2w7{6lL~3CsUn=PD-M&?x5j~6Zv$OL{Wql#Pn4g_3%!=Of zV!l{fm|raG`Et21w@@yL?z#N?`qX@JYH4w?P@MWeBzTok!-Q}|&!ii`Aekj<8(tM6 z6{=n;%w-Zx_7Q`LMl3GL|3JnzB_u%U7&aBrvy&_vX|I3Z&MiTo`nhWc&1F4kYd0s8 zC&$gtjco>Ob9-zew>36Ow_)zM3!>*!JD#fmlANwvj>UC7>(xa9+I`hp3m9en;6)bN z{~pLL`b*6ns=4DBN)NRLUTcxppT^a$*zQ}$Nt7PA8-5Xf`SE9|eRZFFvd>@RIL8pf$-2{)C1rL27ff0?=1a;dhu<1YU4j+RFm5P diff --git a/harness2/core/__pycache__/prompts.cpython-312.pyc b/harness2/core/__pycache__/prompts.cpython-312.pyc deleted file mode 100644 index b587b845558d281ab67abc2cf28199b39f198eba..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 76063 zcmeFaYjB)bdgs@T8-WIR0ZEC}g;I-@L}ElVb(xW7D2Xx%G)YJ#zyUx};*1S6&<(Ij z+}zzDMGQDgUv@3|Hj2GC$*JwDya%+${`n}R82OM)H)wh zP^46C^dXhs|2gNp{WjR7sF6JO?g}#k8*jhwdC%oJ&+~t-=YLqgK9%6-zx)6G>P*w4 z$;AJrKf+J5`;h$T50Z(*n~8klR3e|uH%upI8csDN^|^7n@l+%KHcdCpG@ojoX*t!x z_2zub^tzeWQ?2%S-E`Yb>Qsu)t*6@aU(dJwdefl`HoW?^XpG-%5ON; zncsM7bAHpQE&0wdyQ(@>{uo8~@(HzuWV7p1Lc)?bO}*{rRW!+kd_J z)IIsTPTiZo`_zv7J*V!=-+Su*{Ek!U{C%e$$lrfzXFh#uSN?%h59W8CdZ-~$XwL8Y z;e+oo7X36N4kz*t{dyw*@VgD}%De8L#Hok#&*XRWw%z$Xr}pHZ&3Ey+i|4zdJHC=X zkpBu-AIU$Le}sSc=6@l-mw&tSJ^60_%`gYe`Nn+a`;BdhoBc0;Fu(8DTTbn>dmsIN zL%uiv7=J&S@5?_<8;>#OC;0dA{8vvsk^kDMujaF~{v>1m8s9v{H&0$@>^hV?!+^5q z3Z)C_`O@6Xd?j5uT}aOqDyQf2>2hHYm0nwkv(|NY6<6<-rr_-kyPU?~$H?!&zRkP@2q56jI~U za}#G7`gCEUQpl&rU5|#{x5M*=iQVP&Y;LBINe|40IXhioXu8NC)1`&+QgMR!rOOj@ zT%V&e=`#!YsX`|8*CLXx&jy4bgnd8D3>4aer&>1rjeocUSlGW{Ah>|Zzd|ycV|4c+FkyacI8sy zUGCS<8NYAcU+#CH+6^(70p;~HeG<0 zSdMuOk=51`W)@q%J^P&}_Fnv?A(dK5rn(we8cKzgMiz6WndL0bGrT6zSXuo|fAajk z6J-{6-xImQzRY~CT$tN;ey()3QYsY6`yM;DP&)U@V_)BQ{;}@DD}{-L${e?b`Piqa z-{(lWoSDC{(mYxz<}Z{wY4`#DEAQghrNmF$*MINH*PeX6>#Y|rk6dm4%C+`Kue3jU zwf*t;My|AfW4Y-YA8}hE_dE0_MmBwe>4CwK^pTzyvqtBlYlvblTKcI{E-%uXC>5d3 zVh&=MEuV)f&lWE1O_vJLF=WsFn44bE1c^o*vE*hioHuGJ&ZaNSEtFCda}f9|6}6D( z6_wL!moJNzboun$!ZZ|GNHZhsg}g|7ZWhYyo71iQc{V*s(~$kl-gMqwtgzK9b7@Uv zYHp#D-g*9X&h2{jnm(S-%~r}g{R?elpVu%T;`!4h79l-dJX=Wb)L!yW6kcIl@C^N; zWrkqAOwoUBko>{~U3y*{!0m7dBzk9YmW5xKu#FJ)S3`~8#7;lM3)s$VFh+eoll$E- zSLEltBGHfmHvU@k)ad4 zBmIK|!>QDuQI%Y3p3+|e4f3@p4C1?Y}lT1EK6%x zhD*#>_NL~i7pAz0*8Q8D>I$BaqVndFm5<+Y$89G zvXu0^P!|9%(Y`SM)aNy-*gC~a5g9GW~Cx(s>4m(zd z0C$2sR~j!e!!Q#~WBK&L$gqzcJZC<^j-<~_E8Mwow zrU(vW!epV$tclB<7&tcA*MF!#+ZPtey;G0aO}Tt-UhEIv2N+R*L`i}h;)0bqnB@7> z#fj76((XfdclvmL?+g6{hZ$;m@WjaR6C>$ELxackXY74PdWHtF!^0xxo^ zzZ|CvCQNwo1h+CxHmzlDnD(ej(n#=#huInnd~rehn+=}HJ!Sj)?*fqkxT^b=LCyn2 zVG&c3d`u;@5>0$;PV3F((*V3rnaeL^?tbfUohJg8ONCzsr2`?fnbtCIKXK7kjPb|% zd@yh)@xjy=&j(}tFIAWS1^)drFu+W1a(fMstKxsBuXq`t4U`k8GL{54o@e_3iZbbJ z4lZ&U&XtQcPDR|RQiK;x%$3U-N-d@5bCuKSrHCu@4*kK_Wbg9_%-M^$ z^ipc6ySqF3k7viq7s?eNwP{3|nfw^fmgc4rAH0sM6NUT&w7-<@IeN0^W#ihgIFY!( z-dSFvQ4}g{^7BQyAxc0so2{VC%$<*T^0+`+Mek3KO)ca~`BJVpUH0!onL}2>oJtE4 z%WK|MM*DFyU{4>No1GddlxB`)j|~pJ>|3CZxp6w3pPNCTrkxbNbvu>0x#`h-!O2Kz zOZ)7~)B-{qp5+^sWPzU|3~Xe&<Kj9YD=d_)qXNm3 z(=@&WzXcrUB@X+t$6)+>9hR+Gz;n!W-pw95&8SUq^opLt*@2Ph6?yc_DZ}jCX*X&` zr9&|>MZn46$$>x*(ikcWCB8HL+ve9zNN5}dM-&0wvBagB;%u>ulAS(Xn3gJR4U6mN z3j#3dbH!YGJXeMowayN9_=(6=P`A~P2P4tq3}p}Z507MrvJBivTmTAG2iS|ITrz2% zMosmvoYwYn5=D|MS2ndnB0uFZM(ql2&*gwXil-zbow(n?JIUACV_Ki>cGw+kSL@o2{}EzKgHKG3+#8aFZDCvJH9-hyHM_yjE&SbG@`H; zJcjWD>a>ReQ0EHMZc7#`7){1vbaS(5nVHKM_R}{3%wok{oDMn4nTl zX{S#zANGi5Q(Z}ybLgbbf}?A;HQ_|?jXIyL&I7+bsyDr7-Fo>Gd zxq0KDFetYD%zo$!U<#aPSr*2-h1cPLbL`sw0Rb?$;B>Bpny4YUFM0-sPiBYsCS`EN zMrPc39PrEZ0cUIrs^=b9>TB#uMNpI=4Z8U(cvF+>N!=#a9*`aqv3F0bB5d}cz84>H zL=6TLxj@eBPWPqb#Wp!9eOX}mj{TS!3n~yxD$H`(G=16B#68upmT+2FM);InFrP^u z#~P*unH5EqD@MFXZnGmi&BUD+-g8edlwLX#)M1m6(4nmO~<>VP$ za<8pgp~ARQw;|6Mx+u!cJbPAxnw6hKxJhbiub!s=;`SPcOnADKt7rkxr=){ciqqDB z7m?gH#rUjUC_y$3n*c3wCf$=aLCF`a=}hh{-x()F7G}+SGOaB)S4kmAK@~j+;Ms1^ zovZYyZbps9x-i3v&1y%hv&C7HI!rVha^6f!dmV}LC39IQ?N6oCOhDRXoM{=sLgN^Z zA>nb6nXq5k6|uCp-axKNuzq&Zfs^)!(QiRZh4 zLnxrOAXDrr6EY?b(C=k_JU@LQYW>7Plg7UNz>}CtU~&<$EJpA6kMR!mel27220gxU zjLCQW_|X%G`v)9Pip|GfOzPTiX8DFJ#0~=^sd5<_8!u?*0WoLu8Vy4ARQhEW+@t1y)j=nuRarO;^aZwW>7VH_j8=80{Mw!ywd1qpE_R*)U{Tw3SVO02`B#Ep7Y)NGHq^1?_ zS}2uxL78dEfvJ4I&lU=_8By=k%-{UHdZb$`1~?Gb0b26IadY3 z5;!8>3s*1}22>O>(NxNv1HyxwJ%&JdF+23KXIlf=Y@ci}BSZa%4+98}_77ymqC}T7 z(z_5j(}t&Qrfl2D5dn*1d*Hl62FB+G`+$so3|VWBepO!`fR)06tDsX*&p|gWW3H5@ ziwAo~`j4@hHRU(zX}n>LD&^U8AwAr4ESr|;2P`b&;^(CF!Dlq6h=DRhF&XvGvxV}3 zy#{4%@MBJc!GEPYm^fNNo-E8dRO+6i0SEA4p0jKTR)@)E=kcOxh)E32U_`Xv3$LI; z(AwkJD+kjfSu9@|2h#_KdU~M>I9x61ipxvZ30)#U*;hC?kUiSLtnR|p>Gbo(*)v*> z%4v0Rm`y)XIl(p?OiINqO!sNuS5R#H-5eTDF%pY0V3S$cOnvM1)Qr z$!9pARXcWKcmxQ8N`&-g)y;&9Hm30(Fm24sW$86Dydo%=qBvbrKWn$9BdhZ1;(QPc zrc%RF=jY01%5zKXHFmGtfl+8FB41nXZY}+ReK&SW9#&K+uNfgP7`;ccj!vqC4nYha zI;0)1_OpHcBRvP%_roWK4nYCNqaw{JUo0aHK=q;m!70?eS0)PP`V(aZ6Su9#(1`)I z3oT3w++I-Y4DlShG=kk)q4GS-%L)i8_GOQP6Nl7+-NWhOBR$6<#O%?*fy3}8h?84} zvfn(B9UgJc2ClJCT9{zZaZtCMJ~DXHI$;WT%HH#|n2;%BZijeOY%XH=D!vD;qX(8C z7&fR04Y;PX7tM4TsxM6Eu{=-U&YPL#L-}-}P$}VS+ zLx}>sw*O=lr(MV%Zmbbh(V#u${3O~XZV0gM984Yu98TCA9x~}Oz7r+DjCh=^0vJji zs)^x+U`UgB!T1_1nk7L@^6nJQXoMIrm9Bf&IJ6=l=46`p{R?jh=bS4R}yssC47mb(?&lp09eS)(2l*IR~E6kl_aw9m`amyie>Pz^BkL^XmguM zANIx|H?aY37g9DwY!_Mu#7i=tE0&8gKuO`Bor(x9kcpXBc)8}cs!j#v*zc4uZJAV$ zn*(>dsDb|=1{w&K0Z%t9XODyzUV{@^@iy+6S(uhvIL_ION>a|CfzzlPb*+1078a-5 zX#;J2f~O3fP8Wbz4paL{gA`z;Ma(+tzG$26v&%@H4RNgJ1-v>w*x?3aCqoHGa{WN5oO`IT;9z=}TYmDDR_);>3PYl)K$zt<5F9DFc?{=uLj`59w``oQSDt8?HYJiLh!7cRfc{U7-7ma9I^iljrU=`ENK`87(z7Uprac$%*QRa79O)u`h{%v_~^kNe1B;P z3cr|Gk;UnGyD@j@Z<=-$-(X6u=-X-5kh(l`Rc-CYo|PBowb5zcX(mD2R}aP)*!q4+ zBnpI$+&Y-fKL`Q@{mjvw17k#?C~QZMg+&y$-EZ4bCV0Pg;=V$RPJ;$?rm7p=&FIWr zMa+@~0}Pu$vsZXfgVo7kCA$)y;>T+^+eqY{a zP$pQ$Z^N=L+4o4yj11AjX(~nB&+?}h(gz3oUKXi~(N-M*&U#+|r2%?<#TnB*R+GwH z)ZpcT5d(9e%bDkEjq))wu=h~h%za>kcD|scG^pvk&@_8h?SG07A?kZTuZ9~xk1ruJ zfnoENeX>=SGiTT!8vk^0vM_OBg7voRT$x10tLr=fZ3HVHjM`iMrEr^}`ZIVIqMXp#ye~urEwdUIkfbgTEpuR8j1kn35p^ z8l(^m44C(1*FDe z8(FYYb{4bjNh(U15}hY)FxZT^JTHq#USxj0Afam-Gw|J%Pv^JDr5rVT80$&_%6IIW3tnetVA)VT9r@tb);_B*Wpcy;I-TLauD33s2Y!|9iFl=3-_|L3$wX% zNM)ILqfkJHJ^)}y*ReW#m&PPypC86}?-+RRbZ#ME=5NdyFsh<&-RM=cK9A=GgI4VT4_c@(suwJC6A#_UAAradf6(Y3R;VGe;9J z0|ng|b^?{;{@mFu+jVG)?dE*? za<4E=5IZCd!*Ms`0GKEcWWpSZh};k$M4J+LRk?x@GQDR^R>4uxrUqnx*m8_LosNx8 zs#-bazD7ACsDKHe0~iouca4R2gJ%(tdUejdZ>Kzznj5B;@fhOS>7+?6wbN-iJ3+gO zQ8RGpE3W_;GR3u7Tf3zPW><2G|-1T|gb-u!H>YQ&%7 zDD$A)no7yiM~6O_bIimsvvY%U>;)P0{Q9AeVN><6aJJhNSUQd}|1``_E={oO^xPEA za@G-M6bTM56UTz%R+A#SP8l0y!}L0x$2t21hQ(mNbiv&uq{Qe6qFO|GK6Ls0!u_lYJ+~7&G#{U+*Y4-j&kOm@P_{n$6K^UO3Io!++Th(`6 z@XsxsZPvCq+});(wPrspx7%v zAwD^Sn@-np`A9b1n90`YG(pHay$t zXmqTKX3Hs4RRxd%dQZ!(-7G=(bauOdNL_2@Y;t?f83~0)BI{r9<*l*6f9Xq69G;fM zsgrE4FMTOr`cmWogA~5>rQl&hxB%SQB64IncKxL<o*W!{VfZ)!344850N)Y7YQE<0^bH>C=^q&7p1#o&$B$=+F4p=z<*o}} zKnq>Mo@1@bvrk}CcF5(N5%4gA_x2zOUTfZJo<~X5V}rvZDS3A-NP>T>TO}MB(G!bYW781Rzoo)}wd|nS`AK)`j<`eA=EV&vOKm&8CTN!*Qxu zdRfiQA$pz|Cn6La(1@IEiUSzO3>JX4<;the%mGLUH#}b`%%&=eiC65-ZaLhsp7^V(yf%X)=kJ58N{x5 zKsDb?R@XAk$OxyDC2*6xbbHe$hx!RU(mZP^SpVVd(L=x~riw|#M2FdtoMqX3T%wL9 zd#AxIm{t@`g6~RBrYT_~BwxOztQs0;8z~AjbUs z(mPm61tiX^)6t{OW*(kk`U}&O%!-`Ea_I6}JkRvU45iLtx97&`$z(D+yFxgJg<&X| z!%{i*jEh=|2Iy#0Rttq|%V7}_bwqbOs=-5Qk(3JjcOM$H^e zd>C2ba!1I3Pq<-kw(rCc`yB79e8eQ8LmBu&d#e85V#tF&!1Y1>Y*K+eZ%oJ1uvYzoD9u>ND)mj8rdV>QQ$nO@TdHiDf5|1+D4fwm2^ z`^eDl8+Qs<)5nN8B2;|$)9KyeBRfSn!E{?09O6H6H@ThdcH7SQm(|Zu2$Nid6A-&= zYGl-J{Yldi6^4f}enS)!Ffi9;X~e1tLVWB*QK9}`a}>{GFFwtxvXK^Moev>TB*bJz z9Jjoa)-}cWm*~QTYkw!preY!rW4uY9?M6Bt0z_aH;{})oc@QOZYai9H&l3|l>_pM_ z4W|$F5A+Wo!CmL`j0vr@B;;(*(9wRh?~$J27lthe!3hpeU1kap;b~>bAt12Uwr7oy z85kUiV#~B6Jo!hd(TbsAF5E`uh9Yrak^m;5#MfAVh1cM<68y8+kI;t1Wr(J=dJzzF zmHS%ABinc{8PmsKmb^Az9nFGog<0tC#xbO500_fH&+Ntd+HIm2RxJ4zVgUn%Pmwvu z+G#`)qY=+GXDEV(W-w^sZ6Y(eSSY2mv)IOgI~sey`Jz}-=kGavVVYnSg)!K12@0y9c)|+3%6%Y@VL@9QB49` zidM(nq~PdOJn4){qJObKX~)uH+P@agmlEK+A1%>)@Xp{=>t6a_kR}{qf-PJ$skiyO*1~OPl$+rsEAe%EgOfq?S`0p)=DoQi?vb(1J(;5e6aO zr!tSl$+WkK)|2d{32_qkjrW3EHHGwu;?|A&v`k7a-V5J-Ma(Yccoy7kZk)PD=a7GS zg&iY|CSay&-$jsEI7G-^^GFagy0U31=>j6(Je<}U*zm0K3%ZGv9z2@P4h;k%qcW>59|aLE3ORIi@Z|n99aB1a=ofeOA08m{57l+x z#h#)5o`I454l_r+jYOHTZr26l$6{>oV>ugGiCOB1v2v4g)vJmK$zSILh(hFluPMK5Nj<F!WRx4>S&kJIBurG2*Z`%iu;r}U z%o~^M8R$9s@<@L#CYcurwKvz8dJc*BtWjn~T-q!Yk=j6Cd~m(Gr8<0UGI+a~HQ`bv z()S=|XxcI@KrXZ*cVLZC1NRf%%@i@0@`0oYwpwvI6pT<4y&#?tAc4m$p`^&5V&Ykz@uwn(!U#){9UbLz zPsF)rFa=?>ppjr5v7FMqveiirLOaHg)(pi5X3NOo6eE$F_FkFR>tb~63MJ`Q9pWl3 zz=lE#(x*`S%uFu4jUG9Mh=S&NkW z@CusP1*)`xPmChDX=;BUmz-I^n1zBe0eJ=fdscs=*d z#5-rd|HNC1fBf8X`-|7wzkQ|s+gICvX}Rf_e!O_?xxp*X4K5!azWUtAnlDYw%YCmy z2r0^84K@Az;2jcDtuu!OEt&VS!#s&4R*`A zU=vTqv;^a}qQ)7-*Sbt?u5sj-KygjJ zkFP>jFSBRjDc*w?Qol@C?Kl)tFYGopPq-?KF(fv*q$HOrbh9F26JJ*-zDk->P|Uyx z^EV~DkG};q>{b=Tc46>h3gwopR>pA(7&}jZ3zPH>HN~*B-zsqmbYf;=uY~z3W4BXI zOMMa8bt%QBE}GBB3zE^{rb<^Ts?y~$ zSJHdTE`)0tB_e&i?~oJW(%tk{mkzA_I_$;h(?WdGkgX69rNYE zJZ-BI9@8O+gc2(MhWQjDK^JDT5(+ z5kvLG(1a-pA>AEtq;e5PQNJLohU_cdx#B+8HR@m>o6*WCTrq1tbhC$;UXij=F6OF1 zp4jH{_{|KIR+6B4t(r~Wvx!EorCm+dr=K2yi6mL*2 zdM%^0nbZVD_1kQh`bMN2QppHS0*H(wu##~_J2TOs!f?$X%+0O>UNEc)?_x)Vyx6`l z5LseVse`DEF83X(XAn4}l{jmQ=b^J)KY6kZc@hLSnWJ#Tkeq{#QEFRFQKgM%OdXcS zJWkAq>;XU#<-BV<<>YkDV*n7*t56(vMq-U+*Tp;PJ(rmX?MTxm$o=b+N3 z)@H0Ko~pW$zLb1NE0?OzoP-c;d&cb)IKY(bLA=c2nesjYMPgaiY}&Zc)t5n(jRl}g z)8<;X8n#ZxL`xnPkjf`v864MU2VkVpItGa)de#h#1r+iX10ZVJIsDBmg(IOnivfCc zLcvvH2Xh{2zo)63Z!AN>4ntZ{mr46JOm2kcRoOu_%DzZRWw`W0>Fj z)Z9K>gVr_wb@B}afqjxd-O>WLO4bjhnUt%36O1qN`kO2f;4=n3wLpNu@z1JCxQ%R~ zJYQ&qfX|~xtWC3At54KPMrLuo{)N^O`1$D=&uICMo30=ZW6RK*Eh zMyH+8X4#OzTrEO2{jREj2veIs084@0OY>3#%}5gJ-t)Q7wZ2eRC?5xgDK5NhSbQn;drLO#yOKbxfO)g0L1|{Ea9IHJX8p z7YZY_gO#MYcjH@-g2hcj4 zbHWFn)dqU_>SQI=epTFBusu+nE6MpG_H;V+Pf)R5d&20ap{<;;6(Rqu{!5 zf#D;ALqz;?+K2)!10Q#Xf~fI{*-OJwX4VhI@cpxDe3@W+E@UXXpa+c1cI^%rDSV0-#aSoKSO@vL%#RvMCDa*{r zL}8^tgN>3mKv>eF6fdAcQ|36=9(00ka$yF0Ezne!WKA;*m*OIkHtiES&u`gBU>G`A-tVD6N{3QsXf?s1V=4Wf{8;g7#XrYVsB_7MsdMi zA!C+})R|?g)Wm@YvZ?D#AgB%SBs!E~QnC4!ImI;?wbW8cWKGrbueB2U1=Twu=H#rW z={=!ep4rm6o1I|%Mv5pb!TN5@(XgTfC(ZfeJ^T=5V{$>6)wsZ1LrXp0?L(efeIJYj zy;eg8J)2_}l08>1GB4sc1nv5gi&aXEbF1_o6XDKjN}Q7m%ZzhZqgcy_aHC{%1n$~p z1m@Y;o*3K1`_CqbI$^g(tW~wMV8OO_)6*mX=+qurhh3RD+b8nH$cF6awc2;9wF?{& zrxRgCFO@Y|p{LML7SSG8QOjf0^deVV`pR76v35K=eewIU|b+D=xGQh)d zo&TvE1z`_I_(M@5xK@~L28pyXUhIOQEVx1JAr=aR8&`i(RNdBxt%n#V8>9f^@Y^Dc zq&CLXftm0QaYLp5_=k9hA%G~oD+-Bn-RChEJXOg1cB9ym3UWHZ z{X&Ew!BlWMOQ7`uZ1mrSd$Af4gjUMgxFL`mo2CMuY1GR|p&sYy@PT;7G0pO?*D6vL zY$JGJwQFfyl2`NZd5;5;0xHH{EsU&)&yG6+MaxK!d)ijbhg`XED}y4nbLy}giK#ll z#gE+%2#k;kZad+Xn4!qR@?5BP%-9`>$AAPgLU@g?!0#r<7%X zeoAB_kD~b84t7TkyWUHtvjkNNpT1xt<=bjSOZpPfny<yP%I%I+oW#So?)g~x^3EOdx8n0;yXyu4-5^&r{KL$2+$%sQmB0wy2;J#M+I z?vKWr6 z)qB_Zx5y*aBL}Cswe)f+=yEzbL(UpU8YWR?n|E}m{%AEUh(w%8!i@z8v9Kw6;wHdB z|8TzzrZ%Bhq!rH{69ckiPsTjrTbnAn*XkNL1u32#8kIt*Q|^E&NXnT%ar9`&FXcD| zj&nJwV)T6+yOp&gU52bU0ScJmX&8}MF_?}6FHMxqQD{JlZG>wzeX<;DkDNb!A(K7` zVkLijtbd>{`;rxoCK%NpI7$qJ9QNXLwCVR8$V!>hE^mK7pImqf+sX=t+vbydO<~(n zZzNA8k$^!u@zr)UD}~th=iy8uzZ3;p2~nqx1@dm5^u!crbc!)G{alXql$1agP${o! z8}-0wB``0cLj->m-K;qGqJC3nbSvGv-gY)Q$~NT#C02MB&biU5VRP zSys7RuCS|EzZ83pJ|O@?^jS>+q(}tXNVu&LgFKAf~M<0`%@!sl~|l zLS6xTUsfRY)gA#xx3Sea=f%;q(k@_lB<|}QMf*khLS87)aNz%bp$Ne0vaZ{3J*7l; zfI>djWm&lFyf4*|lWl^yC?p4e1uY*MX2ICw;o%cZmg+k}=Mkzh(y?su$GkDZF?Lb$Y1hW8yOx=z}kvX63;Dg(ZYY?Yg z8zGGz1js~cz_4I>IB(sIIsH7GYwVBAzWoE1Lo;DS@dyR&yPHQ)3q({8TCNT)v}i_x zo1KmjAC2it(Rj+73<;O4o~r=Hy8TlWAVu)XXE{Nkc74YrH9z|a3g#0QZ{Zwgvj-ve zZtGkGe+q(#&RCP!PArDPq@dK~>PZOZeTdala>qFnQy3y@YaW-;QZSDf7$=QDsv~g* z3)Iuw<4^SuTl-$?jAfTbs;52Tnm~+mgeM@O)}9rL_ZR~Xn9!WMDwS#?$3q+^ehefc z6h8Ag(QAWFs)*f8tPITUB$h}RT|zoH?W)t4!)on;HP7U_!vRWCO%an?ka8FQs0@+! z9RL@VLIbS5MKnsvpq7@>&8&4m;DGip2bofVS|+w>cLWPEe}DRs3_qXar$5V)3@|3E zi=coK$CKh)9}=sHDEZVg6tqrm8{_T$l+Y0{sZJvJtY;};*tD}0Qa*$P$-Xm-I=pmJ z4uCJ{Yz6YeEw4}5@8@u~LU@}Qdnknqgf`y`3O(3i461dE{ucPb5*~_O>+TE=jT>y>b4JR z@WC&~%`hWJom?bdGm`xb$t77}<9CqrGeyc^c|x10IjAAXC!gYU!$A$^DlBJm-@q+1 zjJ^7A_GoO0lVAt;h~Bc+v2^<*vqcP0{Mz%eb=g)$%VW4tXTa%%9?-Y)eMe-y@S1W+ ziXTvbf6tE4{<41mXFWh0<<;F;k>Ta z>orXT;aqA;eQH zy~Gz8QW=B4tEXW*o2x4|WJO);9RM5ze55T1>C`z0c&w^svhl?viQn+~@_f66+WRs4iU8H;@CZYg)+jtR2xkCbfQ|fMQo&>7zngavtDZ96uFQLBI zZ!J=HTe=!8a|U+Y9X0%yZT!12d4WJE%)pzCBKa1DxYFU2c)!jcmHwPR`FMld$b< z2hv@QdmqXWYk^nd&5YEcswo($5k-oq3jeV8@5@yf5FZP+x*r z_g4sLpiK)6GLJBEg3iLN8W(20f^Mv!HovG1p}EJwq0Tv%n!J&6jD|8()O#Eq4al65 zcbvRs_GR;V(iyy-c2EV;d|Eq;mr}3}L0)0O#DE#Kg|z7#d#$utA3zf7Ibof zA!e(Ada(G_FX-fB4_qve3SH3=OU*DD9OYn^Q|91s3LB1XM2o@C|3vSo+XGXe` zD=loKMPh&A<<%P9AIXngo$5m3R5G79)o_AR+ou}y4TYwBqe^evr>5vrbM&djKCL^| zYwu@-^Yu)H521Y(?938drd)r&((a}p#8w2Y0`F z?>igb8()6#8$bTq)h#`*wXQZMwmE%Oi&s zX}VJEwgebxHN>(Tmx+lwrp|)uUH~ciPpCw`mvvUL%#7{p4Ly@@0-GkPc^>lYam{{*_&EjJ2fp~z;QqM z=DKgHeqZlR$ylGNcUGye;U;6&%-B!e=UejYqA{zAzx^yVOf^L#Y`jUkqAvTXw!74r zZ>={nuGbq;(@k2|K>+qcKcc#l7n>`0gzMEctLSh6`>AXXpYm-Loit!S`P2{F-_?fT zCwA|>?g~7)|RsxBpdxFNc?`mSiMW6xLLwK4XtO`r0v&e*$>`OQDv@~)@C>d2NF z>%L`c{*L^epVIoaSnEys?R8skuG{)u`MdM?d`j#0##(Q#&QpHJ_nY8*_YIUZO^e${ zjtph9R_}S3kY6Hv`}&7_dxrWJwtJVa)D9H*}Na;+DoX zU5jO>zAsinKmo#s!3C*>#QC5;Oe2(pE%ThB`~VsT}hU7rSLl6 z&*J=NnyGj?I@mN<)v<1Sz7{vG$>i>8t~XhRbu?pZ3IQ_=tXr+`->A!6Y*MW4oVd@klxKx&9jsTDL{-n_d^L@#W}pg| zs#1AvOcb-AEm2i_?JrZAeFRj7Rjgf@#ji!twbEE*Xi|8&o(xJ}q`-hsP+)}8_AdUB zT;mYZEpU#r)tVlt=3<&WtLIEu$_A20)T&y8-cby7YtWctFSI2k(cwm*q#m=msZOfo z26pyh)_GmolaZQ?f@ErNjoN#YQ5I(>rit@h{0A>^)Hk7w1Vag(IgNJBNEip}qte*Y zzoeDLMpg^xWZRCiYC~tMONpv-6GjEz)zGAyuQcoTD=RH-5m(mvh?=gAD@{63ZRPIK zk&)q1Uru@S2sr?wgU7RW?4{LqUg?a!4A)n--S~`-$$V*LLv*kEdu#P?i_o?9SMFSU z_4rWsP(&Ar!~n zI%>|!pdF41mzPT~bJzbPJ>svyz&g%p%lE$Z+P8k%@$d&7cfEi2Bg@@~m;1-Bc1(mn zvsXI~t*&coOMTSXvZ3YDq16qE4O`tKySV53ndPrMx7>5;YR9+yUk9&t^u{0fcthfj zo&QN@`SBO7ZasSG*!4|!zS;gp`%gR4{{H^u7e=pkjD66t^Loc!t8LAC)4I-)Rd(EsU%-H0;>5+LE|;BKh(9 zH4XeywF|p=|JKxp#bh?w(XyJzCYxJK>8sJZrQtM6<9Zc3cr#J)8c_mGxTm`;Uj=R7{n*J~4f%%3`f#P%QqW&G`76?8 zel=Oy817t3D&KOZ(|r^FonQCsRL<{91WQ18$Hk_~ zmT=z^eLS-@{8jz;Oms)}TE2BD(u;UX2aK)x`Lw5g*cOd|_kzSz1B-X6a=tTmdTWA# z*TuG{T}aLT^vZ@H+(xISIpM4HRbE>SGt;?N8>i+fE6p~qE9(>u0N^d_ z88KFWW?Un~ElqQ6OauHc`EV)m!JX;L+kUv~{W~+SJ@wwGfK`nvX4FZ|`Y?SFXU`sO>|?0Tc?ie~ZZJx^ZU{N$_aKG<-_ z^^T3#x842bci;H#wQb#3wspTV`{Svr+YVjde$St5{KJjcw(q;LecyW<{(Rll?MFUd z*R(P9Y9k!3W8VC?V5|HZOlCKU(E4nd0(E+fpqRzHmRY%~j@5&&Zb7_K8d~ z;i*I-Fp`VSOU-qeMB+@Os8^qY;Y8+-*wxUl4!)0!9@X!SQAOsB>h*e?F|dLkMy8{S zE%~O4>o9pVpY4NCI`S<`P3#h`@VoWx=B0IuG!zZZh;qbZo?^NR=qU2(0MNGne z@|(^)5~S0o`7%a$z#Xh792sae zu71$C(jH6ETxoIjR90G??e>#D@dwhJHaK3Hv*dh?ujFDV1=;d*F_fYO*r7Y#Ei$gM z!GXsoIN6;%Erj*>m36u#%!%!SNstTaumD&Pc^Qq<2)SQk1@~hu!-~-d<_D zK;;Rf=ET{R7Rx@!ue7@&tfPl8F0OQjjPB8cgC_?1I5pDDR3+gx6VP34?x@yAXl2 z?&}*nzxUnOzI*+F2j6M_(|ccUcys;lt$*+FKYR8^&tBVj_{zq^S2y;ru1l=H_oGD9 z`j$)C)sAG_OG$h#&8gMaL~GltkNxVxPd214-*I`}<;puf?>w=ZNbY-V@y>}qoqppxKW<&VvxgfFCbwVTe)pej_`?m$+jg%e8uol+wXv~n z|HrL~whgb2|5n=vt?TvA_15(tv~IiBdf%1S`~3Ys-uW*w|2%VT|G<^~16TJSr!#H$ zf0SrwOMTpuXzS#G)w|YPU++z{rha4TSC+1=yX(rjyWU@S58r*SDf!!585}q zey0?+w@zH`Y$f; zQaDJ_(tRf5KI2o8aZ}0JKHqRmcVYy#l-^0 zMVwSXzy;WFK_Ux*VQdimO~E%;D8y=oV$f1Bg_9|cH%mrfK%-n2rea*IJTAL7WfbZ$ zSZC*x>y-pQc)Pb=DLN+WV+s(8X6m%yMLx#mTHp(cI+V>)Jx0<>9GZRG*KUbI$+59o zwSU_Gq%#%^bEM}**zYH;L8zD@T}hTpQhFAja$y?Gf=Q~Xq*<8739nNE^EU_^O~YbO zRnB%>c_q10gG}}!2njCpj!K8Pahk{tm|^ibcU^hC{-oe)o9R84C!|*~5&kr%3hNC9KK>Vl+c^#rtoeYnqs=c%qxRcHO zrr%A#k2|iWx+(UMjYJJ0=-!nO>Pn>sWm>P!Vi{B#n=}>Ta&cYsDOxA8KE&eU+(6ze z(X8y`x~?&}p&6%}qO_R6c_w#)t>bI&K0kOcm3l0be%{!nEn+muu0B)*nZpUX``KqM zZ!}DhK(qLvg}Et2Y@y_e`uIX`R-;D>09mLJ_4p!{`C$%dngWlqlJ&EVY$F@7iyvwomSv; zJb?A?dn1GAf9>4tKIMPBm2_MSBzrstiW5O3snQR(mI}JI!D%JJS!Id zq`l5bz`S%Xhg!+^RO9(2%5j#{W3^lCTOSvP*^2VSzCI<(SK||QzAIRywtz;H!*^!7 zVU@xx-jQQ@@d0s$4;726)vmZuRskQ>PBA={#Zw})jxBg0n?26i%U1O3V9#(iePY0Q z_50HO6#PoRNIAN`^x%n+-oay5@(a0*(X#s5ntH6Rur8xuiNsf3BA0}-=cQU_+d&$> zoMd|csJ%FFd=&~qSNvq-%ZLf7sNz^q+TuS;$KZzuk%D8I1t!C-y5C+JEiqQrHKt`& z6aHCHay2g&M?bINFY$v=;Wa3!UVqOO-Qn@-I*Y2h%4|7bfba%QJ;)EW%-rBM_des!W0U55K(&0IiBr0Z#PuuTLpW8(QVd72sOG=-+ub29qI(%I3!q{y zic~zHXxY#gdq6BQ+De#RmEdXq#a#71b;9a2ikCXUqiNSM@Rf|yW>YbpGc_8KMCtD{ zdv;E2={NN0RsGtlUsVON#T3ZWZ|drA>6g^UmE>&cclq}QH`xZ$6eCz%dW)O>AsoEC zlO@t2vTd*Syw-fFXSJ!Z?amwYuGEg}?H#Y~diABZzInOxk50TX@lM0?9gm>6?cI2N zBOea*B|@M1vJWy_q~;SYZ$%nfvw8_m|SBH~dU*EENDw)`{`Fl&R zEnVCAz?F>;yuWc5I^L$I-Yfmt(vOy|J=K5Zss5`^y|Db`(dG5Wu5=9iq;t#aFY3XM z$C8P*t*bfvJkIC4S0}8&JfGXYx8b!7s|EXeGTGGjt>o&I{M`DRqMNP!Ah z6A4;uQIzK^$#O#y^R3jre5?I#yXg}t`Vc8#6)CFrlW(7@XE5woeAj6!x6q`d?YJVa zRNMnM5|)d~Qp`!lAg~J{kulc@SzTCMa``b=&MbCSUVm9`u%wL{tBQYjx-fB;^Cjcr zW;%C)SD1Ly)}B#LG@c7h(kFKZ21>0@$0!|t5HBTA2S(@&f6ihvETf|<>1SEJPLF{* zEKp;bx(pF=3&u%}2{w<|A*e-IK}Ijr-xQ{uS!tf3NPIqbzcIfkqo8LfYc?&(9Pz<5 zDg(UR5Quob3&T*rc)%9^6@~OL(Yn?_Gv0DqI&2FySk7B7@S>=_#AKh)DZGVE-ix<~xf>YocdDg>7$7ByHg#%S__112w&Axeft za5F_dA@b+az4Vt#m$HAv$B%#1Y9xZQxRr2~*f+Qr-$s;BA}C2oTUF{%32VA$o*7XaX;tLlL@yM%LnndzaJHY*ldb~9Qrb-td>R$El%33$p9eo(4weQPIqAyDHp*=&_Ar_Zo#$G zZZo4_i69-{;5Gd1wlC+shND&7)pNNb_9Ir?eecRkt0BwGEb!eN;$9P1$mF(|C`A`L zeqtn2Bo3))vdud9+GSRn8_TT(qQ5AooFhczmeZ>Cf}Eb6LY+WmxNsoVqvRv5e$4{= znAZX6PwIxpFf6svv@4INd2sV#w@kx?co@o|Xh!z}C0xQ;CShl}*SRKC22}HCv&Cd& zQqSzOPNc@h#$0z(rnsQLIx%*mstaQ!vkI&80X@{!TGa`@rv%4WNCsDGg6C{c%Yt#k>Lw6ZzounISCzrkZYqbveWm}D=iUbwm;W&zE+x<~I`GbH_+I;K?LXPP_nmLP(~Iix zXm66=oqfrlZo1?3@!xN|{!r$<-GBMeH~wnV1Mffg&DDmc&Tl61AK|Xr^6-@{4=-=t zeR*lMy}9#iT>M5~@-M&M|L3I-`TGU#Y8*&@EBRr2YbV%#bF#DRdeIf6X^&wSL+0y#NL}Y6J^L$pR%BpdU0n_hD`O|;ClLDl>1`=E3Fo=(nes40#;HM zu+si^GmfneLOPmDZA-1VyPEk9cb5y4m`da~{%P_e-<-L{drEgKwGrBK%X_98ti?9n znco!t%6C@e39+AQfJ-F`N~vC3T2~E1xyfADo7KdbI#QVS=NfGK{UD&Wb;7XU^dt&kx}()Y6m z_=l?ab_SagSHT@8%<32*)8G)6C{U3hHNAKj#f@K)a0OiK z=B>N++TA?1d!-SF{tp_e%*L85&C{-V-AW7kbdF!a_l2m9TK+8IDjDa!+oa<)XoLUT zIMa*&wyIspp5|e(gQEtxod64G(7UIqP+zD>5-vhQ-auFtn~~Yf2k{4GCOAE;MU)&# zUb^K1dd4E#*ckR|Fav4b1Qu}%fHMTa7-V*R#PMVq+<6X;2?rcVAp#Ql5Tmsn%yhLZ z-oHi-bM*j7`|?G(7yozXch$tW>SMT4KpAF0@TJB9QOey3x#Hkm!B081L}RE&vPTyA zQ~JGp8$yL{c3dGl8@0mNovSc-1*k6nxr>ksj9bgHSg=yULLf6k%4SfoQvvrcav~iH zXO6@;R8FE`a9?_j-E`)lut=YC6;+(Y0-XkvzpR}cAAouw@RxTrI^AhkoVHT-3o!#?wPaFcmn3JeEZDe zH_~Y|4qDSO6&`KtmiP+cVTM8=+Y9V(IuH8Bi?{f@-W*QBJ_p>@R~vp9A~CcyJaszK zv8@%TOge7$nP4FgkD2x~0r{4uFFegLW&V5zu%%9STQG$j!k9*3cHmVlW=aZAUECKe zTh)U<7|cRtZgy^FPF8vyf{hQwzcW+1Ab!!VUoyx$qw}^>L7%7jwbIVcMNt@4_@4z3 z#S-iOp6;)d%raZ)Y<*-Y5~KzCM6*b+qsS-aXDoP13ePU}}> z!Y=)1`tvvS>$mmmclfo^h#AEhwlx-EO#frK`YY=gg<={@|AcQ_08iz=0!>{?tZq%T zw!faavMz1$$WOjkes5~|*%z1ZI(c=&OUtbdD`__tL?itZVy?@BYen-@4;3 z*WGvdsmsrOu<@R^4qk4&JoL_E?`7ZLxPQ5IKk>(J6@G8&)uHdb_?s`jv-zLi`GY&J zbv$*Y`tgHYeJ5ymjKO!i%hA10DmtXmQr zHm$Z1U-e-}qIDw?OVJ`Spv}8G6I(**9N!=ZNAoe*Ok`0E|34`v;S!JFB<-*4gdXyYe&9)<;b_M9{Hu^*1N8@j$Uu= z_+HyHS>d+<@L{8X?o@sp;o2;f_0u~;Fo^%ufS%b%<&ue zD{hXDC1x}MPQo*K&j0xM?~QT8{1Q=DON~Elc-I5*VNCZvf~&a-zVl7>Vy5!V)#Q&V z^sAVeBTi68-75({?MXMd_D7A#T$-bhxWgc4%G zn$AQ?>tS^g31Y$Sj)7_BY7r|xDg1sd8T~yrrhDIRej^zmUicTNw`1`Tj<0S6uk)N1 zV*YaToMIVtQ6~{Jk=)YN$In)wAQ8krpjSqFr8o46ZX|M#!f66mcI^u?DU-a`BQKFF z1TbHu*qIc3a~AwN9!n;#uMAa^jb>g^%$)->Ys1WZbeY`XnbPlR`hH)(Ra%CP{l8W6d-y}Bqt(jh9r1&emPnK~mZAK~Kk9$Dda+2It;JQewEk^5La*2TUR z@{c`a<~oIcVXp%Svv?&-PAY%ifG>BXUCaV}UirKL8lqCL$`<6aw?it0Osp7-b9tk0>E*nTpL4@B7cWVZkC5!(U5n&bw&xV|Q~Yq8nv zmWz$_a1kI>%5ge^?MD^*2V)Wl!UV`7%hC!}stYqR>ET%{fV%X4{#yKxqd7xc0M|4A zlvOYbkkPaRhbVd29cm zGEF+KTEPOrEd1EM-2VZm(ko`?fR=7|$nYv9f?x}l936(w+@Z)C&Si627wOXQ3r7~J z$hO8~sAGRLQ$7aDus9O~_ra=akvwpmab-I%RA5YIl|5g`ovjj20dplZO_f5r>^lT3 zMY*!5FPYkbv6#{+=Xo5z)S>>~^f97g21ahc&-M}z!TwB z2HW3EuQ@Q?TBCd!Rj}R)+tcFMTH5&@wlwy&qR|w@3-ot~U>VntI)T~>aK?d!@~_HU z&Vyo&Ytq$(df$JDGNCW0hX#)x#eSXc86F<&?I&ZpFMYCqiH99@5Gll(0 ztv(bi5T_H4%#6w>+1YFoZDArpq@dItUDw%$x#O_YcvN7o#VR?88$9LtumvrPY-*ux`~Paw~cx7$l%fJaL>`KLcXJ&8TD1ff0N*}?rL-x zoUHT@5s!;MLU_5L_Bq1Et4QPTcnsL$CtpYan9o&X%BXJxm$kY9KTA|tB*7#wO+Ffz zEEhPor-CWZiW<5t75DkvMvxd%Im1TUEv5rcXSc2-_bras$h`9IzkyJ)t_`Nkmjuu) zuxpK1$Woe0f1qzoDL3`KRw-X;oXNe?a54n~mY<~yB#dg2c!-fLLi@x<~IFD{>aY5B#MQJGIA)7R4vyj5P_8HJ)X zznFYxwRv6Vv#XnuTee-_w*B>qw|2buzIt@a5cPJ=XOM zL2GwBOVHZg_pY8u?q9!Q_1nqB_PbTN?%KAlE8Du>-}VSWYulfzn`m@p>x)0R>z>t7 zJ@_$!YMu8IsHV>Zs&(!pNKFk+@_Eaf>)%*UoSObVoowvPX_~ffd;OQ+dhCsbw{pM# z?Cbk3cV2$%@-9a8@I;a#6P^4n>i&-!pOH)JfLvM!LY;)53hUn}nk~Wd-Bf)>B|iIY zJXFn9zZKDL%{;&6M+wrXWkja&yNRUWn7=A0W)*qN>?fuWH>0RJ&s9Ty9ja}JWos@e z{kR#mSnYP;+2P6jdi|~@7TL88cCAi)8_H!JMYnFu+s4IVjQnwhF=kor9R*ilG=oRL zjm%h$!7bwI;7bIH3(t{>uL!_$7Hv}(7kAvKE5;gkru_#Tyoxy*%-rWGOdv;q+Dk?qEw7uD8Gk!e7o6W$vub9Bs(k60iv?Q`1Wj zjKV4=E<~fX3@wLRBb*UtCup?-20}JbyoRR6;^K6qLtU%17wZUrA{w=C^YodnoH83z^XH zbc{HZH*O)1)}p~>Z=*+hSPQ{~^+nL(wG0{;Td6oHZ(U9m-EETHi!I%+meee~C0aru z!pL<6mRtOvQ>i1;_HT4(Sx$szyN){xFfkS)!910cvsCNkiuSGi1tgRhx!vI9l3y)~ zxo18UnN^%rY?Aq;EFN*UX)jF8Lg}1>7BwC2doxsoz zzEh?Y$uGY*vHXow%bULCQHB3!wV}DQYjv|Ed(ZEk`>;8AM+U?D-S>WYM|`ji)j>Z>p@57H=8?*U+73 z(632tw8%MApG2G{iP(?k#qv6HW^s>C7mY<>qM;FBq-ZVIF!}y3pv%b0N2Q%j)p&5@ zI%VfnQp`k6`JHv6{_B6vH~)=R{1HBou+YB#TKj!h+V8v8{@|7N2d}n2e5vnW9XzzW z{;3}~y?1W8{lIe5z+W8Xm!)Rs#+geC5*?sFlr!sfQM4Fc1_zG5>@>}4?iZ60!ZbDH zm|xBpxU^_lBI73JrWa=Hp!?GWT+48N3}qOs5JnUjrtWab;uKvSa!f*43v5@0Nk(c7 zAL%)sb)r%4GQCE!LRnZrYxiNsti>w`!|Ge{5%VWUY1q4S`8*toeIl(UC$5i4#^S4q zX@gs9w{~Lz(OpX?v}wvWV%LrL5F6XHG`OJH?mjEm?@9j=nz>`Dqd6~SoH!qVW#yv{Z4g6 z9h6e4lg+7{Bsi-x`<->kOd@B=YmqT|(I=5}ZjNKCruhtZ6o#y5U)kI2P4dA)+&ttg za3Sb57GfKPHbNJIeY{cUtCpD>vu3C8MfUj6X{})t6)q!@HkO@JMnE)G_l9LW>+$(z zC!oR}tZP12)HQ8!4Z(6_c|+J>sVs^i1&^?Li2!BQ>0mBzYp-`u#5yO`Xl9E*HzAZ5 z8+J8pRwiBgq99$1y~b3j!ms0nd^gtCZYH-I1E`u5HRYzGqs)1&OpcoUSW1Lu!@{f) zZn=A4VNYe0I(LGMuO2cU+(!|B3oZW-dVVJ0U(t`WaO3dL!|UPjl*YaxV6fqj*8~jS z#DnQV12@!oFsV2}`FH=u;Y3vh%h#<M61+N^)-^T z6e(GR=+zAZ$EQ;8-l>F`p{KB?eZcn6j#X2=HhbJ`e(zXbl`(p-3@g4=wmlroj z(Q~0g>BalvL35U2<-+FNBfFh%Rvy{XAMtB37Q=@mE(2V_vp;iAmvwLP{a8^9F*lVc z3h7*$n}E>kmS~7^zQ0pBSf%f4&IHR!Z)@Vxkst3SsYr{Bk5bU;P6EVysdO?q`Vp`Y z*@g7t86yS~=UmElObV-xKuAe)-b61bku%s`PkR$@BtHc{kU~B|*EKI#pu0Djse3o{g8g@VA|d>XEeBQVs-N$fnR}ldgH)*% zSxm0qy>n;g+%sp+?~}&P=%bm@Xid~xPsx%ka1@JIV%-v;^r#{}*jHo~E5jm*g{g`l z;!#G12OV7 z&@y7A&JwhZnfkhTp+0bHWc=7^X*rI-c@B?_ievleVJazc#v!)LZoA$gx(ohlo~r90v;eMQtd^1i*lu9R6OFST2&i|MW?*qKVqY)*02}B4fPW<)|v*b88_d21gGtP@4wqX5mABInJGsl1w z4lujrV1H)U$c{!M9@+EG^1}2euW!KfKax2c2JRr3O+q$f@&Wj>!#isD#*4mpH9ugTDHT8KvpV6cpK<7PJA2xtqXADm-P{D{IpSk8$UarSEXc^0C7=% zv_VD%@``oTr}8%BRD%>mB;Yqg9&RNk5h*SsU>y#J`Zpdjw_L06=d17ix1n=~#Ro|0 z)CRSh4Z3i^0gf^+UVQ6O6U0&Y8SB9JB0Q08MQZ=oKo@Nr({Pt9#1T01wP%Ri#Jzd$ zf&}88vJw<;SN@RySIk=fHxaWIT+k~PN(x?7A7ktgXmrr9I2Q(YwEFO3r!NuSc_jQF z#K5&UZ)P)oekd-csOC8}iuQ5F&tK~5S85h$mYY)E*s>9r?rC{z_9MH(@EsbkT&!Bs z#4{&Hk33KMWeK2zHTU0yMC*bUdj&~2(EhOQxtq2q62=o(xM-KKiQ(2>Rc;GMyt+k-DG4Zd)D zaBOLC?AG9!cgC)_-0s@D)V23^*Z!rh{WrUM7dsDp)VI`nXz}c;EBUB65q;6N_eSsS zeM3w8hRCAN6u%pdE6%U=Otey$FP_nKx*xdR_4rcP;~#zhv#!JM_TGB#r4{BtmXNr$ z^%!A%oyB8O(wjZ@;%8k?zT5jr_sW4T#qIS)tpxiWyq}5QJw&?s>ovdF{MY*)vcSGt zJ#aSamD06OO4oiVUHdmux+?lDFQuz|<*TirUzO9NTHsFYnj6?UmBk&m;09i6|1z89 zxx#B{ELL~}irV_CC&#e_>EI*JsLxM=zX+YFQjizQZBN#l6xMmemyy0zhyjBrZ1| z7~(uTRT0bPn9BJp3Knr5Ut*JV2D55Au*oXuYWl!rxZ0~)gA9nanx68fM!aQU^9dbG zN2q}@d=$qmWrxB^%Ond^9%oLDW2wX~klL|XRZVKl9T@^jw)hLPXl-GCd>oCBS~lhS z8b}zMVWuTm5(x@JD6ow4kbLPRhh@w+Oi{2wnzqrpIF^@EeKtumRKiONILZ*f z=gIm@HA|*jSVpvln!Iqs`iID*sTpz_&O+2Bis%U!UhY^z9gKW&DbcYt+3k?dZtWPB z7g6kpd3`bkvAUio!?n!#LDu3sOQi9P7ijM4a-xoP{FUYa6y(p^Q{Kyt_Hz5N!$*!g zcRN0193Lgt{T^_I{T1+d(m13yuRJEqpYV4do2&}!(zgaCFi}cCk}b-E2kFw#RbWHG zK42h7`$+cWqiMzlrj$QUdmG57kPlTcGlb{kn927__m+UOb{myb0L1Z&4itv}q&!p^ zSrYJYMTR~7Pg(`)E4Tq>2>+iZ)=^@QmVRCaK_2fZonh}*|KfOKSbUebgBgIC5;cT? zpwc~n;LPU^k&x-)nUVZuS&W8b8dVEIgu?FYTrZt%8R-}NGGPYs49qZ#1?7(5xwW~$ zP(w>hyE55=mC}LPclzhyb4=HWsUz0j7Tt!%HdKcw5}93s1AEElu2)*%3+Hq~Z91ijA0 zz7&{3h_Fz|1BHy%FkrCn$a!WQ!9iM#t&2Q#p@{)`ddj_UDBC@8L&HTnmC+IYq)OiS zY3zAa4H}E%5#Z*qnh$6R6olR9<|~%bfvr&SdHx(_aTTKszhz>Alw0aSuoeMmoRv&Y z&y)2j7$O{e%`jFPeYP|aEb(Q&` z0z=$CB;}6AQTLmDAX|ugxKn}ae!AV^+CtOB83FE1&T5gJ}NeUgQEl zpf!HPgX`Uh<>`a;+ZY!qThQKAq(a#?=bV$&#Kq$gDJT0RCiY8AJR~vkP{{vMTTxLi ztIM=pVDHUWF54fJBW>3W34N76g-7;8Bjo1!cKEaMPPJ%2d^CE%thk z29o2IEA97lu2gsZiYrF(XWhtV5LfkH+oxhr%5dqfHtr)Qa(DkXy0Wo<37Jn2hOID^>Ic_=i&67N-xi<68(7- zM5!>JS7$jUQx63L5qeduK64fLEmy8@oF4JJ*y2h(bGxXUOcd{CE357K!t@ZcTYz7 zXab_Le_O|5t{YGNp8J~?a~;W4)X~_u(2(?X+JF7U)&~}vlikgYo&30CXXjIkxwhm~ zw6*&PBey5}_fRlVlX)p>?wa6g>&lK?&E|J@+=@E>RZ}08k{r!yK3CJ3=+D)*+Nh1d zhPtP;n60agGf`AnIhD_~JbbgbYoTeiuqA3U~zn&Av#DO%t^mtn_g#>6?C;{5gP$+F_OK3Y%z+8Oi_=4l}o;#D+ z7|B$XAcGB~B9RVI({)qniZniSQuk#qT0cyhG&waY?lqH;+I>LUhji(tickChpY!?B z6uN1T<>%#||MUL;|Nbw3X=(}*NY}Yfhu>`>0y#qnHRrbBX4+wHp_Cpzx+vEV0 zP4YrH2xYVEhM#c76(I9}@*+9(hIb$=x64gH*#dOUa(F(a{D9m7R1xTnT;b#nc>%N* zP={w zq&k$NlA&suCxXGJw2^FD!G?%qm4rrRMHZE@Y+6mIMmleeRK$cPD=>v*h}xj8(BnX4 zq!f|nGBC=>(M*q+rHZa(5{jrLMc6?$X9Qu+jAn>RM#~MQI>iGpVnm|H6e>z-UBf*N zDKtyfjDfpX#FRucijH+yC`&atm#`PIjMY#j_OcE{}Q%_zf zftR@1ze42_F_3_zBKe?RAFlJ7-Gk)^^~On;;jzCLUBf=R?vzFD`^MXA(3LPc;Eb|s z|J!JLmVQ!nIrSJYC&>Z%Sm54WLGX3>| zvW{0OisIe9T`vRg>0OUVN>^uA(iN?1RHMfXswjHb#^X79d~D;EuF;L_l`$oeGc+Kr zbJV6T;B2J}*cJJ&Gn+S?8Vlo+6cB@~g1B;>hb|wf`q$06`H**p@E-4s zmozV&_1Vo?nEl%<#lr{2Qp=!ni4?heF;zhzq#qc!uw}m@fi!DI3V51bwKSLpF=T?P zGf__~LsB9yN`r9gIYY4?vyuG#1%)nEe5}?vzE&p2IGO7dyRn&EBI;16+8|)nt!D5x5Tx>|?1tuT%a%E1bK| z;~LAIS?roU8`vhk4v>nK#6SdLX%(T-`|jWC?EfG+w{w(?xet<2F6J-%yjRo>h9|N* zq=3N86C}jhd8)agihIPP`?QR5R2)&H3;>Zh2=E*P(6)882qp%qcr}w!sA_;(x0OB0 zr?ns`evJ|7&<7O9rF2daSKw4DI@x6VOdeaN7vK%_y~(YeWk{qWu5HU@D8M70V8oqQ{Ko&QGPk{&fnq40^cl15%`FJe{IVdc|P}8G~8B}U09RNgv z#uEd^6fnTOO|g@Fk-wKY1tP;;cYRP-LL4y~$_BJ=!zlYch%_94Y?x58`wCxlg8~DG zLM6ZCPI`*G!CVs9DY;B!-e^GMy+F_wXrT05!gKqXtzqH7Q81|Ds(iHJZFGuH zN$E7mI(QCJ5@d#nGf-ui6^@Pq$Tg_R5g^NV9*DV2A1GH4W!>~EV_07H9++GwXlY|4dk3Eortqp6bcDoKk z)V_<|m$qKmIx%!j`l$P6>ytP6UJ&hz{P>>Iu4&$Tdhe;dXY}%}n|w!2XemASUCRWG zdnSNqpmGVkT*oT8mwo5h9*(bMG^`M0$9s7+wouqi%=BDCs6fxTIdYKXNkfZq`(j?Y z6Lw^J@HuqG2G&~#7+ACC4eT0AC|PuSV;tQD)N>6viw+-(;@c#!dn;rB#4fidIvs93 zUl=c3>7Mjl?ztYFTD<;N$Cjy%E!B>#)o@S6-}BA15Cxd?L_Z6+0=jvkcXfpL*Ec;% zeiP{B+p0CsfUjTCh-shNha-@`gFmq}wN;OMum=P3I0S~6)_1GF2?`t$(` zX9BN)y_V?)X{OuZ1Cw(2iqC7w>Wfe_xsmyr7u3G(@zVo9G7NwHDab$|LnN@Eyt?Xd zFZF&Vgv#6{&jn9et+t52*!@Y%!?keqGoif}ZmkKS^N)@{T7IA^EWUE$T2J-C%~QhW z3cs0&4e%IhvBuM)#nOjq?Bk;j>fZpi&tJ^dsfhy@F~dqp(f zSga_iK`cm2YeQmM%_uDJBthf~f(xUme1nnj(J2e(fD4E0gf?Dh!JW~>!VYo$`g)iL zP`4iQ0&2Sf90-eG1gjD1!eE&l>tZ}q2-i8Xu6D!%FD~CMGR7)I>nu4pTqPD_xNIXt z1@X1}hM_Q!VeJ?~B8Vsqj_7`@IgIL=!>IkxW%8pEs5JMGX6X3XbCie=t|ku7EIkO{ z$KbEO0T~1>{1Jy&Ymw;11LdbH9Y3su4_5pKYd-(!+^O7|gH_+6E1N&@t*AxB$;f+c z@3c+ktIIc5#7#F=eK>kUt%RSf_@A6^Sv>Je{VmR)W##6VB+b zpQuGvPbMZ`zP9R`aV=TtK3oYOsrZl7gf@%ay)4RbH~1n1R>UIs6qe;%x5ce+Av2}{ zs#!b&V=l}%w$S=DY~qcA!pdUdjz< ze%sex;=#ffLM7j<*T;Ehqb_dgY|zE^bF;xHw`^tu+1|@lUVZH=cZ7?8S2DxOB@bF< z=i^rS%mG&3_5@}W~MXl^dEQ9X(h`o*DaYkZ8KB<(HT+EN$cq! z{XOsQ-iIVevl>0O@8`ZB&-*;T=Xu_r#A0Cq&v(UtxbOp65dM`C?&G&##MRTHAY2zz zAtk7y>dA?DPs$_GyEo^}ODT!JOF3WOpYro}UoMajrh@rUDulFO4dlZ4NGgK&pc=|W z^RZM+6tt#PTn(q1J%T2wk?%&|##iX^2*(69c2!WD-uBozZ`(gYs%68vanx;ISGQGd zNwsZQzZLb{*41yv@gly7l%yQr~Qb1GYyl$E-|e916nqf{*B zW_QU-L6r--T`8|KEg24}9pre#XZ$m)e% za*^rzqG{|xy_}XYd0s9%&J;GAKxS#&FnQ|%b4m*Ow-U#FoKbwguA_E2>uXY9(mJMmBYsX$n?XFKBWe zv&ikjD^J~bLz0#kY-35&*bXc3EHxTdEgx@fNmJR$Da)@-Oz6z~=n`gRd9wx6@)@R9 zG_uW2$OzEFgP|;xjb{rZ<+!`#@|-+^kyt@|obN%#kr;dxNm^kU#nnH>%XMK+xWKa; zi7HMH2mW>**k+z|CYNRH=#E7}%FU(1bU~jfxGXP!-1?ZTT>`(En#wl=+@A39 z7M5QhzkmWOl1>{M_}ygbv=uyZ{LskA(UW6V_<1^fM;X)ETVQ;grU`=>1Q9L4ue%s!l1apc%#^Lm7ylAp-+S z7H6$kI*p@hrn7~tnNF8C)p1p_p=dY0Ym9=-3zbOxYx`f`e(yxDcX0# zy!rgC&wUX6lvDcp@g_ z=u&vgWBHk8mRP}(h-nfLE0|6fAic02;dC1OSjy3RG@brJNy#}Sfpl8cGsupo({&Of zoo3zWlx-%`OQes;RwADSc}rjq(|dwQt4sRvVmuA<72#*nkq^VME5nsY>y?p8Yxn%q z*N!Yp-IbR1`J>nNE=%o|wyybO*T$Bmu1dUR{>ZhqE6-Ns{`uqA^ku34o+SCsiuNa{ z_l-rzJ1BXO=x_vvD?a>n;=>PtK|&_Tg@WG^7$KetLts!@#3>8&lr38PsR&QSoU$lS zxq>FfQ_Z}rCDp`JtvuDCwn6N~XWJ8}c<^2unaL|}iiePe6yJT`bfHXrI z0)T2m4QNH|Ii-tAX1YPHkzC<&O~paIzg&Y1>NXuU0!87upA34#Q(=-=|1V98I{Gib7In`Dm1#x;2--@=_TV98Fj*ttud)J-|zXP-tV z>;RF+h!F3yCy9I(B;m6{Rgh$bHNr(4pjiG%rT}6P*=;>#65iFFmI39UTF-lC!6CxUz&;icm@^&DtF<`@)DaAfgMWz7z5ikOnD`=W( zBokqFlt%dpA`hH{<;$8{-mrW*eFh-T&!AnQ!Y^ixES;Y4L{`hGTFIlO=+z)xfUdm(Ob z%a1O}kKT<1+QQcYE6u_~i9g=*hg-h0bt$^#+KC&!N^95khrj;tHwLdA`KPYY1#hLT z^TzhyK5(a}ztXewR`}-JiXiSwR+4*ey|B3Ng%z*3=|yqX>)qVBB6vGG@A`!9QzGa? zd$=SYVY@JwL^B)04ny)nIBz% zqS{jF?Q=5wDm}gPBMZ=Iy_Ib{7o|-%_AX01D_wnd=Y9924qvA|BpOJF0*?MIcdFTpc@}toms|()&RM~XZJinHGw97`WMa#5+$|E<$(%lJJZ(wiI*hM4KBmv4VSn1JQEsvgCg53Wc+P5O8ldV$O-+-D2poX#=|BPn zymeq`+)_-}DQ?4R!4ypJR!y^KPyH1}*25=@kEcz{IMbkbNj`yj#MPS*P`oumf*S!< zV?q_7?$E*72TI3NYtv{1pv4A?SM_q$UhA36iw)|%=BZchst-yKAeB@r>TjUBFN#o% zt!AxX|D4#^heU?Y2I^f6OhXm3O?7R98`p%Q4ZY#5saP}Su7?`FO1*CY2%f5{eYf3Z z&zy%f)qO2OD!hKSP}O=F&^7#6K4NjBe587oNT09gmzrjG8Ov7g zytS*|;V^I;TR|J!>8!l9*0p;GTs=LJ#R1JW@5DI>S6S|f@^RaF!Sy#&3X>Y%pqnQY z(*59C1Lu4KHl6X9TrxEI;1lwm-Ra%Ccgq9FS4ufkew3#Z4ctq>4q8N9J&BiR1d&6> zbG^h7sxT)47PSI&fwR{>s_3pe6JOahC;pyzT@-~^`T&hQ-}BCh39ls%mAx?aM&Qf{ z!r@}fWV!#uliY2Ih5_DyUgtX93Ox?D5o}d8r|!ZfJ$-p=R#WOGd{c!4cehmfxV-RzHxu;)zBhW9F zKU@3ES}`G;$~0DU943T};md~)jU631^z6|wTOg7oG!Dp#AbSZNTCw4gu``EGo=iVG z`qa@e8w*XKfSo5YMOFL;+!&h5R3s8!8~aG`hiPmKIR!y0^%s{b#D z18sVjBE8>!b}6!lBb%MKq}v1gmj?DP?tGjq-plXljR40lM4N9%b}U79EDj!CiX7&x zo`XqwyKQi(ZE&%5H^4~CBfugAiM+9<)u7NZc-}k++ z5bs!OKYLRGoO)k=bh-WPa?{ys-hYfV@z$po$DUt2{la4N=jtFD?Dlv)Dr)*As7S$D zu?&b6swxM*4UBKmBo)&ZaqE07B05V2M6&odz5qgqC}kVy@wO|YmDY}BsiP9xv@C6^ zH1)#*uhf9jL;V0samwr61|)Q5S=#oqox8vFMZ!XbW$BU1wuc=A^l+tb`=ZpbU@lAB zD;=9{EVcQb)a7fk!4QpsCQ*HeWAKQp1<(gz6+$QyM+iQ=t;H&YT3p-Rr$&&nu}T+pcrj|#ahD+8~&YDWQ#~;f# zk<%5E)H@tXB}29RGlWD8+m`of%y#<@RUDGt{6?tn&b$PGkcQB%x7fs z2V!%lw7eoeOmGrT5OM~}@Ml18!3|#>S8~K&Z|FPN;2@4p2*4&Kk9oPHVVX%N`z_R5 zB^~l5+lnD$d=N<%0fB{i-@X0{Ra&#dS16CjxA1Q?gU}8?$B+6wN%W06UNx$c!GHDf z+HU}VczM9K)g1EeYKQ8->Vuc9Qw?A%{rC;yH=uT@A^Zl_uo`(?3<+!h)NVC;)ej3{ z6QW*l#;ZMQ6MiG=W;KrAsM@PG<2Qx~QVV{Y)IRRCkE>g`7p-|VoRF<%u30FAbjS{< zl!u4&fZE*lM+61Hfp~*rDKk^hP;NYkn^-u$P9SMY1MgrHopYblC$pI(mz8i9!}Q>5 zX@#nLoS}`Xf3R*1Of-B(F{b6LYBLD9wwMDK$}nJvso<7_)IfB~2|H2P3gIf(ufj9{ zEVYy&UxLF;V1@yF4=xevr~zyEOnk~HcdlKDB+vf=TCv|CLZ*O5FW8LT9tBJz-|XEGC}|}ND^xJF#yzb<_z{NXRrYp0YqQJa6^tCQJ-*} z9(9|Std{-oR~z;S4V%6uD|X<_4%;EzxNvV5~Wt zeJT1lgXyWHiIhqBc@4}8;)neF_Z@PW51G3M%9}SHF)#d!h5-ZOa({n@5GzvGAuCgMFNi_rAOE0@ z*8_HRM=N))TOz0AU26LOKS%qY%97N9NO2d&KGJ05&_Ob@PzAOl(qY?a%nHinNqY{? z5n^AY!PZ$8>@qSoIyf$fcyHrz-f#bgPs35qWfEfE6hYJ3EW!(Xlupp$?4cuPhDS%v z4V@i5Ja+W-^M{<&(bK0#PoIOZP~d`ckG^9L@Nsh_+F%V>;o4U=;>PL_ezNf_kjtO7 zTsT@gKE%vquF8W45iG>#c(f3L%b6weD3}WkUobJ%BIIy&H=Hf1p8wy+$Lbf~YN>Oi z*eKbcGzX8Ta}c^nv+`(sQ3XHuU!H$F>Po} z%+H<0S`ZbbgP(TJba~6V3Dg&V&^Y*Crn0|N?QD=#KIBZsl`uFF7Zpfic-oTkX(bEU zQwu0Vi6el9i2|ZigbqqOPa(RhR3h!E7x*DVv7sH{O9Fq4flVaTF~J2|W?F_VF%HFw z-G$uHh(9L~m>5TBY@LjARIJ1U)pY{GiaFp^gKJOI%6-*YIRGiyP=32{jj*8&&)N-s z`$S2_nCbjLbmQ*IS`mGLmrp1-g=*a$6^hI#a#>?KSG<^WfefeW5@m4P0*ioYlP;5w zU@4I#b|x)xo}hh_cYU4?FLIR$sK(j_9)SORK5N1$v2LMk|AZ}&NFy`|Bt~AM4WJV@ zMk6pG{cyd0wa=N?*=(HClE{B2$C#}T@Cw< zglj7P4N_NSgl|o!fo&9!{zAv39=Ks?=Z;gZzbB`6~7Jbt2y&LiQ=UL4?#Ed!5K1 z5V=X@n?(MI2<_rp6X+V{5&0hejjJGJ0*!qbCWAKKi-GDhm9Cx@{+rlm{~kPK|Bk-Me?JVxR(+_q+AR8x^Y(U+FY)&gP7w|BdwtP+ zVG!F|reQTe@oKOe_XuHjnN>lHD>=A&=w8XW?5PEG=c)m17W4>np1G?3vd{q!vkTnt zCHL9Ldw$tlyTCK&HLG54_adNipysH(9v2B$QN>%aEgh;3REH-@~-iY{&vFPq*sq%Yx*QA#vi~ zF)-4=S0m+^6U*kwfpR3_;8h5ma@Vff4I?f%Pw@{KiI5eTwC@$cD+4h=(wx1FIkPkf zu2&e``HqVXcBm+HX0~YGH?kxXubky^PcGn$&<3ZTi2`qL&~6bNwb5uC9$dO)=-3hl z`!f{(3;vB+ka^+XLd^?ZHwJIDFNc!zhwp6JadZ6j&%Zpr(Dlmnt(iZWdu#6Y(2=E~ zBlE}ZL^>8GU!VIRvg=M$*PVFV!m;1Hd^^5lDZXQ|=@U2e?;LyQu^&yZ&0qY)6L6cx zo`l;p9){aA6vlyww#|=l-i5p(_Z^?5edUKAv@4Xy{YoA5MgwL$h!`Lv5Dnj_1h3jC z8hA&E9v=Rs3rZ6uTwcNjB}S?UcT(K@ikiV_kr0D}>?E_91d-*(+Cud*KlL7fbq%;c)SEI1s7*L~&#R@)d-=#Xp zevE2#9E|?}!P1H1-9Dcv{a2G9ZvVNk?WaQbKMTp93-V8eEpWih2R@AKoe$pi3!cbA z{|92%FFbMathgc&UG)l{?iHSKx4%}QUqmb`oah!|+`%*XDw*J=%j@N$DVwJ7<# zh~i&(!eaZ1074~^6`t%7JZ<+pesNS>IKC>-FBP_}@T`?qUb;t&E?im_@Z*%x+msj* Q+y1jV67hT~z&Yao02Sk#)Bpeg diff --git a/harness2/core/__pycache__/tree.cpython-312.pyc b/harness2/core/__pycache__/tree.cpython-312.pyc deleted file mode 100644 index cfab2c3c8ab15525e22fb6d0c83f46f41575edeb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 25937 zcmdUY3v^Uxe&4e<) zXb`czjJvoVubr%@owM>;CrjFMQquHvoi!as`g9CwQoxlT^x zMO`o7r|Z=56xa9aJN4{s=rrJM=r#74I!!Fi*lX^ybXr*4)NAdtb=p|m+-vW1bUJv> z@ASL;u5-E)G}GzUaemJQPPDwuzl+xB>C6$Wow@jK5^e9<)vr3PjT0R&a-#EHos#ce z<;8X8A+Jqzz2{c*TDhw zx?Ef*F2`3vTrUcEmx~+3a=a_P+D9uVJ|$Kltuk90eG@l|l}N8byLx|)San;UJ*T)y zT!FS%h?~V~+KPJ0e_t^EcMV!>^XN%`17$>g+ZU>>+=Vs9@#GnXGesNA?cj2+ut$h?{2W! zc3kmGBY_J&{pW?i1z&&_^GmXD)*l%5`}^_HF9e1M1mAgT@f=k(JRn_?g&tWzdp=>H zUpSLd%P$Kll_#|D-CsZ z`_Om%FdW2GJgrA9Uz!Jf-Ip2$N4~TU^{Y`kEnSy5(2eiG{FA==;OhF#r&I5)Q}w4; z9Y!T4OkvR9-<>)>>_t`L;UE1i!dsl5>*R6lbw_a6anN;sgQ(w%6w!c~k;P1inOTfd zEn3W~#cW#4uEiW$%*kRdly$S12eBM2mP;`lYom5xO=2E$?`*GMP`A>+ zIjpUJ@$PNHp`bk~&>j_h{o;b{$F^xhQZEOjq@f=hKIje%4fgs^;zOO#(9m!?Y4!CE z`$puhf&Qevr$3N%YQMYsd{>kC>7Qt!3(G7cr&Yex)7#r6_DD&$5~a$!sJWyA=kxq| zztn}37)W}2LxBPH%ekJ+^xmF+f6{PvV4ycyC=Z=w#}W-(9_o?&a#!Dg=>O7&ehML> z9Rp9=aJ|AwGeP6niO(pvSG^*1{JEe8`ui%i&JCg+jW00*&>89K*^$Z5& zWTE80=#lp7B?(Q?B!jib5Bo$vO#0^5i zQyMpvD&a3GYTvFM+dGMB>Jt?;QA0_1ecVu!s9YH}l!ja5hLwrJM@UgisYV z6ovEShN?vM%CYvz14#bJTRXNRYOkI#)JlzNPd~9xclo3lJ6i4u`nwJItv+@{0EGBQ zf6i!*K#CUNxQi)luV$ooNTU@-4PTQ}uhImg#*CbqEvG6D8GZm@7ci;q;Fscw6O76LsZK4hEXjg+HV_2O(ES28=l-+ zMryY8(0cKeIdr7wA!HI6&<~k9IIpWcsB2sqw0j#n_U_)d?^HvdnB*ISC5@@0-*~dQ zzEeS9T@AoowLu4`t#bMMyAjf85zj=DFh1H*K%mB2q-0Dt~Xs)W+!Az0t~j z@#6h+`GvRk+}snXdv9p!{PfA_^5%H{_PN~rTdQuaiWEhxQEzj!YB!UwL5@rsYR8k!-BF{`Y!3J)P~>3@Fx?* z9^>Zh?n(14$4y80{Oq#E*s{jyqS-Yqu{ACCj?L~k7Ta+wZa*G19RKRzsVi*os}@1Z z5T~B%i%4JmsD#wT6nB@IPTZZ!RgLl?{vvI+MGtbMmO#!E^~hsglmS1}d9DCC1qi0I zM>K@EAL!Ds>bs9?LxW3FQHU2!qPdiNY=6>u`RmA?ZOrNZ0ODAJ2ipgvKCm@1=r?dV zy&%A^_!;F^ho3}=5Z9s{S7KPs0pWsvC$j(I0LUa!2!L+^M zCy=xq_4N+sjWU56-Z7XNLGM*r4K7X%t2uUaLCyyZ*<36_7$6Kyh zZn!2~aZB;6MTl91NPa|)TWZIe6K2axp%+4vEi>lgaBHObM{RGlO|`teFSc~`oU1sx zWW#j#^s#8kwz%u*sO9N|&3>b9qHZ!UV_TYVILCeCXUBJp^YePn?objQ*p_~T_{NEi zlk2W+1wo+f+e}3q1XITiCA3Fc7SML(IDZa1_C-tf zoUNR?*VP@{!A(%DSq?V#DLBR4(9DOQg@p@mwHB?IOxszP@~VPAj>9 zw!T%Ir02O4+z3C+UDco9hIy|wxcArvKhA*!J{w1f9S`E56&Zj^wX2*}gU}2n8eh91 zbZDYu*AMOiJiZU-Rc_e100GcmT1V94Q;59A@8x))fe?2ZAbFI}*i|}Eim5(+pTEWP z-1S@oH^Tj0nup3>Kw# zVAwAsSI6wt5qZX5OP@L?w$0ir zV)ly2x*2;7eK|g{W!5gl>_Q}O#=hbsG-%J8yga_`zJ1&6!;$jHuD5DuD>lX|Hs008 zE4IeVwnYuwKL4zoa}<8c@y3E*xbx@rcz-F=9DdKbbym>rP|EM)e_GHXIw$==!Vts6i&2Wq2~4 z{WaZnuK~4=Y9%}Hi)UYkvFPn=7kxrl`Hn%^7$$i@f-Tck<)8 zd+y{%DyOBNhTad|Yl&~%69 zmhuBc9li8z-#`7w?(1f5I;nx$0fOeF9aM&nhq}WvnvgEx`#+%!MF@x-&yVGAhz$Q|^oOJI{0(vY zhNxi!BVr)2uqf1_U(~iN)oQ{Yo98k7%ADs6@dV#P03j=JsuC%(E~M|^0(2VLlR-)t z(j?Dg2yTsRQUO#eclrvJ!x%DN)W~nOW=#^G z1R4)QCNG>(m;Veg0Nn$(>AW(I5F$sKC}=Dx9M zV$<~|NFK*+bB?00Iqnd~cFZ~RCYx`y-E0fDyt41-&hl|S;r84ZnHahL?Qq}JSb^AK~M<6_8(y>mdx7Trshg65S=?8c+!CnuNP3>aI+9if_4Z zx+D4-PZg3AdHJ`t+}tu{zH6P_63^QXSwhRpmiYqCRWV=4Iozs9Artp&KywP?A?_rD ziqsC-%lyO>k1{Cg4WOtqj&e2;gSyqP^71m|*7#_~)$oj~$-;B|gFtjE29ZG)<1~%( z69CXbz=VsblZGtXqr##T054epz`eXQgq$f5p5%Q}5I+~t$qEQB#a3kU>93N=FWPi{f*IUu~nk=mfj-n`cIDW`Lo=<~5zXJyP;8Mjp3 zx14xSAFn-eZ+N!t`2Duyh{ji*nAhu!rW7(@N2HsOAw4vg8XjoJ*UXI*(nKc26lN7r z@FM@7CVE-8b<}MF5jPz~_MjGRyQtqFcF_n#)Mij2Brrky*JA<@qWKwYtT7wYnjVMN zCsPL7mWG8%<8P7-v`id{vt_g7d>iS=s)%x zBa2L*u5O$P0G0#G(3mGr;bFbDJq(KU~?_}9Xyxt?iC;_Nit5#L_5*}xTq$_@LOFBK(LN6i25h{v&lox7! z1l(uN(!}8~ZLTK=o1pi3|oa z>@E${PX((F^+SJqi9tlD9awXq9GDeD(#8xox(L@Njj&VblNH$l;drl20iQm^lLpZb zaU?_i7Dj|aiz|Jb8etNJ=P3`g7ySVYhE&Dkh)f8I5e6(!Ag|gU0pCfa-@%_uECeGD z9G*$TwWhJwgu(LCo)`8^1|s6v9_R<>+@;}-ad+j|E}{)uN!<{ZUpX*!c-FZ#gF0A1 z5*XsSE8?!|as8aVVDfwz!h_0)JayrR&qFwQW7ov4f4XPh&bijjJ2+RttaI6na~Wzv z2i8pNLd0UdVVPiZ(Qsf&9CyuF)`47^Y<+oKB1~ z;ifWQ_I_2|su46qSrS1b^4TzdhA1_Y96}j-vJxG7sP1gG1-v0q zJd8Kojz|XK0iaEUN4AC`-{(}Inj$-jL3Z#Yn*(zH6;wo~G5YiVfXY15MwDpBW}!XG zHR}Pk;j?g%|DFnx3@t^16mE~LZk}2*8Mt-z=G6!vS$pewd`M9sOP8Y)Ndt-f7mypQ z^-OZZwXj|3H8k)CT1#VG!k%+u%fyy=;dT}2Z0wH4xnFAVAdeQUM9PeUY z=mA=_NDU!NHWEWLt7J+xR#QdNSy@brTC(3LTG=RUZ|Fy@7-_N6Zf2N`SOEEBh0S7W zE=O&Tow=o%H7W7*Y>nE}Bt!NURQs9E7qY3}h$I#r6o&MdzzKk~`KyrqNpm3;B0USs ztK;QcQdp^g&stGV{$!NoJNfJTTM?E3F~< z0z>?S)Pw=jHxL<+rX--bC-onC0QOl#qjCTkUqSKhO!xPtRk2`z9RdSdpB%^H2MeA9 zPGwe3AU~Q;$*Wd2)OxF=Qy6}dZwMNwm0*rQwm;{fQV_tM_xAW@06yCE3?p|-dsuE8 z$buD!NdzYte33+oy+Q%WQY4&KF!-|HCv{&)^82MO`UuJnA0?(|l!z#U-5;q70}^fJ zq`ueRFI`0p0tUuN5oBXcB=s1ybb~S(&-Gk|G?{ftcdk$Brab?QQVh&&Oy4~u$#h=R zq<|_auxD7DNxy+C{}F$(k0)w?a8@1!|M%_H33tx8HBq@bTDfVwX4ba+zHRy4zOj}$ zJ2Cj{Y_DH}EV8w?z)mdRC1a640ElXL5jqGv~(fiR0m-*^*VUl2y^-+Nm{D zho^Qr*q@3^+ce!u74+dJ`#w#>*DtM5A@Tm zKe5K_o9{Nq?Az|?@7r5QUa@=Me=7Ndl0RHBRr%vpcUHYycQ-$_YTG@-{Z*~e&gaGr z#GkH+;h&{?&Rv$O4J}%njJP=bb4%%b371nlU&=Xh#`Y<+KnhS|);W%r2*05&16A-i zgfa?K;Ln+i59^#f08XEh@T8PXtI1$1rAXi)YP0$FXya3rrR+$kyKRlTOrVS2Bfsa#Z{El%%r@V*L>ipc` z2QWxsE&&WjGTUE8vhF+YBgDu)r;O|gziGDpBo>EW2PW+D7QrXL1cxvY&?yXxZG<`$ zP5(KdY8m=|;-*oT54=8!z#$B;6XXG<_6xA2BZTblM-|<3M|p(4I81{s%XrZFmi!SAl`S`d-i*y~4LESXxCBVq15R1@qI0 zpCmi&(NSUGEEx&~Jx#(;|E2zcVf2!+2*D+aKS8!p!vLsG2s(w8P$$4{poJ{^ztq7l zs9NY7!o>A~%3!8N`*wF6hTZ@lXttoffa3wkf+T-3Sa@3N>KWF*GoWT<@Z*7zlyPAn z?9PWs8F`vHJP4Ga)PL45RKWU>GE^`Z4xc>I-`$8AgW+|f;)$WGC&@5WV#g;7H8gSZ z(BL3bD+?q3Kn){ug2iY9>9j^8rhkTlmY0!g6YRSKA{1LOvVYW4RI_4%I8cTve- z;!pm2g@A#bWzMX1NzA%rZ2P>67|y#kFs_FgW_j(@x(^I9!iMpd8*LM9 zlWjAOrFYkZj)B0!fBW!^z2eS_gvD{gK4G7fCdFv!k!b$W8OyPR#g!;s{$}m#wQtr> zxWW}ZDVv-T*Y z7q{#;?O`b_Mk*plB5feTo}FrmHtvWn-8tiFgLWoS03pw-&rR}kMdgulR8+KT%J*ac z9sl&scl+*++;7;G$S=ON_vYTjk}|b7k;_v#Q~G(1uRF>kEIG#iQm-#4#N?Kig`a)% z$m>TUo2Smk%htq8*3On}ij{17P|}pDhKx`nXQv9L%u|=AbEfstb$g@2z6bgHAG*21 zs!y$4N$tY|Zpq5eKXXHTGHYKRvoD7>!02AECJj^als}rke#Ww4&f@yJc_T7>$@Im) zTF|zF``L=TUB&vJZ7STAr~fd|h`6Q}z*b3HlWfPHS!nG>{FBJ5yevo;E)$doK%^

Harness²
- Recursive Agent Harnessing for an Open World + Tune your coding agent's harness on your own tasks

- Paper + Paper Website - Benchmark guides License

-

- - Install, set up, and run JobBench: uv pip install -e '.[render]', harness2-setup jb, then harness2 --bench jb --k 1 --mode parallel. Click for prerequisites and commands. - -

- -A **harness** is the system around a model that enables it to act as an agent: -prompts, tools, memory, skills, orchestration, and more. **Harnessing** is the -process of optimizing that system. +A coding agent is a model plus a **harness**: the instructions, skills, subagents, helper +scripts and hooks it runs with. `harness2` improves the harness of the agent you already use +(Claude Code's `CLAUDE.md` and `.claude/`, or Codex's `AGENTS.md` and `.agents/`) by running +it on tasks from your own project, studying what happened, and refining the harness from +that evidence. The model stays the same. Your files change only when you apply a result you +have reviewed. -Harness² is a plug-and-play framework for **recursive agent harnessing** in an -open world, where agents encounter tasks unseen during harness design. It -connects two levels of recursion while keeping model weights fixed: +It is a general-purpose command-line tool built on the method of +[Harness²: Recursive Agent Harnessing for an Open World](https://arxiv.org/abs/2610.14224). +This repository is an independent fork of +[google-research/harness2](https://github.com/google-research/harness2); the paper's +benchmark setup lives on as a [plugin](plugins/harness2-benchmarks/). -* **Task-level recursion** refines the execution harness for the current task - through a contrastive **Propose–Probe–Compose** step: propose edits, probe - their effects on the same task, and compose a refined harness from behavioral - contrasts between executions, without ground-truth feedback. -* **Harnessing-level recursion** updates a persistent *improvement harness* - with adopted harnesses and evidence-supported lessons from each task to guide - future refinement. - -

- - Harness² pipeline. Task-level recursion: an agent executor runs the initial harness, a harness editor proposes invariant and domain edits, candidate probes execute them, and a harness composer combines them into the final harness. Harnessing-level recursion: an experience reflector reflects on the step and updates the improvement harness used for the next task. - -

- -Across three professional benchmarks and four model–harness configurations, a -single Harness² step improves GLM-5 and Gemini 3.5 Flash over their base -harnesses, with gains of up to 11.5 points on WorkBuddy-Bench, 4.5 points on -JobBench, and 11.8 percentage points on LAB's partial pass rate. Increasing the -task-level recursion budget brings further gains. +- [How it works](#how-it-works) +- [Install](#install) +- [Quick start](#quick-start) +- [Commands](#commands) +- [Configuration](#configuration) +- [Other agents](#other-agents) +- [The paper's benchmarks](#the-papers-benchmarks) +- [Documentation](#documentation) -The release supports **parallel and sequential recursion**, **OpenCode and -Codex executors**, and **JobBench, LAB, and WorkBuddy-Bench**. +## How it works -## Video overview +For every task you give it, `harness2`: -https://github.com/user-attachments/assets/76b10548-b14b-4c8f-9aa5-a93662df4d56 +1. runs your agent with its current harness and records what it does; +2. has an improver agent propose harness edits from those trajectories, from two angles: + how the agent works, and what it needs to know about this kind of task; +3. runs the agent with each candidate harness, and composes a final one from what the + differences between the runs show; +4. keeps the adopted harness and a short lesson for the next task in the same area. -## Contents - -- [Video overview](#video-overview) -- [Harness components](#harness-components) -- [Install](#install) -- [Run a benchmark](#run-a-benchmark) -- [Experiments](#experiments) -- [Web access](#web-access) -- [Results and resume](#results-and-resume) -- [Citation](#citation) -- [Contributing](#contributing) -- [License](#license) -- [Disclaimer](#disclaimer) - -## Harness components - -Harness² exposes eight editable components around the model: **system prompt, -guardrails, memory, tool guidance, subagents, skills, scripts, and plugins**. -They control context construction, the tool interface, delegation, and behavior -inside the tool loop. +No step needs a grade or a reference answer, so any task you can describe can be used. +If you can also check the result, with a test command or a rubric, `harness2 judge` measures +how much the adopted harness helped. [How it works](docs/how-it-works.md) has the details.

- - Eight harness components grouped by runtime role: system prompt, guardrails, memory, and skills construct context; tool guidance and scripts define the tool interface; subagents orchestrate delegated work; plugins control middleware. + + Harness² pipeline: an agent runs the initial harness, an editor proposes general and domain-specific edits, candidate runs probe them, and a composer combines them into the final harness. A reflector then updates the experience carried to the next task.

## Install -Use **Linux, Python 3.12, and [uv](https://docs.astral.sh/uv/getting-started/installation/)**. -From the repository root: +You need Python 3.12 or 3.13, git, and the agent CLI whose harness you want to tune: +[Claude Code](https://docs.claude.com/en/docs/claude-code) (`claude`) or +[Codex](https://github.com/openai/codex) (`codex`), installed and on your `PATH`. ```bash -uv venv --python 3.12 .venv -source .venv/bin/activate -uv pip install -e '.[render]' -harness2-demo --output runs/demo --k 2 +uv tool install git+https://github.com/3ae3ae/harness2 +# or: pipx install git+https://github.com/3ae3ae/harness2 ``` -The local demo runs both recursion modes with deterministic editors and real -Python executions. It requires no model credentials. +Runs call models through your provider's API, so set the key for the models you choose, +for example `ANTHROPIC_API_KEY` or `OPENAI_API_KEY`. Model calls cost money; the +[quick start](#quick-start) shows how to see the cost of a run before you start it. + +## Quick start -For model-backed benchmarks, install **Node.js 20+, the Google Cloud CLI**, and -the [benchmark dependencies](docs/SETUP.md#requirements). LAB needs Pandoc and -Poppler; WorkBuddy-Bench needs Docker. Then configure your models: +Inside the project whose agent setup you want to improve: ```bash -cp .env.example .env -# Edit .env with your Vertex AI project and model IDs. -gcloud auth application-default login +harness2 init # pick the agent and models, creates .harness2/ +harness2 task new fix-login-redirect # then write the request in .harness2/tasks/fix-login-redirect/task.md +harness2 doctor # check CLIs, keys and tasks +harness2 run --dry-run # the task order and how many model calls a run makes +harness2 run # refine the harness on every task ``` -See the [setup guide](docs/SETUP.md) for model settings, authentication -alternatives, and the [Codex executor](docs/SETUP.md#codex-executor). - -## Run a benchmark - -The following example installs LAB and runs one task, then evaluates and reports -its result. Model-backed runs incur API costs; start with a small task selection. +A run works in temporary git worktrees, so your working tree is never touched. While it +runs, or after it stops: ```bash -harness2-setup lab -harness2-check --bench lab -harness2 --bench lab --k 1 --mode parallel \ - --only-domains antitrust-competition --max-tasks 1 -harness2-judge --bench lab --k 1 --mode parallel -harness2-report --bench lab --k 1 --mode parallel +harness2 status --watch # progress, spend, the harness picked per task +harness2 judge && harness2 report # grade the results of tasks that have a check or rubric +harness2 diff # what the adopted harness changes +harness2 apply # review it and write it into the project ``` -Sequential recursion runs the same commands with `--mode sequential`. `--k` is -the task-level recursion budget; see [Experiments](#experiments). Judging and -reporting must use the same benchmark, executor, subset, `--k`, `--mode`, and -seed as the refinement run. Add `--dry-run` to `harness2` to preview task -selection without model calls. +Run `harness2 run` again to resume an interrupted run; finished steps are reused. Write a +few varied tasks that look like the work you really do: see [Tasks](docs/tasks.md). -| Benchmark | Tasks | Install | Guide | -|---|---|---|---| -| JobBench | Professional work | `harness2-setup jb` | [JobBench](docs/BENCHMARKS.md#jobbench) | -| LAB | Legal work | `harness2-setup lab` | [LAB](docs/BENCHMARKS.md#lab) | -| WorkBuddy-Bench | Office, web, and code | `harness2-setup wb` | [WorkBuddy](docs/BENCHMARKS.md#workbuddy-bench) | +## Commands -Choose one benchmark per command. WorkBuddy also takes `--subset office`, -`--subset web`, or `--subset code` on every command after setup. The -[benchmark guides](docs/BENCHMARKS.md) cover each one's data, domains, and -grading. +| Command | What it does | +|---|---| +| `harness2 init` | Set up `.harness2/` in a project, with a short interactive wizard | +| `harness2 doctor` | Check the agent CLIs, provider keys, settings and tasks; `--live` also makes one tiny model call per model | +| `harness2 task new / ls / import` | Create, list and import tasks | +| `harness2 run` | Refine the harness on the tasks; `--dry-run` previews, `--budget-usd` and `--max-calls` cap spending, `--selector human` lets you pick the final harness yourself | +| `harness2 status` | A run's progress; `--watch` keeps it updated | +| `harness2 judge` | Grade the result each task adopted | +| `harness2 report` | Summarise grades, per run | +| `harness2 runs ls / show` | List runs and inspect one | +| `harness2 diff` | Show how an adopted harness differs from the starting one | +| `harness2 apply` | Write an adopted harness into the project, after showing the changes; `-i` to accept file by file | +| `harness2 config show / get / set` | Read and change settings; `show --origin` says where each value comes from | +| `harness2 chat` | Talk to your agent CLI about tasks, runs and results | +| `harness2 demo` | A local run with scripted editors and no model calls | + +Every command takes `--help`. See the [command reference](docs/cli.md) for all flags, exit +codes and the layout of a run directory. + +## Configuration + +Settings come from `~/.config/harness2/config.toml`, the project's +`.harness2/config.toml`, `HARNESS2_*` environment variables and flags, in that order of +precedence. Models are written `provider/model`; any OpenAI-compatible or +Anthropic-compatible endpoint, Vertex AI and Bedrock can be used: + +```toml +[executor] +name = "claude" +model = "anthropic/claude-sonnet-5-5" + +[improver] +channel = "claude" +model = "anthropic/claude-opus-5-5" + +[providers.local] # e.g. a vLLM or LM Studio server +type = "openai" +base_url = "http://localhost:8000/v1" +api_key_env = "LOCAL_API_KEY" +``` -## Experiments +[Configuration](docs/configuration.md) lists every setting. -Scale task-level recursion in two ways: **parallel recursion** explores multiple -candidate harnesses and composes refinements from their execution evidence; -**sequential recursion** carries the refined harness and accumulated evidence -through successive Propose–Probe–Compose steps. +## Other agents -

- - Two ways to scale task-level recursion. Parallel: probe k general and k domain-specific candidate harnesses, then produce k compositions from the shared evidence. Sequential: carry the refined harness and accumulated evidence through k Propose–Probe–Compose steps. - -

-

Parallel recursion explores wider; sequential recursion refines deeper.

+To tune an agent other than Claude Code or Codex, use the `command` executor: give it the +command that runs one task and the files that make up its harness. -| Option | Behavior | -|---|---| -| `--k 1` | One Propose–Probe–Compose step | -| `--k K --mode parallel` | Probe `K` general and `K` domain-specific candidates, then produce `K` compositions | -| `--k K --mode sequential` | Refine the harness through `K` successive steps | -| `--substrate cx` | Use Codex instead of the default OpenCode executor | - -After each task, harnessing-level recursion retains the adopted harness and -evidence-supported lessons to guide future refinement. See the -[experiment guide](docs/EXPERIMENTS.md) for task selection, concurrency, -reproducibility, and baseline comparisons. - -## Web access +```toml +[executor] +name = "command" +command = "my-agent --task {prompt_file} --workdir {workspace}" +editable = ["prompts/*.md", "agent.yaml"] +``` -Executor network access is unrestricted, and Codex runs with its sandbox -disabled. Public benchmark rubrics may be accessible online. Before evaluation, -configure [network restrictions](docs/EXPERIMENTS.md#web-access) to prevent -access to benchmark answers and grading criteria. +## The paper's benchmarks -## Results and resume +JobBench, LAB and WorkBuddy-Bench, the benchmarks of the paper, are packaged as the +[harness2-benchmarks plugin](plugins/harness2-benchmarks/). Install it next to `harness2` +and they become suites: `harness2 run --suite jb`. -Results are saved under `runs/`, organized by benchmark and subset where -applicable. They include task summaries, deliverables, adopted harnesses, -reflection notes, and evaluation results. +## Documentation -Repeat the same command to resume. Use a new `HARNESS2_RUNS_DIR` when changing -models or task selection. See [results and resume](docs/EXPERIMENTS.md#results-and-resume) -for details; each command also supports `--help`. +- [How it works](docs/how-it-works.md): the method, what a run costs, what it stores +- [Tasks](docs/tasks.md): writing tasks, workspaces, grading +- [Configuration](docs/configuration.md): settings, models and providers +- [Command reference](docs/cli.md): every command and flag +- [Contributing](CONTRIBUTING.md) ## Citation @@ -192,18 +175,8 @@ for details; each command also supports `--help`. } ``` -## Contributing - -See [CONTRIBUTING.md](CONTRIBUTING.md). - ## License -Harness² is licensed under [Apache 2.0](LICENSE). Third-party code under -[`third_party/`](third_party/README.md), benchmarks, and datasets retain their -upstream licenses; see -[third-party notices](dependencies/README.md). - -## Disclaimer - -This is not an officially supported Google product. This project is not eligible -for the [Google Open Source Software Vulnerability Rewards Program](https://bughunters.google.com/open-source-security). +Apache 2.0; see [LICENSE](LICENSE). The method and the original implementation are by the +paper's authors at Google; this fork is not a Google product and is not maintained by +Google. Third-party code in the benchmarks plugin keeps its upstream licenses. diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 00000000..425cb9f9 --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,240 @@ +# Command reference + +``` +harness2 [global options] [options] +``` + +Every command takes `--help`. `python -m harness2` is the same as `harness2`. + +- [Global options](#global-options) +- [Commands](#commands) +- [Exit codes](#exit-codes) +- [A run on disk](#a-run-on-disk) +- [Shell completion](#shell-completion) + +## Global options + +These go before the command: `harness2 -C ../api --json status`. + +| Option | Meaning | +|---|---| +| `-C DIR`, `--project DIR` | Act as if started in `DIR`. The project is the nearest directory upward containing `.harness2/`. | +| `--profile NAME` | Apply the `[profiles.NAME]` settings. | +| `-c KEY=VALUE`, `--config KEY=VALUE` | Override one setting for this command; repeatable. See [Configuration](configuration.md). | +| `-q`, `--quiet` | Print results only, no progress. | +| `-v`, `--verbose` | Print debug details. | +| `--json` | Print results as JSON on stdout; progress still goes to stderr. | +| `--version` | Print the version. | + +## Commands + +### `init` + +Set up harness2 in a project: asks which agent runs your tasks (`claude`, `codex` or a +custom `command`), which models to use, and where the harness lives; writes +`.harness2/config.toml`, `.harness2/.gitignore` and `.harness2/tasks/`. + +| Option | Meaning | +|---|---| +| `--yes`, `-y` | Take every default without asking. | +| `--executor {claude,codex,command}` | Skip the executor question. | +| `--force` | Overwrite an existing config file. | + +### `doctor` + +Check that a run has what it needs: the suite and its tasks, each role's model and +provider, the agent CLIs, and provider keys. Makes no model call unless `--live`. + +| Option | Meaning | +|---|---| +| `--live` | Also call each configured model once, with a tiny request. | + +### `task new | ls | import` + +```bash +harness2 task new ID [--files] [--domain D] [--check CMD] +harness2 task ls +harness2 task import FILE [--force] +``` + +`new` creates `.harness2/tasks/ID/` with a `task.md` to fill in and a `task.toml`. `--files` +makes a files task (with a `files/` directory) instead of a project task. `import` reads a +`.jsonl` or `.csv` file; `--force` overwrites existing tasks. See [Tasks](tasks.md). + +### `run` + +Refine the harness on the tasks. Running the same command again resumes the run and reuses +every finished step. + +| Option | Meaning | +|---|---| +| `--name NAME` | Run name. Default: `-k`, plus `-s` with a seed. | +| `--k K` | Recursion budget per task (default 1). | +| `--mode MODES` | `parallel` (default), `sequential`, or `parallel,sequential`. | +| `--executor NAME` | Override `executor.name`. | +| `--suite NAME` | Task suite: `tasks` (default) or one installed by a plugin. | +| `--split {all,train,test}` | Run one split only. | +| `--only-domains D,...` | Run only these domains. | +| `--only-tasks ID,...` | Run only these tasks. | +| `--max-tasks N` | Run at most N tasks per domain. | +| `--seed N` | Shuffle the task order within each domain. | +| `--workers N` | Domains to run at the same time. | +| `--budget-usd USD` | Stop once the run has spent this much. | +| `--max-calls N` | Stop after N model calls. | +| `--selector human` | Pick the final harness yourself when `k` > 1. | +| `--steer TEXT\|@FILE` | Guidance added to every improver session. | +| `--dry-run` | Show the tasks and the number of model calls; run nothing. | +| `--split-file FILE` | Task partition file, for benchmark suites. | + +A run keeps the settings it started with. Resuming it with a different executor, model, +improver or task list is refused, listing what changed; use a new `--name` for a new +experiment. The limits apply to the whole run, across resumes. + +### `status [NAME]` + +Each task's state, the harness picked for it, rollouts done, and the run's spend. Works +while the run is going, from another terminal. NAME defaults to the most recent run. + +| Option | Meaning | +|---|---| +| `--watch [SECONDS]` | Redraw every SECONDS (default 5) until Ctrl-C. | + +### `judge [NAME]` + +Grade the result each task adopted, with the task's `check` command or `rubric.md`. Tasks +with neither are skipped. Grades are written to the run and reused until the check, the +rubric or the judge model changes. + +| Option | Meaning | +|---|---| +| `--bases` | Grade the runs of the starting harness instead, for comparison. | + +### `report [NAME ...]` + +Average the grades per recursion mode. Several names compare runs. + +### `runs ls` and `runs show NAME` + +List runs with their progress and spend, or show one run's settings, totals and tasks. + +### `diff [NAME]` + +Show how the harness a run adopted for a task differs from the harness the run started +from. + +| Option | Meaning | +|---|---| +| `--task ID` | The task whose adopted harness to show. Default: the last task the run finished. | +| `--mode {parallel,sequential}` | The recursion mode to take it from (default parallel). | +| `--stat` | List the changed files only. | + +### `apply [NAME]` + +Write an adopted harness into the project. Only what the run changed relative to the +harness it started from is applied, so edits you made since are kept; a file changed both +by the run and in the project is skipped unless `--force`. The changes are listed first. + +| Option | Meaning | +|---|---| +| `--task ID`, `--mode` | As for `diff`. | +| `-i`, `--interactive` | Show each file's diff and accept or skip it. | +| `-y`, `--yes` | Apply without asking. | +| `--branch NAME` | Create and switch to a git branch first. | +| `--dry-run` | Show the changes, write nothing. | +| `--force` | Also overwrite files changed in the project since the run started, and write over uncommitted changes. | + +Only the executor's harness files are written. For Claude Code those are `CLAUDE.md`, +`.claude/skills/`, `.claude/agents/`, `.claude/scripts/` and the `hooks` key of +`.claude/settings.json` (its other keys are kept). + +### `config show | get | set | path` + +```bash +harness2 config show [--origin] # every resolved setting, and where it came from +harness2 config get KEY +harness2 config set KEY VALUE [--global] +harness2 config path +``` + +`set` edits `.harness2/config.toml` (or the global file with `--global`) in place, keeping +comments. The value is read as a TOML literal when it parses: `run.k 2` is a number. + +### `chat` + +Start your own Claude Code or Codex in the project, with a guide to harness2 and the +project's tasks and runs added to its instructions. Ask it to help write tasks, a check or a +rubric, or to explain a run's results. + +| Option | Meaning | +|---|---| +| `--with {claude,codex}` | Which CLI to start. Default: the improver's channel, if installed. | + +### `setup SUITE [ARGS...]` + +Run a plugin suite's installer, for example `harness2 setup lab` for the +[benchmarks plugin](../plugins/harness2-benchmarks/). + +### `demo` + +Run the method on two toy tasks with scripted editors: no model calls, a few seconds. +`--output DIR` (default `runs/demo`), `--k K` (default 2), `--mode {parallel,sequential,both}`. + +### `bridge` + +Serve a configured model as an OpenAI Responses API endpoint through LiteLLM, so the Codex +executor or channel can use providers it cannot reach directly. Needs +`pip install 'harness2[bridge]'`. + +| Option | Meaning | +|---|---| +| `--model PROVIDER/MODEL` | The model to serve (required). | +| `--host HOST` | Interface to listen on (default `127.0.0.1`). | +| `--port PORT` | Port. Default: the one used last time, or a free one. | + +It prints the `[providers.bridge]` table to add to your config and the key to put in +`.harness2/.env`. + +### `completion {bash,zsh}` + +Print a shell completion script; see [below](#shell-completion). + +## Exit codes + +| Code | Meaning | +|---|---| +| 0 | Success. | +| 1 | An error: bad settings, a missing CLI or key, a failed check, a run that could not continue. | +| 2 | Invalid command-line usage. | +| 3 | `run` finished, but some tasks failed. | +| 4 | `run` stopped at `--budget-usd` or `--max-calls`. Raise the limit and rerun to resume. | +| 130 | Interrupted with Ctrl-C. Rerun to resume. | + +## A run on disk + +``` +.harness2/runs// + run.json settings, task list and schedule the run started with + events.jsonl one line per agent run, model session, selection and task, with costs + base_harness/ the project's harness when the run started + rollouts// one agent run: prompt.md, stream.jsonl (trajectory), final.md, + deliverable.patch or output/, meta.json, judge.json + adapt/// + summary.json what happened for the task + harnesses/ every candidate harness + selection/ the final pick and why + reflection/ the lesson recorded for later tasks + adapt/_audit/ every improver session's mission and reply + library/ adopted harnesses and lessons per domain +``` + +Stored paths are relative, so a run directory can be moved or copied and still resumes. + +## Shell completion + +```bash +# zsh +harness2 completion zsh > "${fpath[1]}/_harness2" +# bash +mkdir -p ~/.local/share/bash-completion/completions +harness2 completion bash > ~/.local/share/bash-completion/completions/harness2 +``` diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 00000000..d576c19f --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,214 @@ +# Configuration + +Harness² reads its settings from TOML files, environment variables and command-line flags. +Every setting has a default, so a project only needs the few keys that differ. + +- [Where settings come from](#where-settings-come-from) +- [Models and providers](#models-and-providers) +- [Reference](#reference) +- [Profiles](#profiles) +- [Secrets](#secrets) + +## Where settings come from + +Later sources override earlier ones: + +| Source | Location | +|---|---| +| Defaults | built in | +| Global file | `~/.config/harness2/config.toml` (or `$XDG_CONFIG_HOME/harness2/config.toml`) | +| Project file | `/.harness2/config.toml` | +| Profile | a `[profiles.]` table, selected with `--profile ` or `HARNESS2_PROFILE` | +| Environment | `HARNESS2_
__`, for example `HARNESS2_EXECUTOR__MODEL` | +| Command line | `-c key=value` (repeatable) and command flags such as `--k 2` | + +The project is the nearest directory, upward from the current one, that contains +`.harness2/`. `-C ` starts the search somewhere else. `harness2 init` creates +`.harness2/` with a starter `config.toml`. + +To see the resolved settings and where each one comes from: + +```bash +harness2 config show --origin +harness2 config get executor.model +harness2 config set run.workers 2 # edits .harness2/config.toml, keeps comments +harness2 config set --global providers.local.base_url http://localhost:8000/v1 +``` + +Values given in the environment or with `-c` are read as TOML literals, so `-c run.k=2` is +a number and `-c 'executor.editable=["prompts/*.md"]'` is a list. Anything that does not +parse as TOML is taken as a plain string: `-c executor.model=openai/gpt-6.1-sol`. + +## Models and providers + +Every model is written `provider/model`. The provider part names a `[providers.]` +table, which says what kind of endpoint it is and where its key comes from. Two providers +are built in: + +```toml +[providers.anthropic] +type = "anthropic" +api_key_env = "ANTHROPIC_API_KEY" + +[providers.openai] +type = "openai" +base_url = "https://api.openai.com/v1" +api_key_env = "OPENAI_API_KEY" +``` + +So `anthropic/claude-sonnet-5-5` and `openai/gpt-6.1-sol` work as soon as the matching key +is set. Add your own provider for anything else: + +```toml +# An OpenAI-compatible server: vLLM, LM Studio, Ollama, a company gateway ... +[providers.local] +type = "openai" +base_url = "http://localhost:8000/v1" +api_key_env = "LOCAL_API_KEY" + +# An Anthropic-compatible gateway +[providers.gateway] +type = "anthropic" +base_url = "https://llm-gateway.example.com" +api_key_env = "GATEWAY_KEY" + +# Google Vertex AI (uses Application Default Credentials) +[providers.vertex] +type = "vertex" +project = "my-gcp-project" +region = "global" + +# Amazon Bedrock (uses the usual AWS credentials) +[providers.bedrock] +type = "bedrock" +region = "us-east-1" +``` + +Each agent CLI speaks to some provider types and not others: + +| Provider type | Claude Code (`claude`) | Codex (`codex`) | OpenCode (`opencode`) | +|---|---|---|---| +| `anthropic` | yes | via `harness2 bridge` | yes | +| `openai` | no | yes (Responses API) | yes (Chat Completions) | +| `vertex` | yes | via `harness2 bridge` | yes | +| `bedrock` | yes | via `harness2 bridge` | no | + +`harness2 doctor` reports a model whose provider its CLI cannot use. To run Codex on a model +it cannot reach directly, serve that model as a Responses API endpoint with the bridge +(`pip install 'harness2[bridge]'`), then point an `openai`-type provider at it: + +```bash +harness2 bridge --model anthropic/claude-sonnet-5-5 +``` + +The agent CLIs are the ones installed on your system (`claude`, `codex`, `opencode` on +`PATH`). Set `executor.bin` or `improver.bin` to use another binary or a full command. + +**Logins are not used.** Every run gives the agent CLI a fresh, empty configuration +directory (`CLAUDE_CONFIG_DIR` for Claude Code, `CODEX_HOME` for Codex), so your personal +instructions, skills, plugins and settings never leak into the harness being measured. +That also means the CLI does not see your interactive login: models are reached with the +provider's API key. `harness2 chat` is the exception: it starts your own CLI as you normally +use it. + +## Reference + +### `[executor]`: the agent that runs the tasks + +| Key | Default | Meaning | +|---|---|---| +| `name` | `"claude"` | `claude`, `codex` or `command`. Plugin suites accept their own executors (the benchmark plugin uses `opencode` and `codex`). | +| `model` | `""` | `provider/model` the executor runs on. Required for `claude` and `codex`. | +| `bin` | `""` | Binary or command to start the CLI. Empty means the executor name on `PATH`. | +| `effort` | `"high"` | Reasoning effort passed to the CLI. | +| `timeout` | `1800` | Seconds one rollout may take. A task's `timeout` overrides it. | +| `command` | `""` | `command` executor only: the shell command that runs one task. `{workspace}`, `{prompt_file}` and `{output}` are replaced with quoted paths. | +| `editable` | `[]` | `command` executor only: glob patterns, relative to `paths.harness`, of the files the improver may edit. | + +### `[improver]`, `[selector]`, `[reflector]`, `[judge]`: the agents that work on the harness + +The improver proposes and composes harness edits. The selector picks the final harness when +several candidates compete (only when `k` > 1). The reflector writes the lesson that carries +over to later tasks. The judge grades tasks that have a `rubric.md`. + +`[improver]` holds the defaults for all four. `[selector]` and `[judge]` override the +improver; `[reflector]` overrides the selector, or the improver when the selector is a +person. + +| Key | Default | Meaning | +|---|---|---| +| `channel` | `"claude"` | Which CLI runs the session: `claude`, `codex` or `opencode`. `[selector]` also accepts `human`. | +| `model` | `""` | `provider/model`. Required for the improver. | +| `bin` | `""` | Binary or command for the channel's CLI. | +| `effort` | `"high"` | Reasoning effort. | +| `timeout` | `3600` | Seconds one session may take. | +| `steps` | `120` | Step limit for OpenCode sessions. | +| `steer` | `""` | Improver only: guidance added to every improver session, as text or `@path/to/file`. Describe what you want from the agent's work, not how it will be measured. | + +### `[paths]` + +| Key | Default | Meaning | +|---|---|---| +| `harness` | `"."` | Where the harness files live, relative to the project root. | +| `tasks` | `".harness2/tasks"` | The task directory. | +| `runs` | `".harness2/runs"` | Where runs are stored. | + +### `[run]` + +| Key | Default | Meaning | +|---|---|---| +| `k` | `1` | Task-level recursion budget. See [how it works](how-it-works.md#scaling-with-k). | +| `modes` | `"parallel"` | `parallel`, `sequential`, or both separated by a comma. | +| `seed` | unset | Shuffles the task order within each domain. Unset keeps sorted order. | +| `workers` | `1` | Domains run at the same time. | +| `phase_width` | `2` | Rollouts and improver sessions run at the same time within one task. | +| `proposer_concurrency` | `2` | Model sessions running at the same time across the whole run. | +| `transport_attempts` | `2` | Attempts per improver session before it counts as failed. | +| `obs_trunc` | `50000` | Characters of each tool output kept when trajectories are shown to the improver. | +| `budget_usd` | `0` | Stop the run once it has spent this many dollars. `0` means no limit. | +| `max_calls` | `0` | Stop the run after this many model calls (rollouts plus sessions). `0` means no limit. | + +Spending is counted from the costs the CLIs report. Claude Code and OpenCode report dollars; +Codex reports tokens only, so `budget_usd` does not see Codex spending. Use `max_calls` to +bound a Codex run. + +### `[suite]` + +| Key | Default | Meaning | +|---|---|---| +| `name` | `"tasks"` | Which suite supplies the tasks: `tasks` (your own) or a suite installed by a plugin. | +| `work` | `""` | One sentence describing the work, shown to the improver. For example: "maintain a Django web service and its test suite". | + +## Profiles + +A profile is a named set of overrides you switch on with `--profile`. Put profiles in the +global file to share them between projects: + +```toml +[profiles.cheap.executor] +model = "anthropic/claude-haiku-5-5" + +[profiles.cheap.improver] +model = "anthropic/claude-sonnet-5-5" + +[profiles.cheap.run] +max_calls = 200 +``` + +```bash +harness2 run --profile cheap +``` + +## Secrets + +Keys are never written into config files. A provider names the environment variable that +holds its key (`api_key_env`), and a config file that contains a literal `api_key` is +rejected. Put project-specific keys in `.harness2/.env`, which `harness2 init` adds to +`.harness2/.gitignore`: + +```bash +# .harness2/.env +ANTHROPIC_API_KEY=sk-ant-... +``` + +Variables already set in your shell win over `.harness2/.env`. diff --git a/docs/how-it-works.md b/docs/how-it-works.md new file mode 100644 index 00000000..106e7680 --- /dev/null +++ b/docs/how-it-works.md @@ -0,0 +1,108 @@ +# How it works + +Harness² improves an agent without touching the model. It changes the **harness**: the +files the agent reads and runs around every task. For Claude Code those are `CLAUDE.md`, +skills, subagents, helper scripts and hooks; for Codex, `AGENTS.md`, skills, scripts and +hooks. The method is described in the paper +[Harness²: Recursive Agent Harnessing for an Open World](https://arxiv.org/abs/2610.14224). +This page explains what a run does, what it costs, and what you get back. + +- [The harness components](#the-harness-components) +- [One task: propose, probe, compose](#one-task-propose-probe-compose) +- [Scaling with k](#scaling-with-k) +- [Picking the final harness](#picking-the-final-harness) +- [Carrying lessons forward](#carrying-lessons-forward) +- [What a run gives you](#what-a-run-gives-you) + +## The harness components + +Each component has a different cost and reach, and the improver is told to use the cheapest +one that fixes what it saw: + +| Component | Claude Code | Codex | In context | +|---|---|---|---| +| Always-on instructions | `CLAUDE.md` | `AGENTS.md` | every turn, so every word costs | +| Skills | `.claude/skills//SKILL.md` | `.agents/skills//SKILL.md` | only when invoked | +| Subagents | `.claude/agents/.md` | — | in their own context | +| Helper scripts | `.claude/scripts/*.py` | `.agents/scripts/*.py` | only when run | +| Hooks | `hooks` in `.claude/settings.json` | `.codex/hooks.json` | inside the tool loop, cannot be skipped | + +With the `command` executor the harness is whatever files `executor.editable` matches. + +Harness² edits only these files. Anything else in `.claude/` or in a skill directory, such as +reference documents, is left alone. + +## One task: propose, probe, compose + +For each task, a run: + +1. **Executes** the agent with the current harness, `k` times, to see how it behaves today. +2. **Proposes** edits. Two improvers read those trajectories and results independently. One + works on *how the agent works*: procedure, tool use, checking its own result. The other + works on *what a practitioner knows* about this kind of task. Each writes a candidate + harness. +3. **Probes** every candidate by running the agent with it on the same task. +4. **Composes** a final harness from the contrast between all those runs: which edits changed + behaviour, and how. The final harness is run too. + +No step looks at a grade. The improvers see trajectories and results, and decide from the +differences between runs what helped. This is what lets Harness² work on tasks that have no +test at all. + +If the evidence does not support a change, an improver can keep the harness as it is. An +edit that breaks the harness's rules (a skill without a description, a script that is +referenced but missing, malformed hooks) is sent back once with the problems listed, and +otherwise dropped. + +## Scaling with k + +`k` (`--k`, default 1) is how much effort each task gets. There are two ways to spend it: + +- **parallel** (`--mode parallel`, the default): `k` "how it works" and `k` "what it knows" + candidates are proposed from the same evidence, and `k` composed finals are built from all + of them. Wider exploration. +- **sequential** (`--mode sequential`): `k` rounds of propose–probe–compose, each starting + from the previous round's result and seeing all earlier evidence. Deeper refinement. + +Per task and mode, a run makes **4k agent runs** (including the `k` runs of the current +harness, which both modes share) and **3k improver sessions**, plus one reflector session +that writes the task's lesson and, when `k` is more than 1, one selector session that picks +the final. `harness2 run --dry-run` prints the exact counts for your tasks before anything is +spent, and `--budget-usd` / `--max-calls` stop a run at a limit. + +## Picking the final harness + +With `k` = 1 there is one final. With more, a **selector** compares the finals' results +blind: it sees the task and each result under an anonymous id, never which harness produced +it or how that harness was built, and picks the best. That pick is the task's **adopted** +harness. + +You can make the pick yourself with `harness2 run --selector human`: the run stops at each +selection and shows you the anonymous candidates in the terminal. + +## Carrying lessons forward + +After a task adopts a harness, a **reflector** looks at which edits made a difference and +writes a short lesson. The adopted harness and its lesson go into a **library** for the +task's domain. Later tasks in the same domain show the library to their improvers, so useful +components get reused and verified lessons build up. This is the second level of recursion: +the harness that improves harnesses gets better as tasks go by. + +Tasks in a domain run one after another so the library can grow; separate domains run in +parallel (`--workers`). Each run has its own library, so experiments never mix. + +## What a run gives you + +Everything is stored under `.harness2/runs//`: + +- `run.json`: the settings and task list the run was started with. +- `events.jsonl`: one line per agent run, session and decision, with costs. +- `adapt/`: per task, every candidate harness, the improvers' notes, the selection and the + lesson. +- `rollouts/`: per agent run, the trajectory, the result and the final message. +- `library/`: the adopted harnesses and lessons per domain. + +Use `harness2 status` while it runs, `harness2 judge` and `harness2 report` to grade the +results, `harness2 diff` to read what changed in the harness, and `harness2 apply` to write an +adopted harness into your project. Running the same `harness2 run` command again resumes +where it stopped, reusing every finished step. diff --git a/docs/tasks.md b/docs/tasks.md new file mode 100644 index 00000000..83904056 --- /dev/null +++ b/docs/tasks.md @@ -0,0 +1,120 @@ +# Tasks + +A task is one piece of work you want your agent to get better at: a bug to fix, a feature to +add, a report to write. Harness² runs the agent on each task, studies how the runs went, and +refines the agent's harness on that evidence. Good tasks are real work, phrased the way you +would actually ask, and varied enough that a better harness has to be better in general. + +- [Layout](#layout) +- [Workspaces](#workspaces) +- [Deliverables](#deliverables) +- [Grading](#grading) +- [Creating and importing tasks](#creating-and-importing-tasks) + +## Layout + +Each task is a directory under `.harness2/tasks/` (the `paths.tasks` setting). Its name is the +task id: lowercase letters, digits, `.`, `_` and `-`. + +``` +.harness2/tasks/ + fix-login-redirect/ + task.md the instruction the agent receives (required) + task.toml options (optional) + check/ files only the grader sees (optional) + rubric.md criteria for an LLM grader (optional) + summarize-incidents/ + task.md + task.toml + files/ input files copied into the workspace (optional) +``` + +`task.md` is given to the agent as-is. Write it as a request, not as a test: the agent, the +improver and the selector all read it, and none of them ever sees `check/` or `rubric.md`. + +`task.toml` keys, all optional: + +| Key | Default | Meaning | +|---|---|---| +| `domain` | `"default"` | Groups related tasks. Lessons and adopted harnesses carry over only between tasks of the same domain, and domains can run in parallel. | +| `split` | none | `"train"` or `"test"`, for `harness2 run --split`. A task without one is only in `all`. | +| `workspace` | `"files"` if `files/` exists, else `"project"` | Where the agent works. See [Workspaces](#workspaces). | +| `ref` | `"HEAD"` | Project workspaces only: the git revision the agent starts from. | +| `deliverable` | `"diff"` for project workspaces, `"files"` otherwise | What counts as the result. See [Deliverables](#deliverables). | +| `check` | none | A shell command that grades a result. See [Grading](#grading). | +| `timeout` | `executor.timeout` | Seconds one run of this task may take. | + +## Workspaces + +Every run gets a fresh workspace in a temporary directory outside your project, so runs +never touch your working tree and never see each other. + +**`project`**: a clean `git worktree` of your repository at `ref`. The project must be a git +repository, and uncommitted changes are not included: commit what the task should start +from. The `.harness2/` directory and the project's own harness files are removed from the +copy, and the harness being tried is installed in their place. + +**`files`**: an empty directory seeded with the contents of the task's `files/`, plus an +empty `output/` directory. The agent is asked to save its deliverables in `output/`. Use +this for work that is not a change to the repository: documents, analyses, data. + +## Deliverables + +| `deliverable` | The result is | Counts as delivered when | +|---|---|---| +| `diff` | the change the agent made to the repository, as a patch | the patch is not empty | +| `files` | the files in `output/` | `output/` holds at least one file | +| `message` | the agent's final message | the message is not empty | + +The agent's final message is always kept alongside the result. Changes to the harness files +themselves are left out of the patch. + +## Grading + +Refining the harness does not need grades: candidates are compared by an independent +selector that reads their results, never a score. Grades are for you, to measure what a run +achieved. `harness2 judge` grades the result each task finally adopted, and `harness2 report` +summarises the grades. + +**`check`**: a shell command run in a fresh workspace that holds the result: for project +tasks, a new worktree at `ref` with the patch applied; for files tasks, a copy of `files/` +with the result in `output/`. The contents of `check/` are copied in first, so hidden tests +can live there. The command passes when it exits with status 0. For partial credit, print a +JSON object as the last line of output: + +```json +{"n_passed": 7, "n_criteria": 9} +``` + +`HARNESS2_TASK_DIR` and `HARNESS2_RUN_DIR` are set for the command. + +```toml +# task.toml +check = "cp check/test_redirect.py tests/ && python -m pytest -q tests/test_redirect.py" +``` + +**`rubric.md`**: criteria an LLM judge applies to the result, one checkable criterion per +line works best. The judge uses the `[judge]` settings. It reports how many criteria the +result meets. + +A task with neither is still refined; it just has no grade. + +## Creating and importing tasks + +```bash +harness2 task new fix-login-redirect # a project task +harness2 task new summarize-incidents --files # a files task, with files/ +harness2 task new add-pagination --domain api --check "python -m pytest -q" +harness2 task ls +``` + +To bring in many tasks at once, import a JSON Lines or CSV file. `id` and `prompt` are +required; the other fields are the `task.toml` keys above. + +```bash +harness2 task import tasks.jsonl +``` + +```json +{"id": "fix-login-redirect", "prompt": "Users who log in from /settings land on / ...", "domain": "auth", "check": "python -m pytest -q tests/test_login.py"} +``` diff --git a/harness2/cli/main.py b/harness2/cli/main.py index 93dccb3e..cce81083 100644 --- a/harness2/cli/main.py +++ b/harness2/cli/main.py @@ -24,22 +24,22 @@ from ._common import Context COMMANDS = ( - "run", + "init", "doctor", - "setup", + "task", + "run", + "status", "judge", "report", "runs", "diff", - "task", + "apply", "config", + "chat", + "setup", "demo", "bridge", "completion", - "init", - "status", - "chat", - "apply", ) ALIASES = {"harness2-" + name: name for name in ("judge", "report", "setup", "demo", "bridge")} ALIASES["harness2-check"] = "doctor" diff --git a/harness2/cli/task.py b/harness2/cli/task.py index cb47089a..593395bd 100644 --- a/harness2/cli/task.py +++ b/harness2/cli/task.py @@ -49,7 +49,13 @@ def run(args, ctx) -> int: result = suite.new_task( tasks_dir, args.id, files=args.files, domain=args.domain, check=args.check ) - ctx.out({"path": result}, lambda data: str(data["path"])) + ctx.out( + {"path": result}, + lambda data: ( + f"Created {data['path']}. Write the request in its task.md" + + (" and put the input files in files/." if args.files else ".") + ), + ) elif args.action == "ls": ctx.out( suite.list_tasks(tasks_dir), diff --git a/plugins/harness2-benchmarks/README.md b/plugins/harness2-benchmarks/README.md new file mode 100644 index 00000000..b8532135 --- /dev/null +++ b/plugins/harness2-benchmarks/README.md @@ -0,0 +1,53 @@ +# harness2-benchmarks + +The three benchmarks of the paper +[Harness²: Recursive Agent Harnessing for an Open World](https://arxiv.org/abs/2610.14224), +packaged as suites for the `harness2` command: + +| Suite | Benchmark | Work | +|---|---|---| +| `jb` | [JobBench](https://github.com/Job-Bench/job-bench-eval) | professional deliverables across occupations | +| `lab` | [LAB](https://github.com/harveyai/harvey-labs) | legal work products | +| `wb-office`, `wb-web`, `wb-code` | [WorkBuddy-Bench](https://github.com/Tencent/workbuddy-bench) | office documents, web apps, code | + +They run on the OpenCode or Codex executor, use the pinned upstream revisions and +compatibility patches of the paper, and grade with each benchmark's own judge. + +## Install + +From a checkout of this repository, next to `harness2`: + +```bash +uv pip install -e . -e 'plugins/harness2-benchmarks[render]' +harness2 setup lab # or jb, wb-office, wb-web, wb-code +``` + +Setup downloads the upstream checkouts, task data and a pinned OpenCode build into +`~/.cache/harness2/benchmarks/` (set `HARNESS2_BENCH_DEPS` to put them elsewhere). +[Setup](docs/SETUP.md) covers system requirements, model credentials and the Codex +executor. + +## Run + +The benchmarks keep the paper's configuration: models and endpoints come from +`HARNESS2_*` environment variables (or `.harness2/.env`), as described in +[Setup](docs/SETUP.md#models-and-credentials). Choose the executor with +`-c executor.name=opencode` or `-c executor.name=codex`: + +```bash +harness2 -c executor.name=opencode -c suite.name=lab doctor +harness2 -c executor.name=opencode run --suite lab --k 1 --mode parallel \ + --only-domains antitrust-competition --max-tasks 1 +harness2 judge +harness2 report +``` + +[Benchmarks](docs/BENCHMARKS.md) describes each benchmark's data, domains and grading; +[Experiments](docs/EXPERIMENTS.md) covers task selection, concurrency, reproducibility and +network restrictions during evaluation. + +## Licenses + +The vendored runners and patches under `harness2_benchmarks/third_party/` keep their +upstream licenses; see [third-party notices](harness2_benchmarks/third_party/README.md). +Benchmark data is downloaded from its publishers under their terms. diff --git a/plugins/harness2-benchmarks/docs/BENCHMARKS.md b/plugins/harness2-benchmarks/docs/BENCHMARKS.md index 84845558..5f9d7aae 100644 --- a/plugins/harness2-benchmarks/docs/BENCHMARKS.md +++ b/plugins/harness2-benchmarks/docs/BENCHMARKS.md @@ -121,4 +121,7 @@ uses a fresh benchmark container. The adapter reads the verifier's `score.json`; the improver never receives verifier output. Missing verifier results remain unscored. Runs live under `.harness2/runs//`. -For Codex, start the bridge with `harness2-bridge --host 0.0.0.0` so that the containers can reach it. +For Codex, start the bridge with `--host 0.0.0.0` (for example +`harness2 bridge --model vertex/gemini-3.5-flash --host 0.0.0.0`) so that the containers can +reach it, and set `HARNESS2_CODEX_CONTAINER_URL` as described in +[Setup](SETUP.md#codex-executor). diff --git a/plugins/harness2-benchmarks/docs/SETUP.md b/plugins/harness2-benchmarks/docs/SETUP.md index 13144714..c11783f2 100644 --- a/plugins/harness2-benchmarks/docs/SETUP.md +++ b/plugins/harness2-benchmarks/docs/SETUP.md @@ -27,8 +27,16 @@ Keep the main Harness² environment active when running the commands below. ## Models and credentials -Copy `.env.example` to `.harness2/.env` if you have not already done so. Set your Vertex AI -project and model IDs, then authenticate: +Put your Vertex AI project and model IDs in `.harness2/.env` (or export them in the +shell), then authenticate: + +```bash +# .harness2/.env +GOOGLE_CLOUD_PROJECT=my-gcp-project +HARNESS2_SOLVER_MODEL=google-vertex/gemini-3.5-flash +HARNESS2_IMPROVER_MODEL=google-vertex/gemini-3.5-flash +HARNESS2_JUDGE_MODEL=gemini-3.5-flash +``` ```bash gcloud auth application-default login @@ -105,20 +113,28 @@ return a nonzero exit status when incomplete. ## Codex executor The Codex executor connects to a Responses API endpoint. For Gemini on Vertex -AI, the included `harness2-bridge` provides this interface through a local -LiteLLM server, using the same Application Default Credentials and -`GOOGLE_CLOUD_PROJECT`. +AI, `harness2 bridge` provides this interface through a local LiteLLM server, using +the same Application Default Credentials. -Install the optional dependencies and start the bridge in a separate session: +Add a Vertex provider to `.harness2/config.toml`, install the optional dependencies and +start the bridge in a separate session: + +```toml +[providers.vertex] +type = "vertex" +project = "my-gcp-project" +region = "global" +``` ```bash uv pip install -e '.[bridge]' -e 'plugins/harness2-benchmarks[render]' -harness2-bridge # for WorkBuddy: harness2-bridge --host 0.0.0.0 +harness2 bridge --model vertex/gemini-3.5-flash # for WorkBuddy: add --host 0.0.0.0 ``` -The bridge prints the `HARNESS2_CODEX_*` settings to add to `.harness2/.env`. Its port and -generated access key are stored under `~/.cache/harness2/bridge/` and reused on -restart. Pass `-c executor.name=codex` to setup, checks, refinement, judging, and +The bridge prints its endpoint and access key. Set `HARNESS2_CODEX_BASE_URL` to that +endpoint (`http://127.0.0.1:/v1`), `HARNESS2_CODEX_API_KEY` to the key and +`HARNESS2_CODEX_MODEL` to the model name in `.harness2/.env`. The port and key are stored +under `~/.cache/harness2/bridge/` and reused on restart. Pass `-c executor.name=codex` to setup, checks, refinement, judging, and reporting: ```bash @@ -129,8 +145,9 @@ harness2 -c executor.name=codex run --suite lab --k 1 --mode parallel \ ``` WorkBuddy executes Codex inside Docker containers, so the bridge must listen on -`--host 0.0.0.0`; it also prints `HARNESS2_CODEX_CONTAINER_URL`. This binds the -bridge to all network interfaces; restrict access to the intended clients. +`--host 0.0.0.0`, and `HARNESS2_CODEX_CONTAINER_URL` must give the address the containers +reach it at, usually `http://host.docker.internal:/v1`. This binds the bridge to all +network interfaces; restrict access to the intended clients. To use another Responses API endpoint, set `HARNESS2_CODEX_BASE_URL`, `HARNESS2_CODEX_API_KEY`, and `HARNESS2_CODEX_MODEL` directly. Setup records the diff --git a/plugins/harness2-benchmarks/harness2_benchmarks/dependencies_meta/ASSETS.json b/plugins/harness2-benchmarks/harness2_benchmarks/dependencies_meta/ASSETS.json index 001c9d29..81d6c07a 100644 --- a/plugins/harness2-benchmarks/harness2_benchmarks/dependencies_meta/ASSETS.json +++ b/plugins/harness2-benchmarks/harness2_benchmarks/dependencies_meta/ASSETS.json @@ -1,6 +1,6 @@ { "third_party/jb_runner/LICENSE": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4", - "third_party/jb_runner/PROVENANCE.md": "ef40bc6abe5ce4e1367e454cc8a4b18a23be59db43ea42650cce5d0771ba4444", + "third_party/jb_runner/PROVENANCE.md": "db5d8237e3670dcdc098fb93f7b6b57b255b6c6b38dc6e37824561491d1bb1d9", "third_party/jb_runner/codex_base_harness/AGENTS.md": "b5df5264478dfce7d6af52f8443a1797b17967d1bc4a8c4f8c5fe43933516a89", "third_party/jb_runner/harness_baseline_oc/LongTermMEMORY.md": "c78b55245ef1433c017bc3f6f796094e16c640b98543836f6d930972243d28d2", "third_party/jb_runner/harness_baseline_oc/guardrails.md": "ebd0b0f287bcc9bea5efa063d632677b3c0fdf014c7b67bec088c11253605a53", diff --git a/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/PROVENANCE.md b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/PROVENANCE.md index baf98997..5f831350 100644 --- a/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/PROVENANCE.md +++ b/plugins/harness2-benchmarks/harness2_benchmarks/third_party/jb_runner/PROVENANCE.md @@ -19,7 +19,7 @@ directly from the package. The OpenCode launcher that used to sit beside it, `run_benchmark_opencode_harness_adapt.sh`, was a fork that shared almost nothing with upstream. It has been replaced by the first-party Python module -[`harness2/benchmarks/jb_opencode_runner.py`](../../benchmarks/jb_opencode_runner.py), +[`harness2_benchmarks/jb_opencode_runner.py`](../../jb_opencode_runner.py), which the adapter launches as a subprocess in its place. That module is release code, not a vendored asset, so it is not listed in `ASSETS.json`. diff --git a/plugins/harness2-benchmarks/pyproject.toml b/plugins/harness2-benchmarks/pyproject.toml index cca55f09..5e991366 100644 --- a/plugins/harness2-benchmarks/pyproject.toml +++ b/plugins/harness2-benchmarks/pyproject.toml @@ -19,6 +19,8 @@ build-backend = "setuptools.build_meta" [project] name = "harness2-benchmarks" version = "0.1.0" +description = "The Harness² paper benchmarks (JobBench, LAB, WorkBuddy-Bench) as harness2 suites" +readme = "README.md" license = "Apache-2.0" requires-python = ">=3.12,<3.14" dependencies = ["harness2"] diff --git a/pyproject.toml b/pyproject.toml index dca5f5c9..0d03a308 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "harness2" version = "0.1.0" -description = "Harness²: Recursive Agent Harnessing for an Open World" +description = "Tune your coding agent's harness on your own tasks" readme = "README.md" license = "Apache-2.0" requires-python = ">=3.12,<3.14" @@ -16,9 +16,14 @@ classifiers = [ ] authors = [ { name = "Google LLC" }, + { name = "The Harness2 Authors" }, ] dependencies = [] +[project.urls] +Homepage = "https://github.com/3ae3ae/harness2" +Paper = "https://arxiv.org/abs/2610.14224" + [project.optional-dependencies] bridge = ["litellm[proxy]==1.95.0", "fastapi==0.139.2", "google-auth==2.56.3"] From ca56c60a8e5a96fa6178b0af711e0f0a2f6361a7 Mon Sep 17 00:00:00 2001 From: Bae Seongmin <83870598+3ae3ae@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:40:00 +0900 Subject: [PATCH 11/21] feat: consolidate a run into one project harness, and eval it (#12) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Harness² adopts one harness per task. `harness2 consolidate` gives the improver the run's starting harness, every task's adopted harness, its request and the lesson the run recorded, and asks for one harness for the project's next tasks; the result is validated against the executor's surface (one retry with the problems listed) and stored in /consolidated// with the improver's notes. `harness2 eval` runs the starting and the consolidated harness on the run's gradable tasks (optionally repeated), grades both and prints the per-task and mean pass rates with the difference. `apply` and `diff` now default to the consolidated harness when there is one; `--task` still picks a single task's adopted harness. Co-authored-by: Claude Opus 5.5 --- harness2/cli/_adopt.py | 8 + harness2/cli/apply.py | 9 +- harness2/cli/consolidate.py | 178 ++++++++++++++++ harness2/cli/diff.py | 2 +- harness2/cli/eval.py | 188 +++++++++++++++++ harness2/cli/main.py | 2 + harness2/core/prompts.py | 37 ++++ tests/test_consolidate.py | 408 ++++++++++++++++++++++++++++++++++++ 8 files changed, 829 insertions(+), 3 deletions(-) create mode 100644 harness2/cli/consolidate.py create mode 100644 harness2/cli/eval.py create mode 100644 tests/test_consolidate.py diff --git a/harness2/cli/_adopt.py b/harness2/cli/_adopt.py index 1fc5eb75..b0a8bd0b 100644 --- a/harness2/cli/_adopt.py +++ b/harness2/cli/_adopt.py @@ -76,6 +76,14 @@ def base_tree(adapter) -> Tree: return read_tree(adapter.substrate.surface(), adapter.base_harness()) +def harness_for(adapter, run_dir, task: str | None, mode: str) -> tuple[str, Tree]: + if task is None: + path = run_dir / "consolidated" / C.mode_label(schedule.mode_token(mode)) / "harness" + if path.is_dir(): + return "consolidated", read_tree(adapter.substrate.surface(), path) + return adopted_tree(adapter, run_dir, task, mode) + + def project_tree(adapter, settings) -> Tree: return adapter.substrate.import_tree(settings.path("paths.harness")) diff --git a/harness2/cli/apply.py b/harness2/cli/apply.py index 94d55392..40125482 100644 --- a/harness2/cli/apply.py +++ b/harness2/cli/apply.py @@ -63,7 +63,7 @@ def add_parser(sub) -> argparse.ArgumentParser: def run(args, ctx) -> int: adapter, run_dir = A.open_run(ctx, args.name) - task, adopted = A.adopted_tree(adapter, run_dir, args.task, args.mode) + task, adopted = A.harness_for(adapter, run_dir, args.task, args.mode) base = A.base_tree(adapter) current = A.project_tree(adapter, ctx.settings) where = ctx.settings.path("paths.harness") @@ -75,7 +75,12 @@ def run(args, ctx) -> int: moved = {rel for rel, _ in delta if current.get(rel) != base.get(rel)} todo = [(rel, kind) for rel, kind in delta if args.force or rel not in moved] - print(f"Harness adopted for {task} ({args.mode}, run {run_dir.name}) -> {where}\n") + label = ( + "consolidated from the run's tasks" + if args.task is None and task == "consolidated" + else f"adopted for {task}" + ) + print(f"Harness {label} ({args.mode}, run {run_dir.name}) -> {where}\n") for rel, kind in delta: mark = "!" if rel in moved else kind print(f" {mark} {rel:<48} {_delta(rel, current, adopted)}") diff --git a/harness2/cli/consolidate.py b/harness2/cli/consolidate.py new file mode 100644 index 00000000..47897bab --- /dev/null +++ b/harness2/cli/consolidate.py @@ -0,0 +1,178 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compose a project harness from a run's successful task selections.""" + +from __future__ import annotations + +import json +import re +import tempfile +from datetime import datetime, timezone +from pathlib import Path + +from ..core import events, library, prompts, recursion, schedule, tree +from ..core.proposer import Proposer +from ..settings import ConfigError +from . import _adopt as A +from . import _common as C +from .run import _stopped + + +def add_parser(subparsers): + p = subparsers.add_parser( + "consolidate", + help="combine a run's adopted harnesses into one for the project", + description="A run adopts one harness per task. Combine them, with the lessons the run " + "recorded, into one harness for the project's next tasks. apply and diff then use it " + "by default. Makes one improver session.", + ) + p.add_argument("name", nargs="?", help="run name (default: the most recent run)") + p.add_argument( + "--mode", + choices=("parallel", "sequential"), + default="parallel", + help="take the harnesses adopted in this recursion mode", + ) + return p + + +def _context( + adapter, root: Path, record: dict, mode: str, dest: Path, base: tree.Tree +) -> list[str]: + surface = adapter.substrate.surface() + token = schedule.mode_token(mode) + plan = record["schedule"] + namespace = C.summary_namespace(adapter, plan["k"], seed=plan.get("seed")) + selected = { + row["task_id"] + for path, row in C.summaries(root) + if path.parent.parent.name == namespace + and (selection := (row.get("selections") or {}).get(token) or {}).get("ok") + and selection.get("chosen_label") + } + tasks, index = [], [] + tree.write_tree(surface, base, dest / "harness", stubs=True) + lib = C.library_root_for(adapter, plan["k"], seed=plan.get("seed")) / token + for domain, task in record["stream"]: + if task not in selected: + continue + _, adopted = A.adopted_tree(adapter, root, task, mode) + task_dir = dest / "tasks" / library.slug(task) + tree.write_tree(surface, adopted, task_dir / "harness", stubs=False) + (task_dir / "task.md").write_text(adapter.task_prompt(task), encoding="utf-8") + lesson = ( + lib / library.domain_dir(domain) / "harnesses" / library.slug(task) / "EXPERIENCE.md" + ) + if lesson.is_file(): + (task_dir / "EXPERIENCE.md").write_text(lesson.read_text(), encoding="utf-8") + index.append( + f"- {task} ({domain}): adopted harness " + f"{'differs from' if adopted != base else 'same as'} the start; " + f"lesson: {'yes' if lesson.is_file() else 'no'}" + ) + tasks.append(task) + if not tasks: + raise ConfigError(f"no successful selection for any task in {mode}") + (dest / "INDEX.md").write_text("\n".join(index) + "\n", encoding="utf-8") + return tasks + + +def run(args, ctx) -> int: + with ctx.progress(): + adapter, root = A.open_run(ctx, args.name) + base = A.base_tree(adapter) + record = C.load_run(root) + surface = adapter.substrate.surface() + config = ctx.settings + events.start( + root / "events.jsonl", + budget_usd=config.get("run.budget_usd"), + max_calls=config.get("run.max_calls"), + ) + prop = Proposer(adapter.spec.channels["proposer"], audit_dir=root / "consolidated/_audit") + cost = 0.0 + try: + with tempfile.TemporaryDirectory(prefix="harness2-consolidate-") as temporary: + context = Path(temporary) + tasks = _context(adapter, root, record, args.mode, context, base) + mission = prompts.CONSOLIDATE_MISSION.replace("{n}", str(len(tasks))).replace( + "{surface}", prompts._st(surface).surface_md + ) + err = "" + for _ in range(2): + with ctx.progress(): + reply = prop.call( + mission + + (f"\n\n# YOUR PREVIOUS ATTEMPT WAS REJECTED\n\n{err}\n\n" if err else ""), + context, + f"consolidate.{args.mode}", + harvest="context/harness", + ) + cost += reply.cost_usd + if reply.workspace is None: + err = "the harness directory did not come back from the session" + continue + rejected = [] + new = tree.read_tree(surface, reply.workspace, rejected=rejected) + _, err = recursion._accept_tree(surface, adapter, new, base, "consolidate") + if rejected: + err = "files outside the editable surface: " + ", ".join(sorted(rejected)) + if not err: + break + tree.write_tree(surface, new, context / "harness", stubs=True) + else: + raise ConfigError(err) + except events.Stopped as exc: + reason = str(exc) + ctx.out(events.totals(), lambda totals: _stopped(root.name, reason, totals)) + return 4 + notes = re.search(r"(.*?)", reply.text, re.S) + notes = notes.group(1).strip() if notes else reply.text.strip() + dest = root / "consolidated" / args.mode + path = tree.write_tree(surface, new, dest / "harness", stubs=False) + (dest / "notes.md").write_text(notes + "\n", encoding="utf-8") + (dest / "consolidation.json").write_text( + json.dumps( + { + "tasks": tasks, + "model": prop.spec.model, + "cost_usd": cost, + "created": datetime.now(timezone.utc).isoformat(), + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + changes = A.changes(base, new) + ctx.out( + {"path": path, "tasks": tasks, "changes": changes, "notes": notes}, + lambda _: "\n".join( + [ + f"Consolidated the harnesses adopted for {len(tasks)} task(s) ({args.mode}) into {path}", + *( + [f" {kind} {rel}" for rel, kind in changes] + if changes + else [ + "The consolidated harness is the same as the starting one: " + "no task's edits generalized." + ] + ), + f"Next: harness2 eval {root.name} compares it with the starting harness; " + f"harness2 apply {root.name} writes it into the project.", + ] + ), + ) + return 0 diff --git a/harness2/cli/diff.py b/harness2/cli/diff.py index 98997cbd..274d9944 100644 --- a/harness2/cli/diff.py +++ b/harness2/cli/diff.py @@ -43,7 +43,7 @@ def add_parser(subparsers): def run(args, ctx) -> int: with ctx.progress(): adapter, root = _adopt.open_run(ctx, args.name) - task, after = _adopt.adopted_tree(adapter, root, args.task, args.mode) + task, after = _adopt.harness_for(adapter, root, args.task, args.mode) before = _adopt.base_tree(adapter) data = ( diff_summary(adapter.substrate.surface(), before, after) diff --git a/harness2/cli/eval.py b/harness2/cli/eval.py new file mode 100644 index 00000000..5f438f34 --- /dev/null +++ b/harness2/cli/eval.py @@ -0,0 +1,188 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compare consolidated and starting harnesses on a run's tasks.""" + +from __future__ import annotations + +import json +from concurrent.futures import ThreadPoolExecutor + +from ..core import events +from ..settings import ConfigError +from . import _adopt as A +from . import _common as C +from .run import _csv, _stopped + + +def add_parser(subparsers): + p = subparsers.add_parser( + "eval", + help="compare the consolidated harness with the starting one", + description="Run the agent with the starting harness and with the consolidated one on " + "the run's tasks, grade both with each task's check or rubric, and compare. Costs one " + "agent run per task, harness and repeat.", + ) + p.add_argument("name", nargs="?", help="run name (default: the most recent run)") + p.add_argument( + "--mode", + choices=("parallel", "sequential"), + default="parallel", + help="compare the harness consolidated in this recursion mode", + ) + p.add_argument( + "--split", choices=("all", "train", "test"), default="all", help="compare one split only" + ) + p.add_argument("--only-tasks", default="", metavar="ID,...", help="compare only these tasks") + p.add_argument("--repeat", type=int, default=1, metavar="N", help="runs per task and harness") + return p + + +def _rollout(adapter, task, harness, run_id): + events.check() + run_dir = adapter.execute(task, harness, run_id) + cost = getattr(adapter, "rollout_cost", None) + events.emit( + "rollout", + task=task, + run_id=run_id, + cost_usd=float(cost(run_dir)) if callable(cost) else 0.0, + ) + try: + grade = adapter.score(run_dir, task) + except events.Stopped: + raise + except Exception: + return None + if not grade: + return None + passed, total = grade.get("n_passed"), grade.get("n_criteria") + if type(passed) is not int or type(total) is not int or total <= 0 or not 0 <= passed <= total: + return None + return {"n_passed": passed, "n_criteria": total} + + +def _mean(values): + graded = [value for value in values if value is not None] + return sum(graded) / len(graded) if graded else None + + +def _text(data): + def percent(value): + return "-" if value is None else f"{value * 100:g}%" + + lines = [ + f"Comparing the consolidated harness with the starting one on {len(data['tasks'])} " + f"task(s), {data['repeat']} run(s) each.", + "TASK START CONSOLIDATED", + ] + for task, row in data["tasks"].items(): + lines.append( + f"{task} {percent(row['start']['mean'])} {percent(row['consolidated']['mean'])}" + ) + start, consolidated = data["mean"]["start"], data["mean"]["consolidated"] + delta = ( + f"{(consolidated - start) * 100:+g}" + if start is not None and consolidated is not None + else "-" + ) + lines.append(f"mean {percent(start)} {percent(consolidated)} ({delta} points)") + if data["skipped"]: + lines.append( + f"Skipped {len(data['skipped'])} task(s) with no check or rubric: " + + ", ".join(data["skipped"]) + ) + lines.append(f"Saved to {data['path']}") + return "\n".join(lines) + + +def run(args, ctx) -> int: + if args.repeat < 1: + raise ConfigError("--repeat must be a positive integer") + with ctx.progress(): + adapter, root = A.open_run(ctx, args.name) + harness = root / "consolidated" / args.mode / "harness" + if not harness.is_dir(): + raise ConfigError("run `harness2 consolidate` first") + record = C.load_run(root) + allowed = set(adapter.tasks(args.split)) + wanted = set(_csv(args.only_tasks)) + tasks = [ + task for _, task in record["stream"] if task in allowed and (not wanted or task in wanted) + ] + missing = wanted - set(tasks) + if missing: + raise ConfigError( + f"--only-tasks names unknown task id(s) {sorted(missing)} for split {args.split!r}" + ) + has_judge = getattr(adapter, "has_judge", lambda task: True) + skipped = [task for task in tasks if not has_judge(task)] + tasks = [task for task in tasks if task not in skipped] + config = ctx.settings + events.start( + root / "events.jsonl", + budget_usd=config.get("run.budget_usd"), + max_calls=config.get("run.max_calls"), + ) + rows = {task: {name: {"runs": []} for name in ("start", "consolidated")} for task in tasks} + jobs = [ + (task, name, directory, f"eval/{name}/{task}/{i}") + for task in tasks + for i in range(args.repeat) + for name, directory in (("start", None), ("consolidated", harness)) + ] + try: + with ctx.progress(), ThreadPoolExecutor(max_workers=config.get("run.phase_width")) as pool: + futures = [ + pool.submit(_rollout, adapter, task, directory, run_id) + for task, _, directory, run_id in jobs + ] + for (task, name, _, _), future in zip(jobs, futures): + rows[task][name]["runs"].append(future.result()) + except events.Stopped as exc: + reason = str(exc) + ctx.out(events.totals(), lambda totals: _stopped(root.name, reason, totals)) + return 4 + for row in rows.values(): + for result in row.values(): + result["mean"] = _mean( + [ + grade["n_passed"] / grade["n_criteria"] if grade else None + for grade in result["runs"] + ] + ) + path = root / "eval" / args.mode / "results.json" + data = { + "tasks": rows, + "mean": { + name: _mean([row[name]["mean"] for row in rows.values()]) + for name in ("start", "consolidated") + }, + "repeat": args.repeat, + "skipped": skipped, + "path": str(path), + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") + ctx.out(data, _text) + return ( + 3 + if any( + grade is None + for row in rows.values() + for result in row.values() + for grade in result["runs"] + ) + else 0 + ) diff --git a/harness2/cli/main.py b/harness2/cli/main.py index cce81083..2e7a7ac9 100644 --- a/harness2/cli/main.py +++ b/harness2/cli/main.py @@ -33,6 +33,8 @@ "report", "runs", "diff", + "consolidate", + "eval", "apply", "config", "chat", diff --git a/harness2/core/prompts.py b/harness2/core/prompts.py index b38b5221..d553df72 100644 --- a/harness2/core/prompts.py +++ b/harness2/core/prompts.py @@ -1367,3 +1367,40 @@ def experience(bp: BenchPrompts) -> str: ] ) ) + + +CONSOLIDATE_MISSION = """\ +You are consolidating an autonomous agent's harness for the project it works on. + +The agent has already been run on {n} task(s) from this project. For each task, its harness +was refined on evidence from that task's own runs, and the best result was adopted. Those +adopted harnesses are task-specific: each was shaped for one request. Your job is to compose +ONE harness for the project's next tasks -- tasks nobody has seen yet. + +# YOUR EVIDENCE + +context/harness/ the harness the runs started from -- edit THIS one +context/tasks//task.md the request each task made +context/tasks//harness/ the harness adopted for that task +context/tasks//EXPERIENCE.md which edits made a difference and the lesson drawn, when + one was recorded +context/INDEX.md the tasks, their domains, and which adopted harnesses + differ from the start + +Read INDEX.md, then compare each adopted harness with context/harness/ and read the lessons. + +# WHAT TO KEEP + +- A component or rule that helped on several tasks, or that a lesson supports as general, + belongs in the consolidated harness. +- A workspace fact tied to one task -- a file it touched, a value it computed, a mapping it + verified -- does not. Keep the procedure that found it, if that procedure generalizes. +- Where two tasks' edits conflict, keep the one with the stronger evidence, or neither. +- When the adopted harnesses mostly left the start unchanged, so should you: an edit no task + supported costs context on every future run and buys nothing. +- Knowledge that applies only in some situations belongs in an on-demand component, not in + always-on text. Keep always-on text short. + +{surface} + +In your ``, name for each change the tasks and lessons that support it.""" diff --git a/tests/test_consolidate.py b/tests/test_consolidate.py new file mode 100644 index 00000000..6f1fd024 --- /dev/null +++ b/tests/test_consolidate.py @@ -0,0 +1,408 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline consolidation, adoption, and evaluation through a fake Claude CLI.""" + +from __future__ import annotations + +import contextlib +import io +import json +import os +import shlex +import sys +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock, patch + +from harness2.cli import _adopt, _common, consolidate, eval +from harness2.cli.main import COMMANDS, build_parser, main +from harness2.core import events +from harness2.settings import ConfigError +from harness2.suite import GenericAdapter + +FAKE_CLAUDE = r"""import json +import os +import sys +from pathlib import Path + +if "--version" in sys.argv: + print("fake claude 1.0") + sys.exit(0) + +prompt = sys.stdin.read() +assert prompt.strip() +assert os.environ["ANTHROPIC_API_KEY"] == "offline-fixture" +cwd = Path.cwd() +context = cwd / "context" +output_format = sys.argv[sys.argv.index("--output-format") + 1] +if output_format == "stream-json": + output = cwd / "output/result.txt" + output.parent.mkdir(exist_ok=True) + output.write_text("pass" if "Use shared procedure." in (cwd / "CLAUDE.md").read_text() else "fail") + print(json.dumps({"type": "result", "subtype": "success", "is_error": False, + "result": "Saved the result.", "total_cost_usd": 0.125})) + sys.exit(0) + +harness = context / "harness/CLAUDE.md" +if "You are consolidating an autonomous agent" in prompt: + counter = Path(os.environ["WC_CALLS"]) + call = int(counter.read_text()) + 1 if counter.exists() else 1 + counter.write_text(str(call)) + task_dirs = sorted((context / "tasks").iterdir()) + index = (context / "INDEX.md").read_text().splitlines() + assert len(index) == len(task_dirs) + assert f"run on {len(task_dirs)} task(s)" in prompt + for task in task_dirs: + assert (task / "task.md").read_text() == f"Complete {task.name}.\n" + assert "Task note." in (task / "harness/CLAUDE.md").read_text() + assert (task / "EXPERIENCE.md").is_file() == any( + line.startswith(f"- {task.name} ") and line.endswith("lesson: yes") for line in index) + assert all("(ops/review): adopted harness differs from the start; lesson:" in line for line in index) + assert not list(context.rglob("rubric.md")) + behavior = os.environ.get("WC_BEHAVIOR", "retry") + invalid = context / "harness/.claude/scripts/broken.py" + if behavior == "missing": + import shutil + shutil.rmtree(context / "harness") + elif behavior == "invalid" or (behavior == "retry" and call == 1): + invalid.parent.mkdir(parents=True, exist_ok=True) + invalid.write_text("def broken(") + else: + if behavior == "retry" and call == 2: + assert "# YOUR PREVIOUS ATTEMPT WAS REJECTED" in prompt + assert "broken.py" in prompt + assert invalid.is_file() + invalid.unlink(missing_ok=True) + harness.write_text("Read sources.\n" if behavior == "unchanged" else "Use shared procedure.\n") + text = "a-one and b-two support the shared procedure." +elif (context / "adoption.json").is_file(): + text = ("\ncomponent: CLAUDE.md\ncandidates: inv1\n" + "change: added a procedure\nobserved: the output was produced\nverdict: helped\n\n" + "\ncue: When doing project work\nlesson: Use a shared procedure.\n" + "evidence: A task note accompanied the completed output.\n" + "confidence: specific to this task\n") +else: + harness.write_text(harness.read_text() + "Task note.\n") + text = "Added a task note." +print(json.dumps({"type": "result", "subtype": "success", "is_error": False, + "result": text, "total_cost_usd": 0.01})) +""" + + +class ConsolidateTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.project = self.root / "project" + self.state = self.project / ".harness2" + self.state.mkdir(parents=True) + (self.project / "CLAUDE.md").write_text("Read sources.\n") + (self.state / "config.toml").write_text( + '[executor]\nname = "claude"\nmodel = "anthropic/x"\n' + '[improver]\nchannel = "claude"\nmodel = "anthropic/y"\n' + '[providers.anthropic]\napi_key_env = "WC_KEY"\n' + ) + binaries = self.root / "bin" + binaries.mkdir() + binary = binaries / "claude" + binary.write_text(f"#!{sys.executable}\n" + FAKE_CLAUDE) + binary.chmod(0o755) + self.calls = self.root / "consolidate-calls" + environment = patch.dict( + os.environ, + { + "PATH": f"{binaries}{os.pathsep}{os.defpath}", + "XDG_CONFIG_HOME": str(self.root / "config"), + "XDG_CACHE_HOME": str(self.root / "cache"), + "WC_KEY": "offline-fixture", + "WC_CALLS": str(self.calls), + "PYTHONDONTWRITEBYTECODE": "1", + }, + clear=True, + ) + environment.start() + self.addCleanup(environment.stop) + self.addCleanup(events.start, None) + for task, split in (("a-one", "train"), ("b-two", "test")): + self.new_task(task, split=split) + self.run_dir = self.state / "runs/trial" + + def invoke(self, *argv, code=0): + stdout, stderr = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + result = main(["-C", str(self.project), *argv]) + self.assertEqual(result, code, f"{argv!r}\n{stdout.getvalue()}\n{stderr.getvalue()}") + return stdout.getvalue(), stderr.getvalue() + + def new_task(self, task, *, split="train", judge=True): + check = shlex.join( + [ + sys.executable, + "-B", + "-c", + "from pathlib import Path; assert Path('output/result.txt').read_text() == 'pass'", + ] + ) + self.invoke( + "task", + "new", + task, + "--files", + "--domain", + "ops/review", + *(["--check", check] if judge else []), + ) + directory = self.state / "tasks" / task + (directory / "task.md").write_text(f"Complete {task}.\n") + with (directory / "task.toml").open("a") as stream: + stream.write(f'split = "{split}"\n') + + def execute(self, *argv): + self.invoke("run", "--name", "trial", *argv) + + def adapter(self): + args = build_parser().parse_args(["-C", str(self.project), "diff", "trial"]) + return _adopt.open_run(_common.Context(args, {}), "trial")[0] + + def test_two_tasks_retry_adoption_eval_and_json(self): + self.execute() + before = _common.event_totals(self.run_dir) + fallback = json.loads(self.invoke("--json", "diff", "trial")[0]) + self.assertEqual(fallback["task"], "b-two") + text, _ = self.invoke("consolidate", "trial") + self.assertEqual(self.calls.read_text(), "2") + path = self.run_dir / "consolidated/parallel/harness" + self.assertEqual((path / "CLAUDE.md").read_text(), "Use shared procedure.\n") + self.assertIn( + f"Consolidated the harnesses adopted for 2 task(s) (parallel) into {path}", text + ) + self.assertIn(" M CLAUDE.md", text) + self.assertIn( + "Next: harness2 eval trial compares it with the starting harness; " + "harness2 apply trial writes it into the project.", + text, + ) + meta = json.loads((path.parent / "consolidation.json").read_text()) + self.assertEqual(meta["tasks"], ["a-one", "b-two"]) + self.assertEqual((meta["model"], meta["cost_usd"]), ("anthropic/y", 0.02)) + self.assertTrue(meta["created"]) + self.assertEqual( + (path.parent / "notes.md").read_text(), + "a-one and b-two support the shared procedure.\n", + ) + data = json.loads(self.invoke("--json", "consolidate", "trial")[0]) + self.assertEqual(set(data), {"path", "tasks", "changes", "notes"}) + self.assertEqual(data["path"], str(path)) + self.assertEqual(data["changes"], [["CLAUDE.md", "M"]]) + self.assertEqual(data["tasks"], ["a-one", "b-two"]) + self.assertEqual(data["notes"], "a-one and b-two support the shared procedure.") + consolidated = json.loads(self.invoke("--json", "diff", "trial")[0]) + self.assertEqual(consolidated["task"], "consolidated") + self.assertIn("+Use shared procedure.", consolidated["diff"]) + task = json.loads(self.invoke("--json", "diff", "trial", "--task", "a-one")[0]) + self.assertEqual(task["task"], "a-one") + self.assertIn("+Task note.", task["diff"]) + applied, _ = self.invoke("apply", "trial", "--dry-run", "--task", "a-one") + self.assertIn("Harness adopted for a-one (parallel, run trial)", applied) + applied, _ = self.invoke("apply", "trial", "--yes") + self.assertIn("Harness consolidated from the run's tasks (parallel, run trial)", applied) + self.assertEqual((self.project / "CLAUDE.md").read_text(), "Use shared procedure.\n") + text, _ = self.invoke("eval", "trial", "--repeat", "2") + self.assertIn( + "Comparing the consolidated harness with the starting one on 2 task(s), 2 run(s) each.", + text, + ) + self.assertIn("TASK START CONSOLIDATED\na-one 0% 100%\nb-two 0% 100%", text) + self.assertIn("mean 0% 100% (+100 points)", text) + self.assertIn(f"Saved to {self.run_dir / 'eval/parallel/results.json'}", text) + results = json.loads((self.run_dir / "eval/parallel/results.json").read_text()) + self.assertEqual(results["mean"], {"start": 0.0, "consolidated": 1.0}) + for row in results["tasks"].values(): + self.assertEqual( + row["start"], {"runs": [{"n_passed": 0, "n_criteria": 1}] * 2, "mean": 0.0} + ) + self.assertEqual( + row["consolidated"], {"runs": [{"n_passed": 1, "n_criteria": 1}] * 2, "mean": 1.0} + ) + event_rows = [ + json.loads(line) for line in (self.run_dir / "events.jsonl").read_text().splitlines() + ] + eval_rows = [row for row in event_rows if row.get("run_id", "").startswith("eval/")] + self.assertEqual(len(eval_rows), 8) + self.assertEqual({row["cost_usd"] for row in eval_rows}, {0.125}) + self.assertAlmostEqual( + _common.event_totals(self.run_dir)["spent_usd"], before["spent_usd"] + 1.03 + ) + self.assertEqual( + {row["run_id"] for row in eval_rows}, + { + f"eval/{name}/{task}/{i}" + for task in ("a-one", "b-two") + for name in ("start", "consolidated") + for i in range(2) + }, + ) + + def test_eval_filters_skipped_failed_grades_and_limits(self): + self.new_task("c-no-judge", judge=False) + self.execute() + self.invoke("consolidate", "trial") + text, _ = self.invoke("eval", "trial") + self.assertIn("Skipped 1 task(s) with no check or rubric: c-no-judge", text) + data = json.loads( + self.invoke( + "--json", + "eval", + "trial", + "--split", + "test", + "--only-tasks", + "b-two", + "--repeat", + "2", + )[0] + ) + self.assertEqual(list(data["tasks"]), ["b-two"]) + self.assertEqual(data["skipped"], []) + self.assertEqual(data["repeat"], 2) + self.assertEqual(json.loads(Path(data["path"]).read_text()), data) + with patch.object(GenericAdapter, "score", return_value=None): + text, _ = self.invoke("eval", "trial", "--only-tasks", "a-one", code=3) + self.assertIn("a-one - -", text) + self.assertIn("mean - - (- points)", text) + data = json.loads((self.run_dir / "eval/parallel/results.json").read_text()) + self.assertEqual(data["tasks"]["a-one"]["start"], {"runs": [None], "mean": None}) + text, _ = self.invoke("-c", "run.max_calls=1", "eval", "trial", code=4) + self.assertIn("Run trial stopped: reached --max-calls 1", text) + with patch.object(GenericAdapter, "execute", side_effect=events.Stopped("usage_limit")): + self.invoke("eval", "trial", code=4) + _, err = self.invoke("eval", "trial", "--repeat", "0", code=1) + self.assertIn("--repeat must be a positive integer", err) + _, err = self.invoke("eval", "trial", "--only-tasks", "missing", code=1) + self.assertIn("unknown task id(s)", err) + data = json.loads(self.invoke("--json", "eval", "trial", "--only-tasks", "c-no-judge")[0]) + self.assertEqual(data["tasks"], {}) + self.assertEqual(data["mean"], {"start": None, "consolidated": None}) + self.assertEqual(data["skipped"], ["c-no-judge"]) + + def test_modes_unchanged_and_validation_failure_preserves_result(self): + self.execute("--mode", "parallel,sequential") + self.invoke("consolidate", "trial", "--mode", "sequential") + adapter = self.adapter() + self.assertEqual(_adopt.harness_for(adapter, self.run_dir, None, "seq")[0], "consolidated") + self.assertEqual( + _adopt.harness_for(adapter, self.run_dir, "a-one", "sequential")[0], "a-one" + ) + self.assertEqual(_adopt.harness_for(adapter, self.run_dir, None, "parallel")[0], "b-two") + path = self.run_dir / "consolidated/sequential/harness/CLAUDE.md" + with patch.dict(os.environ, {"WC_BEHAVIOR": "invalid"}): + _, err = self.invoke("consolidate", "trial", "--mode", "sequential", code=1) + self.assertIn("broken.py", err) + self.assertEqual(path.read_text(), "Use shared procedure.\n") + with patch.dict(os.environ, {"WC_BEHAVIOR": "missing"}): + _, err = self.invoke("consolidate", "trial", code=1) + self.assertIn("the harness directory did not come back", err) + with patch.dict(os.environ, {"WC_BEHAVIOR": "unchanged"}): + text, _ = self.invoke("consolidate", "trial") + self.assertIn( + "The consolidated harness is the same as the starting one: no task's edits generalized.", + text, + ) + self.assertEqual(self.invoke("diff", "trial")[0], "\n") + + def test_context_stream_order_optional_lessons_and_errors(self): + self.execute() + _, err = self.invoke("eval", "trial", code=1) + self.assertIn("run `harness2 consolidate` first", err) + adapter = self.adapter() + record = _common.load_run(self.run_dir) + plan = record["schedule"] + lesson = ( + _common.library_root_for(adapter, plan["k"], seed=plan.get("seed")) + / "agg/ops__review/harnesses/b-two/EXPERIENCE.md" + ) + lesson.unlink() + record["stream"].reverse() + dest = self.root / "context" + base = _adopt.base_tree(adapter) + task_path, row = next( + (path, row) + for path, row in _common.summaries(self.run_dir) + if row["task_id"] == "b-two" + ) + row["selections"]["agg"]["chosen_tree_is_base"] = True + task_path.write_text(json.dumps(row)) + tasks = consolidate._context(adapter, self.run_dir, record, "parallel", dest, base) + self.assertEqual(tasks, ["b-two", "a-one"]) + self.assertEqual( + (dest / "INDEX.md").read_text(), + "- b-two (ops/review): adopted harness same as the start; lesson: no\n" + "- a-one (ops/review): adopted harness differs from the start; lesson: yes\n", + ) + self.assertEqual((dest / "tasks/b-two/task.md").read_text(), "Complete b-two.\n") + self.assertFalse((dest / "tasks/b-two/EXPERIENCE.md").exists()) + for path, row in _common.summaries(self.run_dir): + row["selections"]["agg"]["ok"] = False + path.write_text(json.dumps(row)) + with self.assertRaisesRegex(ConfigError, "no successful selection"): + consolidate._context( + adapter, self.run_dir, record, "parallel", self.root / "empty", base + ) + + def test_rollout_grades_and_optional_cost(self): + adapter = SimpleNamespace(execute=Mock(return_value=self.root), score=Mock()) + for grade in ( + None, + {}, + {"n_passed": 1, "n_criteria": 0}, + {"n_passed": True, "n_criteria": 1}, + {"n_passed": 2, "n_criteria": 1}, + ): + adapter.score.return_value = grade + self.assertIsNone(eval._rollout(adapter, "task", None, "eval/start/task/0")) + adapter.score.side_effect = ValueError("failed judge") + self.assertIsNone(eval._rollout(adapter, "task", None, "eval/start/task/0")) + adapter.score.side_effect = events.Stopped("usage_limit") + with self.assertRaisesRegex(events.Stopped, "usage_limit"): + eval._rollout(adapter, "task", None, "eval/start/task/0") + self.assertEqual(eval._mean([0.0, None, 1.0]), 0.5) + self.assertIsNone(eval._mean([None])) + data = { + "tasks": {}, + "repeat": 1, + "skipped": [], + "path": "results.json", + "mean": {"start": 1.0, "consolidated": 0.0}, + } + self.assertIn("mean 100% 0% (-100 points)", eval._text(data)) + + def test_command_order_and_help(self): + i = COMMANDS.index("diff") + self.assertEqual(COMMANDS[i + 1 : i + 3], ("consolidate", "eval")) + parser = build_parser() + for name in ("consolidate", "eval"): + sub = parser._subparsers._group_actions[0].choices[name] + for action in sub._actions: + self.assertTrue(action.help) + if action.dest != "help": + self.assertEqual(action.help, action.help.lower()) + self.assertFalse(action.help.endswith(".")) + + +if __name__ == "__main__": + unittest.main() From bdfbee8bc0029b6a1eba8c7ad28493a2e545d80d Mon Sep 17 00:00:00 2001 From: Bae Seongmin <83870598+3ae3ae@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:46:10 +0900 Subject: [PATCH 12/21] chore: rename the distribution to harness2-cli, add CI, tidy commands (#13) - The distribution is now harness2-cli (the command and the import package stay `harness2`); the benchmarks plugin depends on it, and the project URLs point at 3ae3ae/harness2-cli with the upstream linked. - GitHub Actions runs the core and plugin tests on Linux and macOS with Python 3.12 and 3.13, plus ruff check and ruff format --check. - `harness2 init` asks whether the models are reached through the agent CLI's own login (a Claude or ChatGPT subscription, the default) or an API key, and says whether that CLI is logged in; --auth answers it. - `harness2 runs rm` deletes a run after asking (--yes to skip), and `runs ls` / `runs show` print tables instead of raw values. Co-authored-by: Claude Opus 5.5 --- .github/workflows/ci.yml | 40 ++++++++++++ harness2/cli/_common.py | 2 +- harness2/cli/init.py | 64 ++++++++++++++++--- harness2/cli/main.py | 4 +- harness2/cli/runs.py | 61 ++++++++++++++++-- .../harness2_benchmarks/preflight.py | 4 +- plugins/harness2-benchmarks/pyproject.toml | 2 +- pyproject.toml | 7 +- tests/test_ui.py | 6 +- 9 files changed, 163 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..bd41756b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,40 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + test: + name: Tests (Python ${{ matrix.python }}, ${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + python: ["3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + with: + python-version: ${{ matrix.python }} + activate-environment: true + - name: Install + run: uv pip install -e . -e plugins/harness2-benchmarks + - name: Core tests + run: python -B -m unittest discover -s tests + - name: Plugin tests + run: python -B -m unittest discover -s plugins/harness2-benchmarks/tests + + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + - run: uvx ruff check harness2 tests plugins + - run: uvx ruff format --check harness2 tests plugins diff --git a/harness2/cli/_common.py b/harness2/cli/_common.py index 087a28ef..2d013b59 100644 --- a/harness2/cli/_common.py +++ b/harness2/cli/_common.py @@ -173,7 +173,7 @@ def save_run(run_dir: Path, config: settings.Settings, stream: list, plan: dict) record = { "name": run_dir.name, "created": old["created"] if old else datetime.now(timezone.utc).isoformat(), - "harness2_version": version("harness2"), + "harness2_version": version("harness2-cli"), "suite": config.get("suite.name"), "executor": config.get("executor.name"), "settings": subset, diff --git a/harness2/cli/init.py b/harness2/cli/init.py index c3626afe..81f63cb0 100644 --- a/harness2/cli/init.py +++ b/harness2/cli/init.py @@ -26,9 +26,16 @@ from .. import settings as S from ..core import tree as HT -DEFAULTS = { - "claude": ("anthropic/claude-sonnet-5-5", "claude", "anthropic/claude-opus-5-5"), - "codex": ("openai/gpt-6.1-sol", "codex", "openai/gpt-6.1-sol"), +# executor/channel -> (model that runs tasks, model that edits the harness) +MODELS = { + "claude": ("claude-sonnet-5-5", "claude-opus-5-5"), + "codex": ("gpt-6.1-sol", "gpt-6.1-sol"), +} +# executor/channel -> (subscription provider, API-key provider) +PROVIDERS = {"claude": ("claude-login", "anthropic"), "codex": ("chatgpt-login", "openai")} +AUTH = { + "claude": ("your Claude subscription, through Claude Code's own login", "ANTHROPIC_API_KEY"), + "codex": ("your ChatGPT subscription, through Codex's own login", "OPENAI_API_KEY"), } CONFIG = """\ @@ -48,7 +55,7 @@ # The selector picks the final harness when several candidates compete, and the judge # grades tasks that have a rubric.md. Both use the improver's settings unless set here. # [selector] -# model = "anthropic/claude-sonnet-5-5" +# model = {selector_example} [paths] # Where the harness lives, relative to the project root. @@ -73,6 +80,11 @@ def add_parser(sub) -> argparse.ArgumentParser: p.add_argument("--yes", "-y", action="store_true", help="take every default, ask nothing") p.add_argument("--force", action="store_true", help="overwrite an existing config file") p.add_argument("--executor", choices=("claude", "codex", "command"), help="skip that question") + p.add_argument( + "--auth", + choices=("login", "api-key"), + help="use the agent CLI's own login (subscription) or a provider API key", + ) return p @@ -105,10 +117,15 @@ def run(args, ctx) -> int: editable = [g for g in editable if g] command_lines = COMMAND_LINES.format(command=_q(command), editable=_q(editable)) channel = "claude" if found["claude"] else "codex" - model, improver = DEFAULTS[channel][0], DEFAULTS[channel][2] else: - model, channel, improver = DEFAULTS[executor] - model = ask("Model that runs the tasks (provider/model)", model) + channel = executor + auth = args.auth or (_pick_auth(ask, channel) if interactive else "login") + provider = PROVIDERS[channel][auth != "login"] + model, improver = (f"{provider}/{m}" for m in MODELS[channel]) + if executor == "command": + model = "" + if executor != "command": + model = ask("Model that runs the tasks (provider/model)", model) improver = ask("Model that edits the harness (provider/model)", improver) harness = ask("Where the harness lives, relative to the project root", ".") @@ -121,6 +138,7 @@ def run(args, ctx) -> int: channel=_q(channel), improver=_q(improver), harness=_q(harness), + selector_example=_q(improver), ), encoding="utf-8", ) @@ -208,14 +226,31 @@ def _harness_summary(where: Path, executor: str, editable: list[str]) -> str: return f"Harness found in {where}: {', '.join(parts)}." +def _pick_auth(ask, channel: str) -> str: + login, key = AUTH[channel] + print(f"\nHow should {channel} reach its models?") + print(f" 1) {login}") + print(f" 2) an API key in {key}") + while True: + answer = ask("Choose", "1") + if answer in ("1", "2"): + return "login" if answer == "1" else "api-key" + print("Type 1 or 2.") + + def _key_summary(root: Path, models: tuple[str, ...]) -> str: - """Whether the API key each chosen model needs is visible to harness2.""" + """Whether each chosen model can be reached: the CLI login, or the API key.""" from .. import providers S.load_dotenv(root / ".harness2/.env") loaded = S.load(None) lines = [] for name in dict.fromkeys(providers.split_model(m)[0] for m in models if "/" in m): + if name in LOGIN_CHECKS: + argv, cli, fix = LOGIN_CHECKS[name] + ok = shutil.which(argv[0]) and _quiet(argv) + lines.append(f"{cli} is logged in." if ok else f"{cli} is not logged in: run `{fix}`.") + continue try: env = loaded.provider(name).api_key_env except S.ConfigError: @@ -232,3 +267,16 @@ def _key_summary(root: Path, models: tuple[str, ...]) -> str: def _q(value) -> str: return S._literal(value) + + +LOGIN_CHECKS = { + "claude-login": (["claude", "auth", "status"], "Claude Code", "claude auth login"), + "chatgpt-login": (["codex", "login", "status"], "Codex", "codex login"), +} + + +def _quiet(argv: list[str]) -> bool: + try: + return subprocess.run(argv, capture_output=True, timeout=20).returncode == 0 + except (OSError, subprocess.TimeoutExpired): + return False diff --git a/harness2/cli/main.py b/harness2/cli/main.py index 2e7a7ac9..091ab2c8 100644 --- a/harness2/cli/main.py +++ b/harness2/cli/main.py @@ -52,7 +52,7 @@ def build_parser() -> argparse.ArgumentParser: prog="harness2", description="Tune your coding agent's harness on your own tasks.", epilog="Start with `harness2 init` in your project. " - "Documentation: https://github.com/3ae3ae/harness2", + "Documentation: https://github.com/3ae3ae/harness2-cli", ) parser.add_argument("-C", "--project", metavar="DIR", help="run as if started in DIR") parser.add_argument("--profile", metavar="NAME", help="apply the settings profile NAME") @@ -67,7 +67,7 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("-q", "--quiet", action="store_true", help="print results only") parser.add_argument("-v", "--verbose", action="store_true", help="print debug details") parser.add_argument("--json", action="store_true", help="print results as JSON") - parser.add_argument("--version", action="version", version=version("harness2")) + parser.add_argument("--version", action="version", version=version("harness2-cli")) subparsers = parser.add_subparsers(dest="command", required=True) for name in COMMANDS: module = importlib.import_module(f"harness2.cli.{name}") diff --git a/harness2/cli/runs.py b/harness2/cli/runs.py index 95162298..54673993 100644 --- a/harness2/cli/runs.py +++ b/harness2/cli/runs.py @@ -16,9 +16,12 @@ from __future__ import annotations +import shutil +import sys from datetime import datetime, timezone from . import _common as C +from . import status as S def add_parser(subparsers): @@ -27,6 +30,9 @@ def add_parser(subparsers): commands.add_parser("ls", help="list runs with their progress and spend") show = commands.add_parser("show", help="show a run's settings, totals and tasks") show.add_argument("name", help="run name") + rm = commands.add_parser("rm", help="delete a run and everything it stored") + rm.add_argument("name", help="run name") + rm.add_argument("-y", "--yes", action="store_true", help="delete without asking") return p @@ -56,7 +62,10 @@ def describe(root) -> dict: def run(args, ctx) -> int: if args.action == "show": - ctx.out(describe(C.resolve_run(ctx, args.name)), str) + root = C.resolve_run(ctx, args.name) + ctx.out(describe(root), lambda data: _show(root, data)) + elif args.action == "rm": + return _rm(ctx, C.resolve_run(ctx, args.name), args.yes) else: ctx.project root = ctx.settings.path("paths.runs") @@ -83,11 +92,49 @@ def run(args, ctx) -> int: "last_activity": detail["last_activity"], } ) - ctx.out( - rows, - lambda data: "\n".join( - f"{row['name']}\t{row['suite']}\t{row['executor']}\t{row['done']}/{row['total']}\t${row['spent_usd']:.6f}\t{row['last_activity']}" - for row in data - ), + ctx.out(rows, _table) + return 0 + + +def _table(rows: list[dict]) -> str: + if not rows: + return "No runs yet; start one with `harness2 run`." + head = ("NAME", "SUITE", "EXECUTOR", "TASKS", "SPENT", "LAST ACTIVITY") + body = [ + ( + r["name"], + r["suite"], + r["executor"], + f"{r['done']}/{r['total']}", + f"${r['spent_usd']:.2f}", + S._when(r["last_activity"]), ) + for r in rows + ] + widths = [max(len(row[i]) for row in (head, *body)) for i in range(len(head))] + return "\n".join( + " ".join(cell.ljust(w) for cell, w in zip(row, widths)).rstrip() for row in (head, *body) + ) + + +def _show(root, data: dict) -> str: + settings = data["run"]["settings"] + keys = ("executor.name", "executor.model", "improver.channel", "improver.model") + lines = [S.render(S.snapshot(root)), "", "Settings:"] + lines += [f" {key} = {settings[key]}" for key in keys if key in settings] + return "\n".join(lines) + + +def _rm(ctx, root, yes: bool) -> int: + size = sum(p.stat().st_size for p in root.rglob("*") if p.is_file()) + if not yes: + if not sys.stdin.isatty(): + print("Not a terminal: pass --yes to delete.", file=sys.stderr) + return 1 + question = f"Delete run {root.name} and everything it stored ({size / 1e6:.1f} MB)? [y/N] " + if input(question).strip().lower() != "y": + ctx.log("Nothing deleted.") + return 0 + shutil.rmtree(root) + ctx.log(f"Deleted run {root.name}.") return 0 diff --git a/plugins/harness2-benchmarks/harness2_benchmarks/preflight.py b/plugins/harness2-benchmarks/harness2_benchmarks/preflight.py index 16e7f7c7..a8ad6ab3 100644 --- a/plugins/harness2-benchmarks/harness2_benchmarks/preflight.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/preflight.py @@ -177,9 +177,7 @@ def codex(*, require_bridge: bool = False) -> None: raise config.ConfigError("HARNESS2_CODEX_BASE_URL must be an HTTP(S) URL") binary = Path(config.CODEX_DIST) / "codex" if not binary.is_file(): - raise RuntimeError( - "Codex is not installed; run harness2 setup --substrate cx" - ) + raise RuntimeError("Codex is not installed; run harness2 setup --substrate cx") if require_bridge and not config.CODEX_CONTAINER_URL: raise RuntimeError( "Set HARNESS2_CODEX_CONTAINER_URL to a container-accessible Responses endpoint" diff --git a/plugins/harness2-benchmarks/pyproject.toml b/plugins/harness2-benchmarks/pyproject.toml index 5e991366..f2ffa175 100644 --- a/plugins/harness2-benchmarks/pyproject.toml +++ b/plugins/harness2-benchmarks/pyproject.toml @@ -23,7 +23,7 @@ description = "The Harness² paper benchmarks (JobBench, LAB, WorkBuddy-Bench) a readme = "README.md" license = "Apache-2.0" requires-python = ">=3.12,<3.14" -dependencies = ["harness2"] +dependencies = ["harness2-cli"] [project.optional-dependencies] render = ["openpyxl>=3.1,<4", "python-pptx>=1,<2"] diff --git a/pyproject.toml b/pyproject.toml index 0d03a308..3e143b60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,9 +3,9 @@ requires = ["setuptools>=77", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "harness2" +name = "harness2-cli" version = "0.1.0" -description = "Tune your coding agent's harness on your own tasks" +description = "General-purpose CLI for Harness²: tune any coding agent's harness on your own tasks" readme = "README.md" license = "Apache-2.0" requires-python = ">=3.12,<3.14" @@ -21,7 +21,8 @@ authors = [ dependencies = [] [project.urls] -Homepage = "https://github.com/3ae3ae/harness2" +Homepage = "https://github.com/3ae3ae/harness2-cli" +Upstream = "https://github.com/google-research/harness2" Paper = "https://arxiv.org/abs/2610.14224" [project.optional-dependencies] diff --git a/tests/test_ui.py b/tests/test_ui.py index d0d9133d..80c32028 100644 --- a/tests/test_ui.py +++ b/tests/test_ui.py @@ -132,7 +132,9 @@ def test_init_writes_a_loadable_config_once(self): fake.chmod(0o755) project = self.root / "project" project.mkdir() - args = argparse.Namespace(project=str(project), yes=True, force=False, executor=None) + args = argparse.Namespace( + project=str(project), yes=True, force=False, executor=None, auth=None + ) env = { "PATH": f"{bin_dir}{os.pathsep}{os.defpath}", "XDG_CONFIG_HOME": str(self.root / "config"), @@ -143,7 +145,7 @@ def test_init_writes_a_loadable_config_once(self): loaded = settings.load(project) self.assertEqual(loaded.get("executor.name"), "claude") - self.assertEqual(loaded.get("improver.model"), "anthropic/claude-opus-5-5") + self.assertEqual(loaded.get("improver.model"), "claude-login/claude-opus-5-5") self.assertTrue((project / ".harness2/tasks").is_dir()) self.assertIn("runs/", (project / ".harness2/.gitignore").read_text()) with patch("sys.stderr"): From 2fbae48f4048d1394655bd4224ad5c9e8dc2868e Mon Sep 17 00:00:00 2001 From: Bae Seongmin <83870598+3ae3ae@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:47:38 +0900 Subject: [PATCH 13/21] feat: run on Claude and ChatGPT subscriptions through the installed CLIs (#14) Two built-in providers, claude-login and chatgpt-login, use the login the installed `claude` / `codex` already has, so a Claude or ChatGPT subscription works without an API key: - claude-login keeps the user's own config dir (where the login lives) and adds --setting-sources project,local, --strict-mcp-config and --no-session-persistence, so personal CLAUDE.md, skills, plugins, MCP servers and session history stay out of the runs; - chatgpt-login keeps the user's own CODEX_HOME and adds --ignore-user-config and --ignore-rules (doctor notes when a global AGENTS.md is still part of every Codex run). API-key providers keep their fully isolated per-run config dirs. Doctor checks `claude auth status` / `codex login status` and rejects a login provider on a CLI that cannot use it. Hitting a provider's usage limit now stops the run cleanly instead of recording failed work: the interrupted rollout is discarded and the same command resumes later. Project worktrees left by killed runs are pruned before new ones are created. Co-authored-by: Claude Opus 5.5 --- harness2/cli/doctor.py | 14 +- harness2/core/proposer.py | 60 ++-- harness2/providers.py | 58 +++- harness2/settings.py | 2 + harness2/substrate/claude.py | 21 +- harness2/substrate/codex.py | 26 +- harness2/suite.py | 36 +- harness2/workspace.py | 7 + tests/test_providers.py | 5 +- tests/test_subscriptions.py | 643 +++++++++++++++++++++++++++++++++++ 10 files changed, 834 insertions(+), 38 deletions(-) create mode 100644 tests/test_subscriptions.py diff --git a/harness2/cli/doctor.py b/harness2/cli/doctor.py index dbd20bf2..5da655ca 100644 --- a/harness2/cli/doctor.py +++ b/harness2/cli/doctor.py @@ -16,7 +16,9 @@ from __future__ import annotations +import os import subprocess +from pathlib import Path from .. import providers, registry from ..core.proposer import Proposer @@ -79,7 +81,7 @@ def resolve(model=model, channel=channel, role=role): if channel == "claude": providers.claude_env(provider) elif channel == "codex": - providers.codex_config(provider, bare, role["effort"]) + providers.codex_args(provider, bare, role["effort"]) elif channel == "opencode": providers.opencode_config(provider, bare) if channel != "command": @@ -128,6 +130,16 @@ def binary_check(binary=binary, channel=channel): "detail": "; ".join(problems) or detail, } ) + if any(config.provider(name).type == "chatgpt-login" for name in referenced): + agents = Path(os.environ.get("CODEX_HOME") or Path.home() / ".codex") / "AGENTS.md" + if agents.is_file() and (text := agents.read_text(encoding="utf-8")): + rows.append( + { + "check": "Codex global AGENTS.md", + "ok": True, + "detail": f"your global AGENTS.md ({len(text)} characters) is part of every Codex run", + } + ) if args.live: for (channel, model), spec in sorted(models.items()): diff --git a/harness2/core/proposer.py b/harness2/core/proposer.py index a0aae876..5c2d23a4 100644 --- a/harness2/core/proposer.py +++ b/harness2/core/proposer.py @@ -35,7 +35,7 @@ from pathlib import Path from .. import providers -from ..substrate.codex import parse_items_text +from ..substrate.codex import CodexSubstrate, parse_items_text from . import events from .spec import ChannelSpec @@ -165,8 +165,11 @@ def call( tmp, mission, system, wall ) spent_usd += cost + if err and providers.USAGE_LIMIT.search(err): + events.request_stop("usage_limit") + raise events.Stopped("usage_limit") if not text.strip(): - last = err or "empty reply" + last = err[-300:] or "empty reply" continue r = Reply( label=label, @@ -193,6 +196,9 @@ def call( last = f"{type(exc).__name__}: {exc}" finally: shutil.rmtree(tmp, ignore_errors=True) + if providers.USAGE_LIMIT.search(last): + events.request_stop("usage_limit") + raise events.Stopped("usage_limit") r = Reply( label=label, @@ -214,8 +220,12 @@ def call( ) def _claude(self, cwd: Path, mission: str, system: str, wall: int) -> tuple[str, float, str]: - home = cwd / ".claude-home" - home.mkdir() + login = self.spec.provider is not None and self.spec.provider.type == "claude-login" + env = self.env() + if not login: + home = cwd / ".claude-home" + home.mkdir() + env["CLAUDE_CONFIG_DIR"] = str(home) cmd = [ *self._bin(), "--dangerously-skip-permissions", @@ -228,6 +238,7 @@ def _claude(self, cwd: Path, mission: str, system: str, wall: int) -> tuple[str, self._model(), "--setting-sources", "project,local", + *(["--strict-mcp-config", "--no-session-persistence"] if login else []), ] stdin = mission if system: @@ -243,7 +254,7 @@ def _claude(self, cwd: Path, mission: str, system: str, wall: int) -> tuple[str, cmd, input=stdin, cwd=cwd, - env=self.env() | {"CLAUDE_CONFIG_DIR": str(home)}, + env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -252,24 +263,28 @@ def _claude(self, cwd: Path, mission: str, system: str, wall: int) -> tuple[str, payload = _parse_json(p.stdout) text = str(payload.get("result") or payload.get("text") or "") err = "" - if payload.get("is_error") or payload.get("subtype") not in (None, "success"): - err = f"CLI error: {payload.get('subtype')} {str(text)[:200]}" + if ( + p.returncode + or payload.get("is_error") + or payload.get("subtype") not in (None, "success") + ): + err = f"rc={p.returncode}: {p.stdout} {p.stderr}" text = "" - elif p.returncode != 0 and not text.strip(): - err = f"rc={p.returncode}: {(p.stderr or '')[-300:]}" return text, float(payload.get("total_cost_usd") or 0.0), err def _codex(self, cwd: Path, mission: str, system: str, wall: int) -> tuple[str, float, str]: - home = cwd / ".codex-home" - home.mkdir() model = self._model() - config = ( - providers.codex_config(self.spec.provider, model, self.spec.effort) - if self.spec.provider is not None - else f"model = {json.dumps(model)}\n" - f"model_reasoning_effort = {json.dumps(self.spec.effort)}\n" - ) - (home / "config.toml").write_text(config, encoding="utf-8") + home = None + if self.spec.provider is None or self.spec.provider.type != "chatgpt-login": + home = cwd / ".codex-home" + home.mkdir() + config = ( + providers.codex_config(self.spec.provider, model, self.spec.effort) + if self.spec.provider is not None + else f"model = {json.dumps(model)}\n" + f"model_reasoning_effort = {json.dumps(self.spec.effort)}\n" + ) + (home / "config.toml").write_text(config, encoding="utf-8") p = subprocess.run( [ *self._bin(), @@ -278,13 +293,14 @@ def _codex(self, cwd: Path, mission: str, system: str, wall: int) -> tuple[str, "--skip-git-repo-check", "--dangerously-bypass-approvals-and-sandbox", "--ephemeral", + *providers.codex_args(self.spec.provider, model, self.spec.effort), "-C", str(cwd), "-", ], input=f"# SYSTEM INSTRUCTIONS\n\n{system}\n\n# END\n\n{mission}" if system else mission, cwd=cwd, - env=self.env() | {"CODEX_HOME": str(home)}, + env=CodexSubstrate().env(self.env(), home, provider=self.spec.provider), text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -299,7 +315,7 @@ def _codex(self, cwd: Path, mission: str, system: str, wall: int) -> tuple[str, if item.get("type") == "agent_message": text = str(item.get("text") or "") if errors or p.returncode: - return "", 0.0, "; ".join(errors)[-300:] or f"rc={p.returncode}: {p.stderr[-300:]}" + return "", 0.0, f"rc={p.returncode}: {p.stdout} {p.stderr}" return text, 0.0, "" def _opencode(self, cwd: Path, mission: str, system: str, wall: int) -> tuple[str, float, str]: @@ -348,10 +364,12 @@ def _opencode(self, cwd: Path, mission: str, system: str, wall: int) -> tuple[st timeout=wall, ) cost = _opencode_cost(p.stdout) + text, err = _opencode_stdout(p.stdout) + if (p.returncode or err) and providers.USAGE_LIMIT.search(p.stdout + p.stderr): + return "", cost, p.stdout + p.stderr ans = out / "answer.md" if ans.is_file() and ans.read_text(errors="replace").strip(): return ans.read_text(errors="replace"), cost, "" - text, err = _opencode_stdout(p.stdout) if text.strip(): return text, cost, "" tail = _strip_oc_banner(p.stderr or "").strip()[-300:] diff --git a/harness2/providers.py b/harness2/providers.py index 9901ef27..96053697 100644 --- a/harness2/providers.py +++ b/harness2/providers.py @@ -18,12 +18,17 @@ import json import os +import re +import subprocess from dataclasses import dataclass from urllib.parse import urlsplit from .settings import ConfigError -PROVIDER_TYPES = ("anthropic", "openai", "vertex", "bedrock") +PROVIDER_TYPES = ("anthropic", "openai", "vertex", "bedrock", "claude-login", "chatgpt-login") +USAGE_LIMIT = re.compile( + r"(?i)usage limit|rate[ _-]?limit|too many requests|\b429\b|quota|credit balance is too low" +) STRIP_ENV = ( "ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", @@ -60,6 +65,10 @@ def split_model(ref: str) -> tuple[str, str]: def claude_env(p: Provider) -> dict[str, str]: + if p.type == "claude-login": + return {} + if p.type == "chatgpt-login": + raise ConfigError("Claude Code cannot use a ChatGPT login.") if p.type == "anthropic": env = {} if p.api_key_env: @@ -104,7 +113,24 @@ def quote(value): ) +def codex_args(p: Provider | None, model: str, effort: str) -> list[str]: + if p is not None and p.type == "chatgpt-login": + return [ + "--ignore-user-config", + "--ignore-rules", + "-m", + model, + "-c", + f"model_reasoning_effort={json.dumps(effort)}", + ] + if p is not None: + codex_config(p, model, effort) + return [] + + def opencode_config(p: Provider, model: str) -> tuple[dict, dict[str, str]]: + if p.type in ("claude-login", "chatgpt-login"): + raise ConfigError("OpenCode needs an API provider; subscription logins are not supported.") if p.type == "vertex": region = p.region or "global" return {}, { @@ -135,6 +161,10 @@ def opencode_config(p: Provider, model: str) -> tuple[dict, dict[str, str]]: def litellm_params(p: Provider, model: str) -> dict: + if p.type in ("claude-login", "chatgpt-login"): + raise ConfigError( + "The bridge needs an API provider; subscription logins are not supported." + ) if p.type in ("anthropic", "openai"): params = {"model": f"{p.type}/{model}", "api_key": f"os.environ/{p.api_key_env}"} if p.base_url or p.type == "openai": @@ -152,6 +182,32 @@ def litellm_params(p: Provider, model: str) -> dict: def check(p: Provider) -> list[str]: + if p.type in ("claude-login", "chatgpt-login"): + argv = ( + ["claude", "auth", "status"] + if p.type == "claude-login" + else ["codex", "login", "status"] + ) + login = "claude auth login" if p.type == "claude-login" else "codex login" + try: + result = subprocess.run( + argv, + env={ + k: v + for k, v in os.environ.items() + if k not in STRIP_ENV + and (p.type != "chatgpt-login" or k not in ("OPENAI_API_KEY", "CODEX_API_KEY")) + }, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=10, + ) + if result.returncode == 0: + return [] + except (OSError, subprocess.SubprocessError): + pass + return [f"{p.type}: not logged in; run `{login}`"] problems = [] if p.api_key_env and not os.environ.get(p.api_key_env): problems.append(f"{p.name}: set {p.api_key_env}") diff --git a/harness2/settings.py b/harness2/settings.py index 5a1fcb1a..90f5d9a2 100644 --- a/harness2/settings.py +++ b/harness2/settings.py @@ -72,6 +72,8 @@ "base_url": "https://api.openai.com/v1", "api_key_env": "OPENAI_API_KEY", }, + "claude-login": {"type": "claude-login"}, + "chatgpt-login": {"type": "chatgpt-login"}, }, } _MISSING = object() diff --git a/harness2/substrate/claude.py b/harness2/substrate/claude.py index c23b7f43..01f2574b 100644 --- a/harness2/substrate/claude.py +++ b/harness2/substrate/claude.py @@ -197,7 +197,15 @@ def strip_native(self, workspace: Path) -> None: def install_into(self, install: Install, workspace: Path) -> None: self.export_tree(self.import_tree(install.staging_dir), workspace, base={}) - def argv(self, prompt_file: Path, workspace: Path, model: str, effort: str) -> list[str]: + def argv( + self, + prompt_file: Path, + workspace: Path, + model: str, + effort: str, + *, + provider: providers.Provider | None = None, + ) -> list[str]: return [ *cli_argv(self.bin, "claude"), "-p", @@ -211,16 +219,23 @@ def argv(self, prompt_file: Path, workspace: Path, model: str, effort: str) -> l model, "--effort", effort, + *( + ["--strict-mcp-config", "--no-session-persistence"] + if provider is not None and provider.type == "claude-login" + else [] + ), ] def env( self, base_env: Mapping[str, str], provider: providers.Provider, config_dir: Path ) -> dict[str, str]: - config_dir.mkdir(parents=True, exist_ok=True) + login = provider.type == "claude-login" + if not login: + config_dir.mkdir(parents=True, exist_ok=True) return { **{k: v for k, v in base_env.items() if k not in providers.STRIP_ENV}, **providers.claude_env(provider), - "CLAUDE_CONFIG_DIR": str(config_dir), + **({} if login else {"CLAUDE_CONFIG_DIR": str(config_dir)}), } def read_trajectory( diff --git a/harness2/substrate/codex.py b/harness2/substrate/codex.py index c555b230..2b8518fb 100644 --- a/harness2/substrate/codex.py +++ b/harness2/substrate/codex.py @@ -210,7 +210,15 @@ def strip_native(self, workspace: Path) -> None: def install_into(self, install: Install, workspace: Path) -> None: self.export_tree(self.import_tree(install.staging_dir), workspace, base={}) - def argv(self, prompt_file: Path, workspace: Path, model: str, effort: str) -> list[str]: + def argv( + self, + prompt_file: Path, + workspace: Path, + model: str, + effort: str, + *, + provider: providers.Provider | None = None, + ) -> list[str]: return [ *cli_argv(self.bin, "codex"), "exec", @@ -219,6 +227,7 @@ def argv(self, prompt_file: Path, workspace: Path, model: str, effort: str) -> l "--dangerously-bypass-approvals-and-sandbox", "--dangerously-bypass-hook-trust", "--ephemeral", + *providers.codex_args(provider, model, effort), "-C", str(workspace), "-", @@ -229,10 +238,19 @@ def write_home(self, home: Path, provider: providers.Provider, model: str, effor home.mkdir(parents=True, exist_ok=True) (home / "config.toml").write_text(content, encoding="utf-8") - def env(self, base_env: Mapping[str, str], home: Path) -> dict[str, str]: + def env( + self, + base_env: Mapping[str, str], + home: Path | None, + *, + provider: providers.Provider | None = None, + ) -> dict[str, str]: + stripped = providers.STRIP_ENV + if provider is not None and provider.type == "chatgpt-login": + stripped += ("OPENAI_API_KEY", "CODEX_API_KEY") return { - **{k: v for k, v in base_env.items() if k not in providers.STRIP_ENV}, - "CODEX_HOME": str(home), + **{k: v for k, v in base_env.items() if k not in stripped}, + **({"CODEX_HOME": str(home)} if home is not None else {}), } def final_message(self, stream_path: Path) -> str: diff --git a/harness2/suite.py b/harness2/suite.py index 2143948e..99af3eab 100644 --- a/harness2/suite.py +++ b/harness2/suite.py @@ -32,14 +32,14 @@ from pathlib import Path from . import providers, workspace -from .core import runmark, tree +from .core import events, runmark, tree from .core.library import slug from .core.proposer import Proposer, ProposerError, _parse_json from .core.spec import BenchPrompts, BenchSpec, ChannelSpec from .registry import SuiteContext from .settings import ConfigError, Settings from .substrate._native import checked_path, cli_argv -from .substrate.claude import ClaudeSubstrate +from .substrate.claude import ClaudeSubstrate, parse_events from .substrate.codex import CodexSubstrate from .substrate.command import CommandSubstrate @@ -450,10 +450,14 @@ def execute( if self.executor["name"] == "claude": env = self.substrate.env(env, provider, run_dir / "claude-home") else: - home = run_dir / "codex-home" - self.substrate.write_home(home, provider, model, self.executor["effort"]) - env = self.substrate.env(env, home) - argv = self.substrate.argv(prompt_file, ws, model, self.executor["effort"]) + home = None + if provider.type != "chatgpt-login": + home = run_dir / "codex-home" + self.substrate.write_home(home, provider, model, self.executor["effort"]) + env = self.substrate.env(env, home, provider=provider) + argv = self.substrate.argv( + prompt_file, ws, model, self.executor["effort"], provider=provider + ) try: if self.executor["name"] == "command": exit_code = self.substrate.run(prompt_file, ws, run_dir, wall, env) @@ -477,6 +481,25 @@ def execute( except OSError as exc: (run_dir / "stderr.log").write_text(str(exc), encoding="utf-8") exit_code = -1 + stream = run_dir / "stream.jsonl" + failed = exit_code != 0 or any( + event.get("type") in ("error", "turn.failed") + or ( + event.get("type") == "result" + and (event.get("is_error") or event.get("subtype") not in (None, "success")) + ) + for event in parse_events(stream) + ) + if failed: + tail = "".join( + path.read_text(encoding="utf-8", errors="replace")[-16000:] + for path in (stream, run_dir / "stderr.log") + if path.is_file() + ) + if providers.USAGE_LIMIT.search(tail): + shutil.rmtree(run_dir) + events.request_stop("usage_limit") + raise events.Stopped("usage_limit") if config["workspace"] == "project": (run_dir / "deliverable.patch").write_text( workspace.diff_since(ws, base, (*self.substrate.NATIVE_PATHS, ".harness2")), @@ -484,7 +507,6 @@ def execute( ) if (ws / "output").is_dir(): shutil.copytree(ws / "output", run_dir / "output") - stream = run_dir / "stream.jsonl" (run_dir / "final.md").write_text( self.substrate.final_message(stream), encoding="utf-8" ) diff --git a/harness2/workspace.py b/harness2/workspace.py index 51b46414..e2de6a24 100644 --- a/harness2/workspace.py +++ b/harness2/workspace.py @@ -30,6 +30,13 @@ def project_workspace(project: Path, ref: str) -> Iterator[Path]: ws = Path(tempfile.mkdtemp(prefix="harness2-ws-")).resolve() try: + pruned = subprocess.run( + ["git", "-C", str(project), "worktree", "prune"], + capture_output=True, + text=True, + ) + if pruned.returncode: + raise ConfigError(f"cannot prune project worktrees: {pruned.stderr.strip()}") result = subprocess.run( ["git", "-C", str(project), "worktree", "add", "--detach", "--", str(ws), ref], capture_output=True, diff --git a/tests/test_providers.py b/tests/test_providers.py index 96c5efc7..a58f67f7 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -43,7 +43,10 @@ def test_provider_is_frozen_and_strip_env_is_exact(self): provider = Provider("custom", "anthropic") with self.assertRaises(FrozenInstanceError): provider.region = "changed" - self.assertEqual(providers.PROVIDER_TYPES, ("anthropic", "openai", "vertex", "bedrock")) + self.assertEqual( + providers.PROVIDER_TYPES, + ("anthropic", "openai", "vertex", "bedrock", "claude-login", "chatgpt-login"), + ) self.assertEqual( providers.STRIP_ENV, ( diff --git a/tests/test_subscriptions.py b/tests/test_subscriptions.py new file mode 100644 index 00000000..4adfcc1f --- /dev/null +++ b/tests/test_subscriptions.py @@ -0,0 +1,643 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Subscription isolation and resumable usage limits against offline CLIs.""" + +from __future__ import annotations + +import contextlib +import io +import json +import os +import subprocess +import sys +import tempfile +import time +import tomllib +import unittest +from pathlib import Path +from unittest.mock import Mock, patch + +from harness2 import providers, settings, suite, workspace +from harness2.cli.main import main +from harness2.core import events, recursion, runmark, schedule +from harness2.core.proposer import Proposer +from harness2.core.spec import ChannelSpec +from harness2.registry import SuiteContext +from harness2.settings import ConfigError + +FAKE_CLI = r"""import json, os, sys +from pathlib import Path + +argv = sys.argv[1:] +kind = Path(sys.argv[0]).name +if "--version" in argv: + print("fixture 1.0") + sys.exit(0) +status = argv in (["login", "status"], ["auth", "status"]) +cwd = Path.cwd() +home = Path(os.environ.get("CODEX_HOME", Path.home() / ".codex")) +config = home / "config.toml" +record = {"kind": kind, "argv": argv, "env": dict(os.environ), "cwd": str(cwd), + "stdin": "" if status else sys.stdin.read(), + "config_exists": config.exists(), + "config": config.read_text() if config.exists() else None} +with open(os.environ["FAKE_RECORD"], "a") as log: + log.write(json.dumps(record) + "\n") +if status: + sys.exit(int(os.environ.get("FAKE_LOGIN_RC", "0"))) +improver = ("--output-format" in argv and argv[argv.index("--output-format") + 1] == "json" + or (cwd / "context").is_dir()) +role = "improver" if improver else "executor" +if os.environ.get("LIMIT_ROLE") == role: + limit = os.environ.get("LIMIT_TEXT", "rate-limit reached") + source = os.environ.get("LIMIT_SOURCE", "event") + if source == "event": + if kind == "claude": + print(json.dumps({"type": "result", "subtype": "error_during_execution", + "is_error": True, "errors": [limit]})) + else: + print(json.dumps({"type": "turn.failed", "error": {"message": limit}})) + sys.exit(0) + print(limit, file=sys.stderr if source == "stderr" else sys.stdout) + sys.exit(7) +if improver: + context = cwd / "context" + if (context / "candidates").is_dir(): + chosen = sorted((context / "candidates").glob("*.md"))[0].stem + text = "Complete.\n```choice\n" + chosen + "\n```" + elif (context / "adoption.json").is_file(): + text = "NO_ENTRY" + else: + for name in ("CLAUDE.md", "AGENTS.md"): + harness = context / "harness" / name + if harness.is_file(): + harness.write_text(harness.read_text() + "Check the output.\n") + text = "Check the output." +else: + (cwd / "output").mkdir(exist_ok=True) + (cwd / "output/report.txt").write_text("done\n") + text = "done" +if kind == "claude": + print(json.dumps({"type": "result", "subtype": "success", "result": text, + "total_cost_usd": 0.01})) +else: + print(json.dumps({"type": "item.completed", "item": { + "id": "reply", "type": "agent_message", "text": text}})) +""" + + +class SubscriptionTests(unittest.TestCase): + maxDiff = None + + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.project = self.root / "project" + self.tasks = self.project / ".harness2/tasks" + self.tasks.mkdir(parents=True) + (self.project / "CLAUDE.md").write_text("Read the task.\n") + (self.project / "AGENTS.md").write_text("Read the task.\n") + binary = self.root / "bin" + binary.mkdir() + for name in ("claude", "codex"): + path = binary / name + path.write_text(f"#!{sys.executable}\n" + FAKE_CLI) + path.chmod(0o755) + self.record = self.root / "records.jsonl" + self.user_home = self.root / "user" + for name in (".codex", ".claude"): + path = self.user_home / name + path.mkdir(parents=True) + (path / "config.toml").write_text("private user config") + (path / "auth.json").write_text("private user auth") + environment = patch.dict( + os.environ, + { + "PATH": f"{binary}{os.pathsep}{os.defpath}", + "HOME": str(self.user_home), + "XDG_CONFIG_HOME": str(self.root / "config"), + "FAKE_RECORD": str(self.record), + "CLAUDE_CODE_OAUTH_TOKEN": "fixture-oauth", + "ANTHROPIC_API_KEY": "ambient-anthropic", + "OPENAI_API_KEY": "ambient-openai", + "CODEX_API_KEY": "ambient-codex", + "ANTHROPIC_BASE_URL": "https://ambient.invalid", + "OPENAI_BASE_URL": "https://ambient.invalid", + "PYTHONDONTWRITEBYTECODE": "1", + }, + clear=True, + ) + environment.start() + self.addCleanup(environment.stop) + events.start(None) + self.addCleanup(events.start, None) + task = suite.new_task(self.tasks, "one", files=True, domain="coding", check="true") + (task / "task.md").write_text("Produce a report.") + + def adapter(self, name="test", **overrides): + values = { + "executor.model": "claude-login/claude-sonnet-5-5", + "improver.model": "claude-login/claude-sonnet-5-5", + } + values.update(overrides) + config = settings.load( + self.project, overrides={key: (value, "test") for key, value in values.items()} + ) + return suite.make_adapter( + SuiteContext(config, self.project, self.project / ".harness2/runs" / name) + ) + + def records(self): + return [json.loads(line) for line in self.record.read_text().splitlines()] + + def launch_record(self, label): + record = self.records()[-1] + env = dict(record["env"]) + env.pop("__CF_USER_TEXT_ENCODING", None) + normalized = json.dumps( + {"argv": [record["kind"], *record["argv"]], "env": env}, sort_keys=True + ) + normalized = normalized.replace(record["cwd"], "") + normalized = normalized.replace(str(self.root), "") + print(f"SUBSCRIPTION {label}: {normalized}") + return record + + def test_builtin_providers_use_cli_auth(self): + config = settings.load(self.project) + for name in ("claude-login", "chatgpt-login"): + self.assertEqual(config.provider(name).type, name) + self.assertEqual(providers.claude_env(config.provider("claude-login")), {}) + with patch.dict(os.environ, {}, clear=True): + self.assertEqual(providers.claude_env(config.provider("claude-login")), {}) + + def test_claude_subscription_executor_and_improver_launches(self): + adapter = self.adapter() + result = adapter.execute("one", None, "claude") + record = self.launch_record("claude-login executor") + self.assertEqual( + record["argv"], + [ + "-p", + "--output-format", + "stream-json", + "--verbose", + "--dangerously-skip-permissions", + "--setting-sources", + "project,local", + "--model", + "claude-sonnet-5-5", + "--effort", + "high", + "--strict-mcp-config", + "--no-session-persistence", + ], + ) + self.assertFalse((result / "claude-home").exists()) + context = self.root / "context" + context.mkdir() + Proposer(adapter.spec.channels["proposer"]).call("mission", context, "improver") + record = self.launch_record("claude-login improver") + self.assertEqual( + record["argv"], + [ + "--dangerously-skip-permissions", + "-p", + "--output-format", + "json", + "--effort", + "high", + "--model", + "claude-sonnet-5-5", + "--setting-sources", + "project,local", + "--strict-mcp-config", + "--no-session-persistence", + ], + ) + for record in self.records(): + self.assertNotIn("CLAUDE_CONFIG_DIR", record["env"]) + self.assertEqual(record["env"]["CLAUDE_CODE_OAUTH_TOKEN"], "fixture-oauth") + self.assertTrue(all(key not in record["env"] for key in providers.STRIP_ENV)) + os.environ.pop("CLAUDE_CODE_OAUTH_TOKEN") + adapter.execute("one", None, "native-login") + self.assertNotIn("CLAUDE_CODE_OAUTH_TOKEN", self.records()[-1]["env"]) + self.assertNotIn("CLAUDE_CONFIG_DIR", self.records()[-1]["env"]) + + def test_chatgpt_subscription_executor_and_improver_launches(self): + adapter = self.adapter( + **{ + "executor.name": "codex", + "executor.model": "chatgpt-login/gpt-6.1-sol", + "improver.channel": "codex", + "improver.model": "chatgpt-login/gpt-6.1-sol", + } + ) + home = self.user_home / ".codex" + before = {p.name: p.read_bytes() for p in home.iterdir()} + result = adapter.execute("one", None, "codex") + record = self.launch_record("chatgpt-login executor") + self.assertEqual( + record["argv"], + [ + "exec", + "--json", + "--skip-git-repo-check", + "--dangerously-bypass-approvals-and-sandbox", + "--dangerously-bypass-hook-trust", + "--ephemeral", + "--ignore-user-config", + "--ignore-rules", + "-m", + "gpt-6.1-sol", + "-c", + 'model_reasoning_effort="high"', + "-C", + record["cwd"], + "-", + ], + ) + self.assertFalse((result / "codex-home").exists()) + context = self.root / "context" + context.mkdir() + Proposer(adapter.spec.channels["proposer"]).call("mission", context, "improver") + record = self.launch_record("chatgpt-login improver") + self.assertEqual( + record["argv"], + [ + "exec", + "--json", + "--skip-git-repo-check", + "--dangerously-bypass-approvals-and-sandbox", + "--ephemeral", + "--ignore-user-config", + "--ignore-rules", + "-m", + "gpt-6.1-sol", + "-c", + 'model_reasoning_effort="high"', + "-C", + record["cwd"], + "-", + ], + ) + for record in self.records(): + self.assertNotIn("CODEX_HOME", record["env"]) + self.assertTrue( + all( + key not in record["env"] + for key in (*providers.STRIP_ENV, "OPENAI_API_KEY", "CODEX_API_KEY") + ) + ) + self.assertEqual({p.name: p.read_bytes() for p in home.iterdir()}, before) + + def test_inherited_login_homes_are_preserved(self): + context = self.root / "context" + context.mkdir() + for kind, provider_name, variable in ( + ("claude", "claude-login", "CLAUDE_CONFIG_DIR"), + ("codex", "chatgpt-login", "CODEX_HOME"), + ): + with ( + self.subTest(kind=kind), + patch.dict(os.environ, {variable: str(self.user_home / ("." + kind))}), + ): + adapter = self.adapter( + kind, + **{ + "executor.name": kind, + "executor.model": provider_name + "/model", + "improver.channel": kind, + "improver.model": provider_name + "/model", + }, + ) + adapter.execute("one", None, "inherited") + self.assertEqual(self.records()[-1]["env"][variable], os.environ[variable]) + Proposer(adapter.spec.channels["proposer"]).call("mission", context, "inherited") + self.assertEqual(self.records()[-1]["env"][variable], os.environ[variable]) + + def test_login_status_success_failure_missing_cli_and_timeout(self): + for kind, argv, login in ( + ("claude-login", ["auth", "status"], "claude auth login"), + ("chatgpt-login", ["login", "status"], "codex login"), + ): + with self.subTest(provider=kind): + provider = providers.Provider(kind, kind) + self.assertEqual(providers.check(provider), []) + record = self.records()[-1] + self.assertEqual(record["argv"], argv) + self.assertNotIn("CLAUDE_CONFIG_DIR", record["env"]) + self.assertNotIn("CODEX_HOME", record["env"]) + self.assertTrue(all(key not in record["env"] for key in providers.STRIP_ENV)) + problem = [f"{kind}: not logged in; run `{login}`"] + with patch.dict(os.environ, {"FAKE_LOGIN_RC": "1"}): + self.assertEqual(providers.check(provider), problem) + with patch.dict(os.environ, {"PATH": ""}): + self.assertEqual(providers.check(provider), problem) + with patch( + "harness2.providers.subprocess.run", + side_effect=subprocess.TimeoutExpired("cli", 10), + ): + self.assertEqual(providers.check(provider), problem) + args = providers.codex_args( + providers.Provider("chatgpt-login", "chatgpt-login"), + 'model"\nnext\x7f', + 'effort"\nnext', + ) + self.assertEqual(args[3], 'model"\nnext\x7f') + self.assertEqual(tomllib.loads(args[5]), {"model_reasoning_effort": 'effort"\nnext'}) + + def test_api_provider_homes_and_ambient_oauth_are_unchanged(self): + for kind, provider_name in (("claude", "anthropic"), ("codex", "openai")): + with self.subTest(kind=kind): + adapter = self.adapter( + kind, + **{ + "executor.name": kind, + "executor.model": provider_name + "/fixture", + "improver.channel": kind, + "improver.model": provider_name + "/fixture", + }, + ) + result = adapter.execute("one", None, "api") + record = self.records()[-1] + self.assertEqual(record["env"]["CLAUDE_CODE_OAUTH_TOKEN"], "fixture-oauth") + if kind == "claude": + self.assertEqual(record["env"]["ANTHROPIC_API_KEY"], "ambient-anthropic") + else: + self.assertEqual(record["env"]["OPENAI_API_KEY"], "ambient-openai") + self.assertEqual(record["env"]["CODEX_HOME"], str(result / "codex-home")) + self.assertEqual( + record["config"], + providers.codex_config( + adapter.settings.provider("openai"), "fixture", "high" + ), + ) + self.assertNotIn("-c", record["argv"]) + Proposer(adapter.spec.channels["proposer"]).call("mission", None, "api-improver") + record = self.records()[-1] + self.assertEqual(record["env"]["CLAUDE_CODE_OAUTH_TOKEN"], "fixture-oauth") + if kind == "codex": + self.assertEqual( + record["config"], + providers.codex_config( + adapter.settings.provider("openai"), "fixture", "high" + ), + ) + self.assertNotEqual(record["env"]["CODEX_HOME"], str(self.user_home / ".codex")) + + def test_wrong_cli_and_bridge_reject_subscription_providers(self): + claude = providers.Provider("claude-login", "claude-login") + chatgpt = providers.Provider("chatgpt-login", "chatgpt-login") + with self.assertRaisesRegex(ConfigError, "cannot use a ChatGPT login"): + providers.claude_env(chatgpt) + with self.assertRaises(ConfigError): + providers.codex_config(claude, "model", "high") + with self.assertRaises(ConfigError): + providers.codex_args(claude, "model", "high") + for provider in (claude, chatgpt): + for operation in (providers.opencode_config, providers.litellm_params): + with self.subTest(provider=provider.type, operation=operation.__name__): + with self.assertRaisesRegex(ConfigError, "API provider"): + operation(provider, "model") + + def test_doctor_per_role_subscription_compatibility(self): + for channel, provider_name in ( + ("claude", "claude-login"), + ("codex", "chatgpt-login"), + ("claude", "chatgpt-login"), + ("codex", "claude-login"), + ("opencode", "claude-login"), + ("opencode", "chatgpt-login"), + ): + with self.subTest(channel=channel, provider=provider_name): + adapter = ( + self.adapter( + **{ + "executor.name": channel, + "executor.model": provider_name + "/model", + "improver.channel": channel, + "improver.model": provider_name + "/model", + } + ) + if channel != "opencode" + else self.adapter( + **{ + "improver.channel": channel, + "improver.model": provider_name + "/model", + } + ) + ) + code, roles = self.doctor(adapter) + compatible = (channel, provider_name) in ( + ("claude", "claude-login"), + ("codex", "chatgpt-login"), + ) + self.assertEqual(code, 0 if compatible else 1) + self.assertEqual(roles["improver"]["ok"], compatible) + self.assertEqual(roles["judge"]["ok"], compatible) + + def doctor(self, adapter): + output = io.StringIO() + argv = ["-C", str(self.project), "--json"] + for key, value in adapter.settings.flat().items(): + if key.startswith(("executor.", "improver.")) and not isinstance(value, list): + argv += ["-c", f"{key}={json.dumps(value)}"] + with contextlib.redirect_stdout(output): + code = main([*argv, "doctor"]) + return code, {row["check"]: row for row in json.loads(output.getvalue())} + + def test_doctor_login_status_and_global_codex_instructions(self): + adapter = self.adapter( + **{ + "executor.name": "codex", + "executor.model": "chatgpt-login/model", + "improver.channel": "codex", + "improver.model": "chatgpt-login/model", + } + ) + agents = self.user_home / ".codex/AGENTS.md" + agents.write_text("Global rules. 한글\n") + code, rows = self.doctor(adapter) + self.assertEqual(code, 0) + self.assertEqual( + rows["Codex global AGENTS.md"], + { + "check": "Codex global AGENTS.md", + "ok": True, + "detail": "your global AGENTS.md (17 characters) is part of every Codex run", + }, + ) + with patch.dict(os.environ, {"FAKE_LOGIN_RC": "1"}): + code, rows = self.doctor(adapter) + self.assertEqual(code, 1) + self.assertEqual( + rows["provider chatgpt-login"]["detail"], + "chatgpt-login: not logged in; run `codex login`", + ) + code, rows = self.doctor(self.adapter()) + self.assertEqual(code, 1) + self.assertEqual( + rows["provider claude-login"]["detail"], + "claude-login: not logged in; run `claude auth login`", + ) + _, rows = self.doctor(self.adapter()) + self.assertNotIn("Codex global AGENTS.md", rows) + agents.write_text("") + _, rows = self.doctor(adapter) + self.assertNotIn("Codex global AGENTS.md", rows) + other = self.root / "custom-codex" + other.mkdir() + (other / "AGENTS.md").write_text("custom") + with patch.dict(os.environ, {"CODEX_HOME": str(other)}): + _, rows = self.doctor(adapter) + self.assertEqual( + rows["Codex global AGENTS.md"]["detail"], + "your global AGENTS.md (6 characters) is part of every Codex run", + ) + self.assertTrue( + all( + record["argv"] in (["auth", "status"], ["login", "status"]) + for record in self.records() + ) + ) + + def test_executor_usage_limits_stop_and_resume_entire_run(self): + adapter = self.adapter() + self.stop_and_resume(adapter, "executor") + + def test_improver_usage_limits_stop_without_retry_and_resume_entire_run(self): + adapter = self.adapter() + self.stop_and_resume(adapter, "improver") + + def stop_and_resume(self, adapter, role): + run_root = adapter.spec.runs_root + event_path = run_root / "events.jsonl" + prop = Proposer(adapter.spec.channels["proposer"], audit_dir=run_root / "audit") + kwargs = dict( + adapter=adapter, + prop=prop, + runs_root=run_root, + library_root=run_root / "library", + modes=("agg",), + ) + events.start(event_path) + with ( + patch.dict(os.environ, {"LIMIT_ROLE": role}), + patch("harness2.core.proposer.time", wraps=time) as clock, + ): + with self.assertRaisesRegex(events.Stopped, "^usage_limit$"): + schedule.run_sweep(1, **kwargs) + clock.sleep.assert_not_called() + self.assertEqual(events.totals()["stopped"], "usage_limit") + self.assertFalse(list(run_root.glob("tasks-*/**/summary.json"))) + if role == "executor": + self.assertEqual(list((run_root / "rollouts").iterdir()), []) + self.assertEqual(events.totals()["calls"], 0) + else: + self.assertFalse(list((run_root / "audit").glob("*.response.md"))) + events.start(event_path) + resumed = schedule.run_sweep(1, **kwargs) + self.assertEqual((resumed["domains"][0]["done"], resumed["domains"][0]["failed"]), (1, 0)) + self.assertTrue(list(run_root.glob("tasks-*/**/summary.json"))) + rollouts = list((run_root / "rollouts").iterdir()) + self.assertTrue(rollouts) + self.assertTrue(all(runmark.is_complete(path, match_harness=False) for path in rollouts)) + + def test_usage_limit_failure_sources_and_success_mentions(self): + for source in ("event", "stderr", "stdout"): + for kind in ("claude", "codex"): + with self.subTest(source=source, kind=kind): + provider_name = "claude-login" if kind == "claude" else "chatgpt-login" + adapter = self.adapter( + f"{kind}-{source}", + **{ + "executor.name": kind, + "executor.model": provider_name + "/model", + }, + ) + events.start(None) + with patch.dict(os.environ, {"LIMIT_ROLE": "executor", "LIMIT_SOURCE": source}): + with self.assertRaisesRegex(events.Stopped, "usage_limit"): + adapter.execute("one", None, "limited") + self.assertEqual(list((adapter.spec.runs_root / "rollouts").iterdir()), []) + events.start(None) + provider = adapter.settings.provider(provider_name) + with patch.dict(os.environ, {"LIMIT_ROLE": "improver", "LIMIT_SOURCE": source}): + context = self.root / "context" + context.mkdir(exist_ok=True) + with self.assertRaisesRegex(events.Stopped, "usage_limit"): + Proposer( + ChannelSpec(kind, provider_name + "/model", provider=provider) + ).call("mission", context, "limited") + for text in ( + "usage limit", + "RATE_LIMIT", + "too many requests", + "HTTP 429", + "quota", + "credit balance is too low", + ): + self.assertIsNotNone(providers.USAGE_LIMIT.search(text)) + self.assertIsNone(providers.USAGE_LIMIT.search("14290")) + events.start(None) + adapter = self.adapter("success") + (self.root / "bin/claude").write_text( + f"#!{sys.executable}\nimport json\n" + "print(json.dumps({'type':'result', 'subtype':'success', 'result':'rate limit fixed'}))\n" + ) + self.assertTrue( + runmark.is_complete( + adapter.execute("one", None, "success"), harness_dir=adapter.base_harness() + ) + ) + self.assertTrue( + Proposer(adapter.spec.channels["proposer"]).call("mission", None, "success") + ) + + def test_core_does_not_swallow_stopped_from_batched_adapter(self): + adapter = self.adapter() + adapter.execute_many = Mock(side_effect=events.Stopped("usage_limit")) + with self.assertRaisesRegex(events.Stopped, "usage_limit"): + schedule._roll_bases("one", adapter, [("a", "a"), ("b", "b")], self.root / "dead") + with patch.object(adapter, "execute", side_effect=events.Stopped("usage_limit")): + with self.assertRaisesRegex(events.Stopped, "usage_limit"): + recursion._roll(adapter, None, "one", "a", None, lambda label: label, self.root) + + def test_opencode_usage_limit_stops_even_with_a_partial_answer(self): + binary = self.root / "bin/opencode" + binary.write_text( + f"#!{sys.executable}\nimport json\nfrom pathlib import Path\n" + "Path('output/answer.md').write_text('partial')\n" + "print(json.dumps({'type':'error','error':{'data':{'message':'quota exceeded'}}}))\n" + ) + binary.chmod(0o755) + provider = providers.Provider("api", "openai") + prop = Proposer(ChannelSpec("opencode", "api/model", provider=provider)) + with patch("harness2.core.proposer.time", wraps=time) as clock: + with self.assertRaisesRegex(events.Stopped, "usage_limit"): + prop.call("mission", None, "limit") + clock.sleep.assert_not_called() + self.assertEqual(events.totals()["stopped"], "usage_limit") + + def test_project_workspace_prunes_before_adding(self): + subprocess.run(["git", "init", "-q", str(self.project)], check=True) + workspace.commit_all(self.project, "fixture") + with patch("harness2.workspace.subprocess.run", wraps=subprocess.run) as run: + with workspace.project_workspace(self.project, "HEAD") as ws: + self.assertTrue(ws.exists()) + argv = [call.args[0] for call in run.call_args_list] + self.assertEqual(argv[0], ["git", "-C", str(self.project), "worktree", "prune"]) + self.assertEqual(argv[1][4], "add") From e0d9290e07db7ad9160ab8761f6dba09e4e5213b Mon Sep 17 00:00:00 2001 From: Bae Seongmin <83870598+3ae3ae@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:48:50 +0900 Subject: [PATCH 14/21] feat: build the run's final harness automatically (#15) A run now ends by combining the harnesses it adopted per task, with the lessons it recorded, into one final harness per recursion mode; that is what `apply`, `diff` and `eval` use, so there is a single result to review and apply. The separate `consolidate` command is gone. If the final step fails, the run's exit code is unchanged and rerunning the same command retries just that step (every earlier step is reused). The dry-run estimate counts the extra session, and a run stopped by a provider's usage limit says to rerun once the limit resets. Co-authored-by: Claude Opus 5.5 --- harness2/cli/_final.py | 136 +++++++++++++++++++++++++++ harness2/cli/consolidate.py | 178 ------------------------------------ harness2/cli/eval.py | 2 +- harness2/cli/main.py | 1 - harness2/cli/run.py | 31 ++++++- tests/test_cli.py | 27 ++++-- tests/test_consolidate.py | 138 ++++++++++++++++++---------- tests/test_e2e.py | 8 +- 8 files changed, 278 insertions(+), 243 deletions(-) create mode 100644 harness2/cli/_final.py delete mode 100644 harness2/cli/consolidate.py diff --git a/harness2/cli/_final.py b/harness2/cli/_final.py new file mode 100644 index 00000000..9a0efd93 --- /dev/null +++ b/harness2/cli/_final.py @@ -0,0 +1,136 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compose a project harness from a run's successful task selections.""" + +from __future__ import annotations + +import json +import re +import shutil +from datetime import datetime, timezone +from pathlib import Path + +from ..core import library, prompts, recursion, schedule, tree +from ..core.proposer import Proposer +from ..settings import ConfigError +from . import _adopt as A +from . import _common as C + + +def _context( + adapter, root: Path, record: dict, mode: str, dest: Path, base: tree.Tree +) -> list[str]: + surface = adapter.substrate.surface() + token = schedule.mode_token(mode) + plan = record["schedule"] + namespace = C.summary_namespace(adapter, plan["k"], seed=plan.get("seed")) + selected = { + row["task_id"] + for path, row in C.summaries(root) + if path.parent.parent.name == namespace + and (selection := (row.get("selections") or {}).get(token) or {}).get("ok") + and selection.get("chosen_label") + } + tasks, index = [], [] + tree.write_tree(surface, base, dest / "harness", stubs=True) + lib = C.library_root_for(adapter, plan["k"], seed=plan.get("seed")) / token + for domain, task in record["stream"]: + if task not in selected: + continue + _, adopted = A.adopted_tree(adapter, root, task, mode) + task_dir = dest / "tasks" / library.slug(task) + tree.write_tree(surface, adopted, task_dir / "harness", stubs=False) + (task_dir / "task.md").write_text(adapter.task_prompt(task), encoding="utf-8") + lesson = ( + lib / library.domain_dir(domain) / "harnesses" / library.slug(task) / "EXPERIENCE.md" + ) + if lesson.is_file(): + (task_dir / "EXPERIENCE.md").write_text(lesson.read_text(), encoding="utf-8") + index.append( + f"- {task} ({domain}): adopted harness " + f"{'differs from' if adopted != base else 'same as'} the start; " + f"lesson: {'yes' if lesson.is_file() else 'no'}" + ) + tasks.append(task) + if not tasks: + raise ConfigError(f"no successful selection for any task in {mode}") + (dest / "INDEX.md").write_text("\n".join(index) + "\n", encoding="utf-8") + return tasks + + +def consolidate(ctx, adapter, root: Path, mode: str) -> dict: + mode = C.mode_label(schedule.mode_token(mode)) + with ctx.progress(): + base = A.base_tree(adapter) + record = C.load_run(root) + surface = adapter.substrate.surface() + prop = Proposer(adapter.spec.channels["proposer"], audit_dir=root / "consolidated/_audit") + cost = 0.0 + # The context path participates in the proposer's cache key. + context = root / "consolidated/_audit" / f"context.{mode}" + if context.exists(): + shutil.rmtree(context) + try: + tasks = _context(adapter, root, record, mode, context, base) + mission = prompts.CONSOLIDATE_MISSION.replace("{n}", str(len(tasks))).replace( + "{surface}", prompts._st(surface).surface_md + ) + err = "" + for _ in range(2): + with ctx.progress(): + reply = prop.call( + mission + + (f"\n\n# YOUR PREVIOUS ATTEMPT WAS REJECTED\n\n{err}\n\n" if err else ""), + context, + f"consolidate.{mode}", + harvest="context/harness", + ) + cost += reply.cost_usd + if reply.workspace is None: + err = "the harness directory did not come back from the session" + continue + rejected = [] + new = tree.read_tree(surface, reply.workspace, rejected=rejected) + _, err = recursion._accept_tree(surface, adapter, new, base, "consolidate") + if rejected: + err = "files outside the editable surface: " + ", ".join(sorted(rejected)) + if not err: + break + tree.write_tree(surface, new, context / "harness", stubs=True) + else: + if reply.workspace is not None: + shutil.rmtree(reply.workspace) + raise ConfigError(err) + finally: + shutil.rmtree(context, ignore_errors=True) + notes = re.search(r"(.*?)", reply.text, re.S) + notes = notes.group(1).strip() if notes else reply.text.strip() + dest = root / "consolidated" / mode + path = tree.write_tree(surface, new, dest / "harness", stubs=False) + (dest / "notes.md").write_text(notes + "\n", encoding="utf-8") + (dest / "consolidation.json").write_text( + json.dumps( + { + "tasks": tasks, + "model": prop.spec.model, + "cost_usd": cost, + "created": datetime.now(timezone.utc).isoformat(), + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + return {"path": path, "tasks": tasks, "changes": A.changes(base, new), "notes": notes} diff --git a/harness2/cli/consolidate.py b/harness2/cli/consolidate.py deleted file mode 100644 index 47897bab..00000000 --- a/harness2/cli/consolidate.py +++ /dev/null @@ -1,178 +0,0 @@ -# Copyright 2026 The Harness2 Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Compose a project harness from a run's successful task selections.""" - -from __future__ import annotations - -import json -import re -import tempfile -from datetime import datetime, timezone -from pathlib import Path - -from ..core import events, library, prompts, recursion, schedule, tree -from ..core.proposer import Proposer -from ..settings import ConfigError -from . import _adopt as A -from . import _common as C -from .run import _stopped - - -def add_parser(subparsers): - p = subparsers.add_parser( - "consolidate", - help="combine a run's adopted harnesses into one for the project", - description="A run adopts one harness per task. Combine them, with the lessons the run " - "recorded, into one harness for the project's next tasks. apply and diff then use it " - "by default. Makes one improver session.", - ) - p.add_argument("name", nargs="?", help="run name (default: the most recent run)") - p.add_argument( - "--mode", - choices=("parallel", "sequential"), - default="parallel", - help="take the harnesses adopted in this recursion mode", - ) - return p - - -def _context( - adapter, root: Path, record: dict, mode: str, dest: Path, base: tree.Tree -) -> list[str]: - surface = adapter.substrate.surface() - token = schedule.mode_token(mode) - plan = record["schedule"] - namespace = C.summary_namespace(adapter, plan["k"], seed=plan.get("seed")) - selected = { - row["task_id"] - for path, row in C.summaries(root) - if path.parent.parent.name == namespace - and (selection := (row.get("selections") or {}).get(token) or {}).get("ok") - and selection.get("chosen_label") - } - tasks, index = [], [] - tree.write_tree(surface, base, dest / "harness", stubs=True) - lib = C.library_root_for(adapter, plan["k"], seed=plan.get("seed")) / token - for domain, task in record["stream"]: - if task not in selected: - continue - _, adopted = A.adopted_tree(adapter, root, task, mode) - task_dir = dest / "tasks" / library.slug(task) - tree.write_tree(surface, adopted, task_dir / "harness", stubs=False) - (task_dir / "task.md").write_text(adapter.task_prompt(task), encoding="utf-8") - lesson = ( - lib / library.domain_dir(domain) / "harnesses" / library.slug(task) / "EXPERIENCE.md" - ) - if lesson.is_file(): - (task_dir / "EXPERIENCE.md").write_text(lesson.read_text(), encoding="utf-8") - index.append( - f"- {task} ({domain}): adopted harness " - f"{'differs from' if adopted != base else 'same as'} the start; " - f"lesson: {'yes' if lesson.is_file() else 'no'}" - ) - tasks.append(task) - if not tasks: - raise ConfigError(f"no successful selection for any task in {mode}") - (dest / "INDEX.md").write_text("\n".join(index) + "\n", encoding="utf-8") - return tasks - - -def run(args, ctx) -> int: - with ctx.progress(): - adapter, root = A.open_run(ctx, args.name) - base = A.base_tree(adapter) - record = C.load_run(root) - surface = adapter.substrate.surface() - config = ctx.settings - events.start( - root / "events.jsonl", - budget_usd=config.get("run.budget_usd"), - max_calls=config.get("run.max_calls"), - ) - prop = Proposer(adapter.spec.channels["proposer"], audit_dir=root / "consolidated/_audit") - cost = 0.0 - try: - with tempfile.TemporaryDirectory(prefix="harness2-consolidate-") as temporary: - context = Path(temporary) - tasks = _context(adapter, root, record, args.mode, context, base) - mission = prompts.CONSOLIDATE_MISSION.replace("{n}", str(len(tasks))).replace( - "{surface}", prompts._st(surface).surface_md - ) - err = "" - for _ in range(2): - with ctx.progress(): - reply = prop.call( - mission - + (f"\n\n# YOUR PREVIOUS ATTEMPT WAS REJECTED\n\n{err}\n\n" if err else ""), - context, - f"consolidate.{args.mode}", - harvest="context/harness", - ) - cost += reply.cost_usd - if reply.workspace is None: - err = "the harness directory did not come back from the session" - continue - rejected = [] - new = tree.read_tree(surface, reply.workspace, rejected=rejected) - _, err = recursion._accept_tree(surface, adapter, new, base, "consolidate") - if rejected: - err = "files outside the editable surface: " + ", ".join(sorted(rejected)) - if not err: - break - tree.write_tree(surface, new, context / "harness", stubs=True) - else: - raise ConfigError(err) - except events.Stopped as exc: - reason = str(exc) - ctx.out(events.totals(), lambda totals: _stopped(root.name, reason, totals)) - return 4 - notes = re.search(r"(.*?)", reply.text, re.S) - notes = notes.group(1).strip() if notes else reply.text.strip() - dest = root / "consolidated" / args.mode - path = tree.write_tree(surface, new, dest / "harness", stubs=False) - (dest / "notes.md").write_text(notes + "\n", encoding="utf-8") - (dest / "consolidation.json").write_text( - json.dumps( - { - "tasks": tasks, - "model": prop.spec.model, - "cost_usd": cost, - "created": datetime.now(timezone.utc).isoformat(), - }, - indent=2, - ) - + "\n", - encoding="utf-8", - ) - changes = A.changes(base, new) - ctx.out( - {"path": path, "tasks": tasks, "changes": changes, "notes": notes}, - lambda _: "\n".join( - [ - f"Consolidated the harnesses adopted for {len(tasks)} task(s) ({args.mode}) into {path}", - *( - [f" {kind} {rel}" for rel, kind in changes] - if changes - else [ - "The consolidated harness is the same as the starting one: " - "no task's edits generalized." - ] - ), - f"Next: harness2 eval {root.name} compares it with the starting harness; " - f"harness2 apply {root.name} writes it into the project.", - ] - ), - ) - return 0 diff --git a/harness2/cli/eval.py b/harness2/cli/eval.py index 5f438f34..2e44c836 100644 --- a/harness2/cli/eval.py +++ b/harness2/cli/eval.py @@ -114,7 +114,7 @@ def run(args, ctx) -> int: adapter, root = A.open_run(ctx, args.name) harness = root / "consolidated" / args.mode / "harness" if not harness.is_dir(): - raise ConfigError("run `harness2 consolidate` first") + raise ConfigError("this run has no final harness yet; run `harness2 run` to finish it") record = C.load_run(root) allowed = set(adapter.tasks(args.split)) wanted = set(_csv(args.only_tasks)) diff --git a/harness2/cli/main.py b/harness2/cli/main.py index 091ab2c8..0449ceb8 100644 --- a/harness2/cli/main.py +++ b/harness2/cli/main.py @@ -33,7 +33,6 @@ "report", "runs", "diff", - "consolidate", "eval", "apply", "config", diff --git a/harness2/cli/run.py b/harness2/cli/run.py index 4d578e72..c95dfe58 100644 --- a/harness2/cli/run.py +++ b/harness2/cli/run.py @@ -17,6 +17,7 @@ from __future__ import annotations import argparse +import sys from pathlib import Path from .. import core, registry @@ -24,6 +25,7 @@ from ..core.proposer import Proposer from ..settings import ConfigError from . import _common as C +from ._final import consolidate FLAGS = { "suite": "suite.name", @@ -97,7 +99,7 @@ def _estimate(tasks: int, k: int, modes: tuple[str, ...], totals: dict) -> dict: rollouts = tasks * (k + (ledger.expected_rollouts(k) - k) * len(modes)) sessions = ( tasks * len(modes) * sum(per_mode[key] for key in ("improver", "selector", "reflector")) - ) + ) + len(modes) return { "per_task_per_mode": per_mode, "rollouts": rollouts, @@ -116,12 +118,15 @@ def _stopped(name: str, reason: str, totals: dict) -> str: "interrupted": "interrupted", "budget_usd": f"reached --budget-usd {totals['budget_usd']:g}", "max_calls": f"reached --max-calls {totals['max_calls']}", + "usage_limit": "reached the model provider's usage limit", }.get(reason, reason) advice = ( "Run the same command to resume." if reason == "interrupted" else "Raise the limit and run the same command to resume." ) + if reason == "usage_limit": + advice = "Run the same command once the limit resets to resume." return f"Run {name} stopped: {why}{_spend(totals)}.\n{advice}" @@ -274,13 +279,33 @@ def run(args, ctx) -> int: else (("failed", "failed tasks") if failed else ("ok", "")) ) done = sum(int(row.get("done", 0)) for row in result["domains"]) + final = {} + for mode in modes: + label = C.mode_label(mode) + try: + if not any( + path.parent.parent.name == audit.name + and (selection := (row.get("selections") or {}).get(mode) or {}).get("ok") + and selection.get("chosen_label") + for path, row in C.summaries(root) + ): + continue + final[label] = consolidate(ctx, adapter, root, label)["path"] + except Exception as exc: + print( + f"Could not build the final harness ({label}): {exc}. " + "Run the same command again to retry.", + file=sys.stderr, + ) ctx.out( - {"result": result, "totals": events.totals()}, + {"result": result, "totals": events.totals(), "final": final}, lambda data: ( f"Run {name}: {done} task(s) done, {failed} failed" + _spend(data["totals"]) + (f"\nErrors: {reason}" if errors else "") - + f"\nNext: harness2 status {name} · harness2 judge {name} · harness2 diff {name}" + + "".join(f"\nFinal harness ({mode}): {path}" for mode, path in final.items()) + + f"\nNext: harness2 diff {name} shows it; harness2 apply {name} writes it into " + f"the project; harness2 eval {name} measures it first." ), ) return 1 if errors else 3 if failed else 0 diff --git a/tests/test_cli.py b/tests/test_cli.py index 1b34f751..5d0bc03a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -30,7 +30,7 @@ from unittest.mock import Mock, patch from harness2 import settings -from harness2.cli import _adopt, completion, judge, main, run, runs +from harness2.cli import _adopt, _final, completion, judge, main, run, runs from harness2.cli import _common as C from harness2.core import events from harness2.core.judge import Scored @@ -81,7 +81,10 @@ def context(self, *args): return C.Context(parsed, getattr(parsed._module, "FLAGS", {})) def execute(self, k=1, *extra): - with patch.object(run, "Proposer", DemoProposer): + with ( + patch.object(run, "Proposer", DemoProposer), + patch.object(_final, "Proposer", DemoProposer), + ): return self.invoke("--json", "run", "--k", str(k), *extra) def test_all_command_arguments(self): @@ -251,7 +254,7 @@ def test_dry_run_estimate_and_stream_filters(self): self.assertEqual(code, 0) data = json.loads(stdout) self.assertEqual(data["stream"], [["accounting", "ledger-2"]]) - self.assertEqual((data["estimate"]["rollouts"], data["estimate"]["sessions"]), (7, 8)) + self.assertEqual((data["estimate"]["rollouts"], data["estimate"]["sessions"]), (7, 10)) self.assertEqual(data["estimate"]["per_task_per_mode"]["reflector"], 1) self.assertEqual(list((self.project / ".harness2/runs").rglob("run.json")), []) self.assertEqual(list((self.project / ".harness2/runs").rglob("events.jsonl")), []) @@ -259,7 +262,7 @@ def test_dry_run_estimate_and_stream_filters(self): self.assertEqual(code, 0) self.assertIn("would refine 1 task(s)", stdout) self.assertIn("accounting", stdout) - self.assertIn("Model calls: 8 agent runs + 8", stdout) + self.assertIn("Model calls: 8 agent runs + 9", stdout) self.assertEqual( run._estimate(1, 2, ("agg",), {"spent_usd": 4, "calls": 2})["average_cost_per_call"], 2 ) @@ -278,7 +281,7 @@ def test_demo_backed_run_judge_report_diff_for_each_scale(self): self.assertEqual(code, 0) data = json.loads(stdout) self.assertEqual( - data["totals"]["calls"], 2 * (k + 6 * k + 6 * k + 2 * (1 + int(k > 1))) + data["totals"]["calls"], 2 * (k + 6 * k + 6 * k + 2 * (1 + int(k > 1))) + 2 ) name = f"claude-k{k}" root = self.project / ".harness2/runs" / name @@ -309,7 +312,7 @@ def test_demo_backed_run_judge_report_diff_for_each_scale(self): code, stdout, _ = self.invoke("--json", "diff", name) self.assertEqual(code, 0) diff = json.loads(stdout) - self.assertEqual(diff["task"], "ledger-2") + self.assertEqual(diff["task"], "consolidated") self.assertIn("-total = sum", diff["diff"]) self.assertIn("+total = sum", diff["diff"]) code, stdout, _ = self.invoke( @@ -321,7 +324,10 @@ def test_demo_backed_run_judge_report_diff_for_each_scale(self): self.assertEqual(self.invoke("--json", "judge", name, "--bases")[0], 0) code, stdout, _ = self.invoke("--json", "runs", "show", name) self.assertEqual((code, json.loads(stdout)["done"]), (0, 2)) - with patch.object(run, "Proposer", DemoProposer): + with ( + patch.object(run, "Proposer", DemoProposer), + patch.object(_final, "Proposer", DemoProposer), + ): code, stdout, _ = self.invoke( "--json", "run", "--k", str(k), "--mode", "parallel,sequential" ) @@ -348,17 +354,23 @@ def test_budget_resume_and_failure_exit_events(self): for exception, expected, status in ( (KeyboardInterrupt(), 130, "interrupted"), (events.Stopped("interrupted"), 130, "interrupted"), + (events.Stopped("usage_limit"), 4, "stopped"), (RuntimeError("fixture crash"), 1, "error"), ): with ( patch.object(run.schedule, "run_sweep", side_effect=exception), patch.object(run, "Proposer", DemoProposer), + patch.object(run, "consolidate") as final, ): code, out, error = self.invoke("run") + final.assert_not_called() self.assertEqual(code, expected) self.assertEqual( json.loads((root / "events.jsonl").read_text().splitlines()[-1])["status"], status ) + if isinstance(exception, events.Stopped) and str(exception) == "usage_limit": + self.assertIn("reached the model provider's usage limit", out) + self.assertIn("Run the same command once the limit resets to resume.", out) if expected == 130: self.assertIn("Run the same command to resume", out) for result, expected in ( @@ -368,6 +380,7 @@ def test_budget_resume_and_failure_exit_events(self): with ( patch.object(run.schedule, "run_sweep", return_value=result), patch.object(run, "Proposer", DemoProposer), + patch.object(_final, "Proposer", DemoProposer), ): self.assertEqual(self.invoke("run")[0], expected) diff --git a/tests/test_consolidate.py b/tests/test_consolidate.py index 6f1fd024..9897723f 100644 --- a/tests/test_consolidate.py +++ b/tests/test_consolidate.py @@ -28,7 +28,7 @@ from types import SimpleNamespace from unittest.mock import Mock, patch -from harness2.cli import _adopt, _common, consolidate, eval +from harness2.cli import _adopt, _common, _final, eval, run from harness2.cli.main import COMMANDS, build_parser, main from harness2.core import events from harness2.settings import ConfigError @@ -172,29 +172,23 @@ def new_task(self, task, *, split="train", judge=True): with (directory / "task.toml").open("a") as stream: stream.write(f'split = "{split}"\n') - def execute(self, *argv): - self.invoke("run", "--name", "trial", *argv) + def execute(self, *argv, code=0): + return self.invoke("run", "--name", "trial", *argv, code=code) def adapter(self): args = build_parser().parse_args(["-C", str(self.project), "diff", "trial"]) return _adopt.open_run(_common.Context(args, {}), "trial")[0] def test_two_tasks_retry_adoption_eval_and_json(self): - self.execute() + text, _ = self.execute() before = _common.event_totals(self.run_dir) - fallback = json.loads(self.invoke("--json", "diff", "trial")[0]) - self.assertEqual(fallback["task"], "b-two") - text, _ = self.invoke("consolidate", "trial") self.assertEqual(self.calls.read_text(), "2") path = self.run_dir / "consolidated/parallel/harness" self.assertEqual((path / "CLAUDE.md").read_text(), "Use shared procedure.\n") + self.assertIn(f"Final harness (parallel): {path}", text) self.assertIn( - f"Consolidated the harnesses adopted for 2 task(s) (parallel) into {path}", text - ) - self.assertIn(" M CLAUDE.md", text) - self.assertIn( - "Next: harness2 eval trial compares it with the starting harness; " - "harness2 apply trial writes it into the project.", + "Next: harness2 diff trial shows it; harness2 apply trial writes it into the project; " + "harness2 eval trial measures it first.", text, ) meta = json.loads((path.parent / "consolidation.json").read_text()) @@ -205,12 +199,13 @@ def test_two_tasks_retry_adoption_eval_and_json(self): (path.parent / "notes.md").read_text(), "a-one and b-two support the shared procedure.\n", ) - data = json.loads(self.invoke("--json", "consolidate", "trial")[0]) - self.assertEqual(set(data), {"path", "tasks", "changes", "notes"}) - self.assertEqual(data["path"], str(path)) - self.assertEqual(data["changes"], [["CLAUDE.md", "M"]]) - self.assertEqual(data["tasks"], ["a-one", "b-two"]) - self.assertEqual(data["notes"], "a-one and b-two support the shared procedure.") + data = json.loads(self.invoke("--json", "run", "--name", "trial")[0]) + self.assertEqual(set(data), {"result", "totals", "final"}) + self.assertEqual(data["final"], {"parallel": str(path)}) + self.assertEqual(data["totals"]["calls"], before["calls"]) + self.assertEqual(data["totals"]["spent_usd"], before["spent_usd"]) + self.assertEqual(self.calls.read_text(), "2") + self.assertTrue(list((self.run_dir / "consolidated/_audit").glob("*.response.md"))) consolidated = json.loads(self.invoke("--json", "diff", "trial")[0]) self.assertEqual(consolidated["task"], "consolidated") self.assertIn("+Use shared procedure.", consolidated["diff"]) @@ -246,7 +241,7 @@ def test_two_tasks_retry_adoption_eval_and_json(self): self.assertEqual(len(eval_rows), 8) self.assertEqual({row["cost_usd"] for row in eval_rows}, {0.125}) self.assertAlmostEqual( - _common.event_totals(self.run_dir)["spent_usd"], before["spent_usd"] + 1.03 + _common.event_totals(self.run_dir)["spent_usd"], before["spent_usd"] + 1.0 ) self.assertEqual( {row["run_id"] for row in eval_rows}, @@ -261,7 +256,6 @@ def test_two_tasks_retry_adoption_eval_and_json(self): def test_eval_filters_skipped_failed_grades_and_limits(self): self.new_task("c-no-judge", judge=False) self.execute() - self.invoke("consolidate", "trial") text, _ = self.invoke("eval", "trial") self.assertIn("Skipped 1 task(s) with no check or rubric: c-no-judge", text) data = json.loads( @@ -300,35 +294,76 @@ def test_eval_filters_skipped_failed_grades_and_limits(self): self.assertEqual(data["mean"], {"start": None, "consolidated": None}) self.assertEqual(data["skipped"], ["c-no-judge"]) - def test_modes_unchanged_and_validation_failure_preserves_result(self): - self.execute("--mode", "parallel,sequential") - self.invoke("consolidate", "trial", "--mode", "sequential") + def test_modes_and_unchanged_harness(self): + text, _ = self.execute("--mode", "parallel,sequential") + self.assertEqual(self.calls.read_text(), "3") adapter = self.adapter() - self.assertEqual(_adopt.harness_for(adapter, self.run_dir, None, "seq")[0], "consolidated") + for mode in ("parallel", "sequential"): + path = self.run_dir / "consolidated" / mode / "harness" + self.assertIn(f"Final harness ({mode}): {path}", text) + self.assertEqual( + _adopt.harness_for(adapter, self.run_dir, None, mode)[0], "consolidated" + ) self.assertEqual( _adopt.harness_for(adapter, self.run_dir, "a-one", "sequential")[0], "a-one" ) - self.assertEqual(_adopt.harness_for(adapter, self.run_dir, None, "parallel")[0], "b-two") - path = self.run_dir / "consolidated/sequential/harness/CLAUDE.md" + with patch.dict(os.environ, {"WC_BEHAVIOR": "unchanged"}): + self.invoke("run", "--name", "unchanged") + self.assertEqual(self.invoke("diff", "unchanged")[0], "\n") + + def test_validation_failure_keeps_exit_code_and_can_retry(self): with patch.dict(os.environ, {"WC_BEHAVIOR": "invalid"}): - _, err = self.invoke("consolidate", "trial", "--mode", "sequential", code=1) + text, err = self.execute() + self.assertIn("Could not build the final harness (parallel): ", err) self.assertIn("broken.py", err) + self.assertTrue(err.rstrip().endswith(". Run the same command again to retry.")) + self.assertNotIn("Final harness (parallel):", text) + self.assertFalse((self.run_dir / "consolidated/parallel/harness").exists()) + self.assertEqual(self.calls.read_text(), "2") + _, err = self.invoke("eval", "trial", code=1) + self.assertIn("this run has no final harness yet; run `harness2 run` to finish it", err) + with patch.dict(os.environ, {"WC_BEHAVIOR": "valid"}): + self.execute() + self.assertEqual(self.calls.read_text(), "3") + path = self.run_dir / "consolidated/parallel/harness/CLAUDE.md" self.assertEqual(path.read_text(), "Use shared procedure.\n") + for failure, expected in ( + ("failure", 0), + ("failure", 3), + ("failure", 1), + (events.Stopped("usage_limit"), 0), + ): + error = failure if isinstance(failure, Exception) else ConfigError(failure) + result = { + "domains": [ + {"failed": int(expected == 3), **({"error": "dead"} if expected == 1 else {})} + ] + } + with ( + patch.object(run.schedule, "run_sweep", return_value=result), + patch.object(run, "consolidate", side_effect=error), + ): + text, err = self.execute(code=expected) + self.assertEqual( + err, + f"Could not build the final harness (parallel): {error}. " + "Run the same command again to retry.\n", + ) + self.assertEqual(path.read_text(), "Use shared procedure.\n") + + def test_missing_workspace_failure_keeps_exit_code(self): with patch.dict(os.environ, {"WC_BEHAVIOR": "missing"}): - _, err = self.invoke("consolidate", "trial", code=1) - self.assertIn("the harness directory did not come back", err) - with patch.dict(os.environ, {"WC_BEHAVIOR": "unchanged"}): - text, _ = self.invoke("consolidate", "trial") + _, err = self.execute() self.assertIn( - "The consolidated harness is the same as the starting one: no task's edits generalized.", - text, + "Could not build the final harness (parallel): " + "the harness directory did not come back from the session. " + "Run the same command again to retry.", + err, ) - self.assertEqual(self.invoke("diff", "trial")[0], "\n") + self.assertEqual(self.calls.read_text(), "2") def test_context_stream_order_optional_lessons_and_errors(self): self.execute() - _, err = self.invoke("eval", "trial", code=1) - self.assertIn("run `harness2 consolidate` first", err) adapter = self.adapter() record = _common.load_run(self.run_dir) plan = record["schedule"] @@ -347,7 +382,7 @@ def test_context_stream_order_optional_lessons_and_errors(self): ) row["selections"]["agg"]["chosen_tree_is_base"] = True task_path.write_text(json.dumps(row)) - tasks = consolidate._context(adapter, self.run_dir, record, "parallel", dest, base) + tasks = _final._context(adapter, self.run_dir, record, "parallel", dest, base) self.assertEqual(tasks, ["b-two", "a-one"]) self.assertEqual( (dest / "INDEX.md").read_text(), @@ -360,9 +395,14 @@ def test_context_stream_order_optional_lessons_and_errors(self): row["selections"]["agg"]["ok"] = False path.write_text(json.dumps(row)) with self.assertRaisesRegex(ConfigError, "no successful selection"): - consolidate._context( - adapter, self.run_dir, record, "parallel", self.root / "empty", base - ) + _final._context(adapter, self.run_dir, record, "parallel", self.root / "empty", base) + with ( + patch.object(run.schedule, "run_sweep", return_value={"domains": [{"failed": 2}]}), + patch.object(run, "consolidate") as final, + ): + data = json.loads(self.invoke("--json", "run", "--name", "trial", code=3)[0]) + final.assert_not_called() + self.assertEqual(data["final"], {}) def test_rollout_grades_and_optional_cost(self): adapter = SimpleNamespace(execute=Mock(return_value=self.root), score=Mock()) @@ -393,15 +433,15 @@ def test_rollout_grades_and_optional_cost(self): def test_command_order_and_help(self): i = COMMANDS.index("diff") - self.assertEqual(COMMANDS[i + 1 : i + 3], ("consolidate", "eval")) + self.assertEqual(COMMANDS[i + 1], "eval") + self.assertNotIn("consolidate", COMMANDS) parser = build_parser() - for name in ("consolidate", "eval"): - sub = parser._subparsers._group_actions[0].choices[name] - for action in sub._actions: - self.assertTrue(action.help) - if action.dest != "help": - self.assertEqual(action.help, action.help.lower()) - self.assertFalse(action.help.endswith(".")) + sub = parser._subparsers._group_actions[0].choices["eval"] + for action in sub._actions: + self.assertTrue(action.help) + if action.dest != "help": + self.assertEqual(action.help, action.help.lower()) + self.assertFalse(action.help.endswith(".")) if __name__ == "__main__": diff --git a/tests/test_e2e.py b/tests/test_e2e.py index a89b733e..c41f3c63 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -230,7 +230,7 @@ def invoke(*argv): Counter(call["role"] for call in all_calls), { "executor": 24, - "improver": 18, + "improver": 20, "selector": 2, "reflector": 4, }, @@ -277,8 +277,8 @@ def invoke(*argv): for row in run_rows }, { - "claude-k1": (2, 2, 1.08), - "k2": (2, 2, 2.16), + "claude-k1": (2, 2, 1.09), + "k2": (2, 2, 2.17), }, ) invoke("judge") @@ -336,7 +336,7 @@ def invoke(*argv): ["git", "worktree", "list", "--porcelain"], cwd=project, text=True ) self.assertEqual(worktrees.count("worktree "), 1) - print("E2E: 2 task types; k=1/k=2; 48 calls; 2 graded; adopted; resume=0 calls") + print("E2E: 2 task types; k=1/k=2; 50 calls; 2 graded; adopted; resume=0 calls") if __name__ == "__main__": From 03d706018761b60dc06c8edaf871263c8f2c6567 Mon Sep 17 00:00:00 2001 From: Bae Seongmin <83870598+3ae3ae@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:52:41 +0900 Subject: [PATCH 15/21] feat: a guide for AI agents, and JSON from init and apply (#16) `harness2 guide` prints how an AI agent should drive harness2 on a person's behalf: ask before spending, bound runs with --max-calls, read --json, answer prompts with flags, never edit runs or harness files directly, the init/task/doctor/run --detach/status/stop/diff/eval/apply workflow, where things are stored and what the exit codes mean, followed by the current project's settings, tasks and runs. `guide --install claude` installs the same text as a user-level Claude Code skill, which harness2's own runs never load. `harness2 chat` now builds on it. `init --json` and `apply --json` print a result object on stdout and keep their human-readable text on stderr; `apply --json` refuses to ask for confirmation and needs --yes or --dry-run. Co-authored-by: Claude Opus 5.5 --- harness2/cli/apply.py | 57 ++++++++++++---- harness2/cli/chat.py | 65 +++---------------- harness2/cli/guide.py | 147 ++++++++++++++++++++++++++++++++++++++++++ harness2/cli/init.py | 33 ++++++++-- harness2/cli/main.py | 1 + tests/test_ui.py | 37 ++++++++++- 6 files changed, 266 insertions(+), 74 deletions(-) create mode 100644 harness2/cli/guide.py diff --git a/harness2/cli/apply.py b/harness2/cli/apply.py index 40125482..2cee874e 100644 --- a/harness2/cli/apply.py +++ b/harness2/cli/apply.py @@ -25,6 +25,8 @@ from __future__ import annotations import argparse +import contextlib +import json import sys from . import _adopt as A @@ -62,6 +64,19 @@ def add_parser(sub) -> argparse.ArgumentParser: def run(args, ctx) -> int: + if not ctx.args.json: + return _apply(args, ctx)[0] + if not (args.yes or args.dry_run): + message = "--json cannot ask for confirmation; add --yes or --dry-run" + print(json.dumps({"error": message, "exit_code": 1})) + return 1 + with contextlib.redirect_stdout(sys.stderr): # the review text; stdout carries the JSON + code, summary = _apply(args, ctx) + print(json.dumps(summary)) + return code + + +def _apply(args, ctx) -> tuple[int, dict]: adapter, run_dir = A.open_run(ctx, args.name) task, adopted = A.harness_for(adapter, run_dir, args.task, args.mode) base = A.base_tree(adapter) @@ -71,7 +86,7 @@ def run(args, ctx) -> int: delta = A.changes(base, adopted) if not delta: ctx.log(f"The harness adopted for {task} is the one the run started from; nothing to do.") - return 0 + return 0, {"run": run_dir.name, "harness": task, "changes": []} moved = {rel for rel, _ in delta if current.get(rel) != base.get(rel)} todo = [(rel, kind) for rel, kind in delta if args.force or rel not in moved] @@ -90,35 +105,55 @@ def run(args, ctx) -> int: + ("will be overwritten (--force)." if args.force else "skipped; --force overwrites.") ) print() + + def summary(applied=(), done=None) -> dict: + return { + "run": run_dir.name, + "harness": task, + "mode": args.mode, + "changes": [ + { + "path": rel, + "kind": kind, + "changed_in_project": rel in moved, + "applied": (rel, kind) in applied, + } + for rel, kind in delta + ], + "written": (done or {}).get("written", []), + "deleted": (done or {}).get("deleted", []), + } + if not todo: ctx.log("Nothing left to apply.") - return 0 + return 0, summary() dirty = A.dirty_paths(ctx.settings, adapter) if dirty and not args.force: - print( + message = ( "These harness files have uncommitted changes:\n " + "\n ".join(dirty) + "\nCommit or stash them first so the result can be reviewed and undone with git, " - "or pass --force.", - file=sys.stderr, + "or pass --force." ) - return 1 + print(message, file=sys.stderr) + return 1, {"error": message, "exit_code": 1} if args.dry_run: - return 0 + return 0, summary() if args.interactive: chosen = _review(todo, current, adopted) elif args.yes: chosen = todo elif not sys.stdin.isatty(): - print("Not a terminal: pass --yes to apply, or --dry-run to only look.", file=sys.stderr) - return 1 + message = "Not a terminal: pass --yes to apply, or --dry-run to only look." + print(message, file=sys.stderr) + return 1, {"error": message, "exit_code": 1} else: chosen = todo if _confirm(f"Apply {len(todo)} change(s)?") else [] if not chosen: ctx.log("Nothing applied.") - return 0 + return 0, summary() final = dict(current) for rel, kind in chosen: @@ -131,7 +166,7 @@ def run(args, ctx) -> int: if args.branch: print(f"On branch {args.branch}.") print("Review with `git diff`; undo with `git restore` on the files listed above.") - return 0 + return 0, summary(chosen, done) def _delta(rel: str, before: dict, after: dict) -> str: diff --git a/harness2/cli/chat.py b/harness2/cli/chat.py index f7e8a23a..9c046b7c 100644 --- a/harness2/cli/chat.py +++ b/harness2/cli/chat.py @@ -27,41 +27,13 @@ import shutil import sys -GUIDE = """\ -You are helping a person use Harness² (the `harness2` command) in the project at {root}. +from . import guide as G -Harness² improves the harness of a coding agent -- the files the agent reads on every run: -CLAUDE.md or AGENTS.md, skills, subagents, helper scripts and hooks. It runs the agent on the -person's own tasks, has an improver propose harness edits from what those runs show, probes -the edits on the same tasks, and composes the best of them. A run never changes the project's -files; `harness2 apply` writes an adopted harness back once the person has reviewed it. +CHAT_INTRO = """\ +You are talking with a person about Harness² (the `harness2` command) in the project at +{root}. Help them write tasks, a check or a rubric, read a run's results, and decide what to +apply. The guide below is how to use harness2. -How you help: -- Writing tasks. A task is a directory .harness2/tasks// holding task.md -- the - instruction the agent receives, written the way the person would really ask -- and an - optional task.toml with: domain, split ("train" or "test"), workspace ("project": a clean - checkout of the repository at `ref`; "files": a scratch directory seeded from files/), - ref, deliverable ("diff", "files" or "message"), check (a shell command that grades the - result), timeout. check/ and rubric.md hold grading material that neither the agent nor - the improver ever sees. Good tasks are real, varied work the person wants done better; a - check command or a rubric.md makes the results measurable. -- Reading results. A run lives in .harness2/runs//: run.json (its settings), - events.jsonl (progress and cost), adapt/// (summary.json, the candidate - harnesses under harnesses/, and the selection/ and reflection/ notes) and library/ (what - carries over between tasks in the same domain). `harness2 status`, `harness2 report` and - `harness2 diff` summarise them. -- Settings. `harness2 config show --origin` prints every setting and where it came from; - `harness2 config set KEY VALUE` changes one. - -Rules: -- `harness2 run`, `harness2 judge` and `harness2 doctor --live` call models and cost money. - Propose the exact command and let the person run or approve it. -- Never edit anything under .harness2/runs/. -- Do not edit the project's harness files to apply a result; `harness2 apply` does that - after review. - -Current state of this project: -{state} """ @@ -82,7 +54,11 @@ def run(args, ctx) -> int: if not binary: print("Neither `claude` nor `codex` is on PATH.", file=sys.stderr) return 1 - guide = GUIDE.format(root=ctx.project, state=_state(ctx)) + guide = ( + CHAT_INTRO.format(root=ctx.project) + + G.GUIDE + + f"\n## This project\n\n{G.state_lines(ctx)}\n" + ) if tool == "claude": argv = [binary, "--append-system-prompt", guide] else: @@ -97,24 +73,3 @@ def _default_tool(settings) -> str | None: if tool in ("claude", "codex") and shutil.which(tool): return tool return "codex" if shutil.which("codex") else None - - -def _state(ctx) -> str: - s = ctx.settings - lines = [ - f"- executor: {s.get('executor.name')} on {s.get('executor.model') or '(model not set)'}", - f"- improver: {s.get('improver.channel')} on {s.get('improver.model') or '(not set)'}", - f"- harness directory: {s.get('paths.harness')}", - ] - tasks_dir = s.path("paths.tasks") - tasks = sorted(p.parent.name for p in tasks_dir.glob("*/task.md")) - shown = ", ".join(tasks[:30]) + (" ..." if len(tasks) > 30 else "") - lines.append(f"- tasks ({len(tasks)}): {shown or 'none yet'}") - runs_dir = s.path("paths.runs") - runs = sorted( - (p for p in runs_dir.glob("*/run.json")), key=lambda p: p.stat().st_mtime, reverse=True - ) - lines.append( - "- runs, newest first: " + (", ".join(p.parent.name for p in runs[:10]) or "none yet") - ) - return "\n".join(lines) diff --git a/harness2/cli/guide.py b/harness2/cli/guide.py new file mode 100644 index 00000000..8dc29532 --- /dev/null +++ b/harness2/cli/guide.py @@ -0,0 +1,147 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The guide for AI agents that drive harness2 on a person's behalf. + +`harness2 guide` prints it with the current project's state; `--install claude` installs it +as a user-level Claude Code skill, which harness2's own runs never load. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +GUIDE = """\ +# harness2: guide for agents + +harness2 improves a coding agent's harness -- the files the agent reads on every run: +CLAUDE.md or AGENTS.md, skills, subagents, helper scripts and hooks. It runs the agent on the +project's own tasks, proposes harness edits from what those runs show, probes them on the +same tasks, and ends with one final harness for the project. A run never changes the +project's files; `harness2 apply` writes the final harness back once the person agrees. + +## Rules + +- Runs spend money or subscription quota. Before `harness2 run` or `harness2 eval`, show the + person the plan from `harness2 run --dry-run` and get their go-ahead. Bound every run with + `--max-calls N` (and `--budget-usd X` when an API key pays for it). +- Read results with `--json`. Errors then arrive on stdout as + {"error": "...", "exit_code": n}. +- Nothing waits for keyboard input when there is no terminal: answer each prompt with its + flag (`--yes`, `--auth`, `--executor`, ...) instead. +- Never edit anything under .harness2/runs/. Never edit the project's harness files to apply + a result: show `harness2 diff`, then run `harness2 apply --yes` once the person agrees. + +## Workflow + +1. `harness2 init --yes [--executor claude|codex] [--auth login|api-key]` creates + .harness2/. `--auth login` uses the agent CLI's own login (a Claude or ChatGPT + subscription). +2. Tasks. `harness2 task new [--files] [--check ""]`, then write the request in + .harness2/tasks//task.md the way the person would really ask for it. Good tasks are + real, varied work the person wants done better. A `check` command (exit status 0 means + pass) makes results measurable; grading files in check/ and rubric.md are never shown to + the agent. More options in task.toml: domain, split ("train"/"test"), workspace + ("project": a clean checkout at `ref`; "files": a scratch directory seeded from files/), + deliverable ("diff", "files", "message"), timeout. List them with + `harness2 task ls --json`. +3. `harness2 doctor --json`: every row must be ok before a run. +4. `harness2 run --dry-run` shows the tasks and the number of model calls. After the + person's go-ahead: `harness2 run --detach --max-calls N`. It returns at once with the run + name; runs take minutes to hours. +5. Follow it with `harness2 status --json`: "active", each task's "state", + "spent_usd", "calls". `harness2 stop ` stops it; running the same `harness2 run` + command again resumes and reuses every finished step. +6. When it has finished, `harness2 diff ` shows the final harness. To measure it + first, `harness2 eval --json` compares it with the starting harness on the tasks + that have a check or rubric. +7. With the person's agreement: `harness2 apply --yes`. + +## Where things are + +- .harness2/config.toml: settings; `harness2 config show --origin --json` resolves them. +- .harness2/runs//: run.json (settings, task list), events.jsonl (every agent run and + model session, with cost), consolidated// (the final harness and the improver's + notes), adapt/ (per-task candidates, selections, lessons), eval/ (eval results). + +## Exit codes + +0 ok, 1 error, 2 bad usage, 3 finished with failed tasks, 4 stopped at a limit (--max-calls, +--budget-usd, or the provider's usage limit), 130 interrupted. After 4 or 130, the same +command resumes. +""" + +SKILL_HEAD = """\ +--- +name: harness2 +description: Use harness2 when the person wants their coding agent's setup (CLAUDE.md, AGENTS.md, skills, subagents, hooks) improved on their own tasks, or asks to set up, run, follow, evaluate or apply a harness2 run. +--- + +""" + + +def add_parser(sub) -> argparse.ArgumentParser: + p = sub.add_parser( + "guide", + help="print the guide for AI agents that use harness2", + description="Print how an AI agent should drive harness2, with this project's current " + "state. Point your agent at it, or install it as a Claude Code skill.", + ) + p.add_argument( + "--install", + choices=("claude",), + help="install the guide as a user-level Claude Code skill (~/.claude/skills/harness2)", + ) + return p + + +def run(args, ctx) -> int: + if args.install: + path = Path.home() / ".claude/skills/harness2/SKILL.md" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(SKILL_HEAD + GUIDE, encoding="utf-8") + ctx.out( + {"installed": str(path)}, + lambda data: ( + f"Installed {data['installed']}. Claude Code loads it in new " + "sessions; harness2's own runs never do." + ), + ) + return 0 + state = state_lines(ctx) if ctx.project_or_none else "" + text = GUIDE + (f"\n## This project\n\n{state}\n" if state else "") + ctx.out({"guide": GUIDE, "project": state or None}, lambda data: text.rstrip()) + return 0 + + +def state_lines(ctx) -> str: + """The project's settings, tasks and runs, in a few lines an agent can act on.""" + s = ctx.settings + lines = [ + f"- root: {ctx.project}", + f"- executor: {s.get('executor.name')} on {s.get('executor.model') or '(model not set)'}", + f"- improver: {s.get('improver.channel')} on {s.get('improver.model') or '(not set)'}", + f"- harness directory: {s.get('paths.harness')}", + ] + tasks = sorted(p.parent.name for p in s.path("paths.tasks").glob("*/task.md")) + shown = ", ".join(tasks[:30]) + (" ..." if len(tasks) > 30 else "") + lines.append(f"- tasks ({len(tasks)}): {shown or 'none yet'}") + runs = sorted( + s.path("paths.runs").glob("*/run.json"), key=lambda p: p.stat().st_mtime, reverse=True + ) + lines.append( + "- runs, newest first: " + (", ".join(p.parent.name for p in runs[:10]) or "none yet") + ) + return "\n".join(lines) diff --git a/harness2/cli/init.py b/harness2/cli/init.py index 81f63cb0..34f9e4dd 100644 --- a/harness2/cli/init.py +++ b/harness2/cli/init.py @@ -17,6 +17,8 @@ from __future__ import annotations import argparse +import contextlib +import json import os import shutil import subprocess @@ -89,11 +91,21 @@ def add_parser(sub) -> argparse.ArgumentParser: def run(args, ctx) -> int: + if not getattr(args, "json", False): + return _run(args)[0] + with contextlib.redirect_stdout(sys.stderr): # the wizard's text; stdout carries the JSON + code, summary = _run(args) + print(json.dumps(summary)) + return code + + +def _run(args) -> tuple[int, dict]: root = Path(args.project or Path.cwd()).resolve() config = S.project_path(root) if config.exists() and not args.force: - print(f"{config} already exists; pass --force to overwrite it.", file=sys.stderr) - return 1 + message = f"{config} already exists; pass --force to overwrite it." + print(message, file=sys.stderr) + return 1, {"error": message, "exit_code": 1} ask = _asker(interactive=not args.yes and sys.stdin.isatty()) print(f"Setting up Harness² in {root}\n") @@ -102,12 +114,12 @@ def run(args, ctx) -> int: default = next((name for name in ("claude", "codex") if found[name]), None) executor = args.executor or (_pick_executor(ask, found, default) if interactive else default) if executor is None: - print( + message = ( "Neither Claude Code (`claude`) nor Codex (`codex`) is on PATH. Install one, or " - "rerun with --executor command to drive your own agent.", - file=sys.stderr, + "rerun with --executor command to drive your own agent." ) - return 1 + print(message, file=sys.stderr) + return 1, {"error": message, "exit_code": 1} command_lines = "" editable: list[str] = [] @@ -159,7 +171,14 @@ def run(args, ctx) -> int: " harness2 doctor check the agent CLIs, keys and tasks\n" " harness2 run --dry-run preview what a run will do and how many calls it makes" ) - return 0 + return 0, { + "config": str(config), + "executor": executor, + "model": model, + "improver": {"channel": channel, "model": improver}, + "auth": auth, + "harness": harness, + } def _asker(*, interactive: bool): diff --git a/harness2/cli/main.py b/harness2/cli/main.py index 0449ceb8..67c82872 100644 --- a/harness2/cli/main.py +++ b/harness2/cli/main.py @@ -37,6 +37,7 @@ "apply", "config", "chat", + "guide", "setup", "demo", "bridge", diff --git a/tests/test_ui.py b/tests/test_ui.py index 80c32028..53a2549e 100644 --- a/tests/test_ui.py +++ b/tests/test_ui.py @@ -1,6 +1,7 @@ """The interactive commands: init, status, apply and the human selector.""" import argparse +import io import json import os import sys @@ -14,7 +15,8 @@ class _Ctx: - def __init__(self, project: Path): + def __init__(self, project: Path, json: bool = False): + self.args = argparse.Namespace(json=json) self.project = project self.settings = type("S", (), {"path": lambda _self, key: project})() self.logged: list[str] = [] @@ -85,6 +87,39 @@ def perform(settings, adapter, tree, *, base, branch): written, {"CLAUDE.md": "base\nmy rule\n", "a.md": "a run\n", "b.md": "b mine\n"} ) + def test_apply_json_reports_the_plan_and_refuses_to_ask(self): + base, adopted = {"CLAUDE.md": "a\n"}, {"CLAUDE.md": "a\nb\n"} + args = argparse.Namespace( + name=None, task=None, mode="parallel", interactive=False, yes=False, + branch=None, dry_run=False, force=False, + ) # fmt: skip + patches = ( + patch.object(apply.A, "open_run", return_value=(None, self.root / "run")), + patch.object(apply.A, "harness_for", return_value=("consolidated", adopted)), + patch.object(apply.A, "base_tree", return_value=base), + patch.object(apply.A, "project_tree", return_value=dict(base)), + patch.object(apply.A, "dirty_paths", return_value=[]), + ) + for item in patches: + item.start() + self.addCleanup(item.stop) + out = io.StringIO() + with patch("sys.stdout", out): + self.assertEqual(apply.run(args, _Ctx(self.root, json=True)), 1) + self.assertIn("--yes or --dry-run", json.loads(out.getvalue())["error"]) + args.dry_run, out = True, io.StringIO() + with patch("sys.stdout", out), patch("sys.stderr"): + self.assertEqual(apply.run(args, _Ctx(self.root, json=True)), 0) + plan = json.loads(out.getvalue()) + self.assertEqual(plan["changes"][0]["path"], "CLAUDE.md") + self.assertFalse(plan["changes"][0]["applied"]) + + def test_guide_mentions_every_agent_contract(self): + from harness2.cli import guide + + for needle in ("--json", "--detach", "--max-calls", "harness2 stop", "apply", "130"): + self.assertIn(needle, guide.GUIDE) + def test_status_reads_tasks_and_spend_from_events(self): run = self.root / "run" run.mkdir() From d23808283ecfc5e1d117eea9573f263f0a9ea41c Mon Sep 17 00:00:00 2001 From: Bae Seongmin <83870598+3ae3ae@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:58:01 +0900 Subject: [PATCH 16/21] feat: background runs, stop, and JSON errors for agent use (#17) - `harness2 run --detach` starts the same run as a background process, logging to /run.log, and returns its name, pid and log path at once (an immediate failure is reported with the log's tail). - A running run records itself in /run.pid; a second start of the same run is refused, and `harness2 stop [NAME]` asks it to stop: the calls in flight finish, the run exits, and rerunning resumes it. Only a detached run's own process group is signalled, never the shell or agent that launched a foreground run. - With --json, a failing command prints {"error", "exit_code"} on stdout instead of an error line; `diff --json` returns the changes and per-file diffs. - `apply -i` and `chat` refuse to start without a terminal instead of waiting for input, and the agent guide says how to change init's choices with `config set`. Co-authored-by: Claude Opus 5.5 --- harness2/cli/_common.py | 28 +++ harness2/cli/apply.py | 4 + harness2/cli/chat.py | 3 + harness2/cli/diff.py | 23 +-- harness2/cli/guide.py | 3 +- harness2/cli/main.py | 9 +- harness2/cli/run.py | 82 ++++++++- harness2/cli/stop.py | 59 +++++++ tests/test_agent_ops.py | 353 ++++++++++++++++++++++++++++++++++++++ tests/test_cli.py | 8 +- tests/test_consolidate.py | 8 +- 11 files changed, 557 insertions(+), 23 deletions(-) create mode 100644 harness2/cli/stop.py create mode 100644 tests/test_agent_ops.py diff --git a/harness2/cli/_common.py b/harness2/cli/_common.py index 2d013b59..c948c96b 100644 --- a/harness2/cli/_common.py +++ b/harness2/cli/_common.py @@ -119,6 +119,34 @@ def resolve_run(ctx: Context, name: str | None) -> Path: return max(runs, key=lambda p: p.stat().st_mtime_ns) +def run_process(run_dir: Path) -> dict | None: + path = run_dir / "run.pid" + try: + record = json.loads(path.read_text(encoding="utf-8")) + if ( + not isinstance(record, dict) + or type(record.get("pid")) is not int + or record["pid"] <= 0 + or not isinstance(record.get("started"), str) + or not isinstance(record.get("argv"), list) + or any(not isinstance(arg, str) for arg in record["argv"]) + ): + raise ValueError("invalid run process record") + except FileNotFoundError: + return None + except ValueError: + path.unlink(missing_ok=True) + return None + try: + os.kill(record["pid"], 0) + except ProcessLookupError: + path.unlink(missing_ok=True) + return None + except PermissionError: + pass + return record + + def load_run(run_dir: Path) -> dict: try: record = json.loads((run_dir / "run.json").read_text(encoding="utf-8")) diff --git a/harness2/cli/apply.py b/harness2/cli/apply.py index 2cee874e..227938af 100644 --- a/harness2/cli/apply.py +++ b/harness2/cli/apply.py @@ -141,6 +141,10 @@ def summary(applied=(), done=None) -> dict: if args.dry_run: return 0, summary() + if args.interactive and not sys.stdin.isatty(): + message = "--interactive needs a terminal; use --yes or --dry-run instead." + print(message, file=sys.stderr) + return 1, {"error": message, "exit_code": 1} if args.interactive: chosen = _review(todo, current, adopted) elif args.yes: diff --git a/harness2/cli/chat.py b/harness2/cli/chat.py index 9c046b7c..6d5f838a 100644 --- a/harness2/cli/chat.py +++ b/harness2/cli/chat.py @@ -49,6 +49,9 @@ def add_parser(sub) -> argparse.ArgumentParser: def run(args, ctx) -> int: + if not sys.stdin.isatty(): + print("harness2 chat is a conversation and needs a terminal.", file=sys.stderr) + return 1 tool = args.tool or _default_tool(ctx.settings) binary = shutil.which(tool) if tool else None if not binary: diff --git a/harness2/cli/diff.py b/harness2/cli/diff.py index 274d9944..4a2e8fcc 100644 --- a/harness2/cli/diff.py +++ b/harness2/cli/diff.py @@ -45,15 +45,18 @@ def run(args, ctx) -> int: adapter, root = _adopt.open_run(ctx, args.name) task, after = _adopt.harness_for(adapter, root, args.task, args.mode) before = _adopt.base_tree(adapter) - data = ( - diff_summary(adapter.substrate.surface(), before, after) - if args.stat - else { - "task": task, - "diff": "".join( - _adopt.unified(rel, before, after) for rel, _ in _adopt.changes(before, after) - ), - } + changes = _adopt.changes(before, after) + data = { + "label": task, + "changes": changes, + "diffs": {rel: _adopt.unified(rel, before, after) for rel, _ in changes}, + } + ctx.out( + data, + lambda row: ( + str(diff_summary(adapter.substrate.surface(), before, after)) + if args.stat + else "".join(row["diffs"].values()).rstrip("\n") + ), ) - ctx.out(data, str if args.stat else lambda row: row["diff"].rstrip("\n")) return 0 diff --git a/harness2/cli/guide.py b/harness2/cli/guide.py index 8dc29532..240adff7 100644 --- a/harness2/cli/guide.py +++ b/harness2/cli/guide.py @@ -48,7 +48,8 @@ 1. `harness2 init --yes [--executor claude|codex] [--auth login|api-key]` creates .harness2/. `--auth login` uses the agent CLI's own login (a Claude or ChatGPT - subscription). + subscription). Change any choice afterwards with `harness2 config set KEY VALUE`, for + example `executor.model`. 2. Tasks. `harness2 task new [--files] [--check ""]`, then write the request in .harness2/tasks//task.md the way the person would really ask for it. Good tasks are real, varied work the person wants done better. A `check` command (exit status 0 means diff --git a/harness2/cli/main.py b/harness2/cli/main.py index 67c82872..aec4bc86 100644 --- a/harness2/cli/main.py +++ b/harness2/cli/main.py @@ -29,6 +29,7 @@ "task", "run", "status", + "stop", "judge", "report", "runs", @@ -94,14 +95,20 @@ def main(argv: list[str] | None = None) -> int: break argv.insert(index, alias) args = build_parser().parse_args(argv) + args.argv = argv ctx = Context(args, getattr(args._module, "FLAGS", {})) try: ctx.debug(f"command: {args.command}") return args._module.run(args, ctx) except KeyboardInterrupt: + if args.json: + ctx.out({"error": "interrupted", "exit_code": 130}, str) return 130 except (OSError, RuntimeError, ValueError, KeyError) as exc: - ctx.log(f"error: {exc}") + if args.json: + ctx.out({"error": str(exc), "exit_code": 1}, str) + else: + ctx.log(f"error: {exc}") return 1 diff --git a/harness2/cli/run.py b/harness2/cli/run.py index c95dfe58..31680703 100644 --- a/harness2/cli/run.py +++ b/harness2/cli/run.py @@ -17,7 +17,11 @@ from __future__ import annotations import argparse +import json +import os +import subprocess import sys +from datetime import datetime, timezone from pathlib import Path from .. import core, registry @@ -81,6 +85,9 @@ def add_parser(subparsers): p.add_argument( "--dry-run", action="store_true", help="show the tasks and the model calls; run nothing" ) + p.add_argument( + "--detach", action="store_true", help="start the run in the background and return at once" + ) return p @@ -159,15 +166,58 @@ def _print_stream(data) -> str: return "\n".join(lines) +def _detach(args, ctx, name: str, root: Path) -> int: + root.mkdir(parents=True, exist_ok=True) + log = root / "run.log" + argv = [arg for arg in args.argv if arg != "--detach"] + if args.name is None: + argv.extend(["--name", name]) + with log.open("a", encoding="utf-8") as output: + child = subprocess.Popen( + [sys.executable, "-m", "harness2", *argv], + stdin=subprocess.DEVNULL, + stdout=output, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + try: + code = child.wait(timeout=2) + except subprocess.TimeoutExpired: + code = None + if code: + print( + "\n".join(log.read_text(encoding="utf-8", errors="replace").splitlines()[-20:]), + file=sys.stderr, + ) + return code + ctx.out( + {"name": name, "pid": child.pid, "log": log}, + lambda data: ( + f"Started run {name} in the background (pid {data['pid']}).\n" + f"Follow it: harness2 status {name} --watch\n" + f"Stop it: harness2 stop {name}\n" + f"Log: {log}" + ), + ) + return 0 + + def run(args, ctx) -> int: events.start(None) status, reason, name = "error", "", args.name or "" + pid_path = None try: + if args.detach and args.dry_run: + raise ConfigError("--detach starts a real run; drop --dry-run") project, config = ctx.project, ctx.settings k, seed = config.get("run.k"), config.get("run.seed") modes = C.parse_modes(config.get("run.modes")) name = args.name or C.default_name(config) root = C.run_path(ctx, name) + if not args.dry_run and (process := C.run_process(root)) is not None: + raise ConfigError(f"run {name} is already running (pid {process['pid']})") + if args.detach: + return _detach(args, ctx, name, root) suite = registry.load(config.get("suite.name")) split_file = (project / args.split_file).resolve() if args.split_file else None suite_ctx = registry.SuiteContext(config, project, root, split_file) @@ -228,6 +278,18 @@ def run(args, ctx) -> int: "split_file": str(split_file) if split_file else None, }, ) + pid_path = root / "run.pid" + pid_path.write_text( + json.dumps( + { + "pid": os.getpid(), + "started": datetime.now(timezone.utc).isoformat(), + "argv": args.argv, + } + ) + + "\n", + encoding="utf-8", + ) core.configure( **{ key: config.get(f"run.{key}") @@ -297,6 +359,8 @@ def run(args, ctx) -> int: "Run the same command again to retry.", file=sys.stderr, ) + if errors and args.json: + raise ConfigError(reason) ctx.out( {"result": result, "totals": events.totals(), "final": final}, lambda data: ( @@ -312,15 +376,27 @@ def run(args, ctx) -> int: except events.Stopped as exc: reason = str(exc) status = "interrupted" if reason == "interrupted" else "stopped" - ctx.out(events.totals(), lambda totals: _stopped(name, reason, totals)) + ctx.out( + {"error": "interrupted", "exit_code": 130} + if status == "interrupted" and args.json + else events.totals(), + lambda totals: _stopped(name, reason, totals), + ) return 130 if status == "interrupted" else 4 except KeyboardInterrupt: status, reason = "interrupted", "interrupted" events.request_stop(reason) - ctx.out(events.totals(), lambda totals: _stopped(name, reason, totals)) + ctx.out( + {"error": "interrupted", "exit_code": 130} if args.json else events.totals(), + lambda totals: _stopped(name, reason, totals), + ) return 130 except Exception as exc: reason = str(exc) raise finally: - events.emit("run_end", status=status, reason=reason) + try: + events.emit("run_end", status=status, reason=reason) + finally: + if pid_path is not None: + pid_path.unlink(missing_ok=True) diff --git a/harness2/cli/stop.py b/harness2/cli/stop.py new file mode 100644 index 00000000..4490d0fa --- /dev/null +++ b/harness2/cli/stop.py @@ -0,0 +1,59 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Ask a run and its agent processes to stop.""" + +from __future__ import annotations + +import os +import signal + +from . import _common as C + + +def add_parser(subparsers): + p = subparsers.add_parser("stop", help="stop a running run; rerunning it resumes") + p.add_argument("name", nargs="?", help="run name (default: the most recent run)") + return p + + +def run(args, ctx) -> int: + root = C.resolve_run(ctx, args.name) + process = C.run_process(root) + if process is not None: + pid = process["pid"] + try: + # A detached run leads its own process group, so the agent CLIs it started stop + # with it. Otherwise the group can include whatever launched the run (a shell, an + # agent), so only the run itself is signalled. + if os.getpgid(pid) == pid: + os.killpg(pid, signal.SIGINT) + else: + os.kill(pid, signal.SIGINT) + except ProcessLookupError: + process = None + ctx.out( + { + "name": root.name, + "pid": process["pid"] if process else None, + "signalled": process is not None, + }, + lambda data: ( + f"Asked run {root.name} (pid {data['pid']}) to stop. It ends the calls in flight, " + "then exits; run the same command again to resume." + if data["signalled"] + else f"Run {root.name} is not running." + ), + ) + return 0 diff --git a/tests/test_agent_ops.py b/tests/test_agent_ops.py new file mode 100644 index 00000000..d34c04e2 --- /dev/null +++ b/tests/test_agent_ops.py @@ -0,0 +1,353 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline checks for machine output and run process controls.""" + +from __future__ import annotations + +import contextlib +import io +import json +import os +import signal +import subprocess +import sys +import tempfile +import unittest +from dataclasses import replace +from datetime import datetime +from pathlib import Path +from unittest.mock import Mock, patch + +from harness2.cli import _adopt, diff, main, run +from harness2.cli import _common as C +from harness2.core import events +from harness2.core.tree import diff_summary +from harness2.demo import DemoAdapter, DemoProposer + + +class AgentOpsTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.project = Path(temporary.name).resolve() + (self.project / ".harness2").mkdir() + env = patch.dict(os.environ, {"XDG_CONFIG_HOME": str(self.project / "config")}, clear=True) + env.start() + self.addCleanup(env.stop) + self.addCleanup(events.start, None) + self.root = self.project / ".harness2/runs/claude-k1" + + def make_adapter(ctx): + adapter = DemoAdapter(ctx.run_root / "fixture") + adapter.spec = replace(adapter.spec, runs_root=ctx.run_root) + return adapter + + self.suite = Mock(make_adapter=Mock(side_effect=make_adapter), check=Mock(return_value=[])) + registry = patch("harness2.registry.load", return_value=self.suite) + registry.start() + self.addCleanup(registry.stop) + + def invoke(self, *args): + stdout, stderr = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + code = main.main(["-C", str(self.project), *args]) + return code, stdout.getvalue(), stderr.getvalue() + + def pid_record(self, root=None): + root = root or self.root + root.mkdir(parents=True, exist_ok=True) + record = {"pid": os.getpid(), "started": "2026-10-19T00:00:00+00:00", "argv": ["run"]} + (root / "run.pid").write_text(json.dumps(record)) + return record + + def test_json_errors_and_text_behavior(self): + for exception in ( + OSError("io"), + RuntimeError("runtime"), + ValueError("value"), + KeyError("key"), + ): + with ( + self.subTest(exception=exception), + patch.object(diff, "run", side_effect=exception), + ): + code, out, err = self.invoke("--json", "--quiet", "diff") + self.assertEqual(code, 1) + self.assertEqual( + out.splitlines(), [json.dumps({"error": str(exception), "exit_code": 1})] + ) + self.assertEqual(err, "") + self.assertEqual(self.invoke("diff"), (1, "", f"error: {exception}\n")) + self.assertEqual(self.invoke("--quiet", "diff"), (1, "", "")) + with patch.object(diff, "run", side_effect=KeyboardInterrupt): + self.assertEqual(self.invoke("diff"), (130, "", "")) + code, out, err = self.invoke("--json", "diff") + self.assertEqual( + (code, json.loads(out), err), (130, {"error": "interrupted", "exit_code": 130}, "") + ) + + def test_argparse_errors_still_exit_two(self): + with ( + contextlib.redirect_stdout(io.StringIO()) as out, + contextlib.redirect_stderr(io.StringIO()) as err, + ): + with self.assertRaises(SystemExit) as raised: + main.main(["--json", "run", "--unknown"]) + self.assertEqual(raised.exception.code, 2) + self.assertEqual(out.getvalue(), "") + self.assertIn("usage: harness2", err.getvalue()) + + def test_detach_preserves_in_process_argv_and_appends_logs(self): + child = Mock(pid=12345) + child.wait.side_effect = subprocess.TimeoutExpired("harness2", 2) + + def popen(argv, **kwargs): + self.assertEqual( + argv, + [ + sys.executable, + "-m", + "harness2", + "-C", + str(self.project), + "--json", + "run", + "--seed", + "0", + "--name", + "claude-k1-s0", + ], + ) + self.assertEqual(kwargs["stdin"], subprocess.DEVNULL) + self.assertEqual(kwargs["stderr"], subprocess.STDOUT) + self.assertTrue(kwargs["start_new_session"]) + self.assertNotIn("cwd", kwargs) + self.assertNotIn("env", kwargs) + kwargs["stdout"].write("child log\n") + return child + + root = self.root.with_name("claude-k1-s0") + root.mkdir(parents=True) + (root / "run.log").write_text("previous log\n") + with patch.object(run.subprocess, "Popen", side_effect=popen): + code, out, err = self.invoke("--json", "run", "--seed", "0", "--detach") + self.assertEqual( + (code, json.loads(out), err), + (0, {"name": root.name, "pid": 12345, "log": str(root / "run.log")}, ""), + ) + self.assertEqual((root / "run.log").read_text(), "previous log\nchild log\n") + child.wait.assert_called_once_with(timeout=2) + self.suite.make_adapter.assert_not_called() + self.assertFalse((root / "run.pid").exists()) + + def test_detach_explicit_name_and_exact_text(self): + child = Mock(pid=12345) + child.wait.return_value = 0 + with patch.object(run.subprocess, "Popen", return_value=child) as popen: + code, out, err = self.invoke("run", "--detach", "--name=chosen") + log = self.root.with_name("chosen") / "run.log" + self.assertEqual( + (code, out, err), + ( + 0, + f"Started run chosen in the background (pid 12345).\nFollow it: harness2 status chosen --watch\nStop it: harness2 stop chosen\nLog: {log}\n", + "", + ), + ) + self.assertEqual(popen.call_args.args[0][-2:], ["run", "--name=chosen"]) + + def test_detach_early_failure_reports_last_twenty_lines(self): + child = Mock(pid=12345) + child.wait.return_value = 7 + + def popen(argv, **kwargs): + kwargs["stdout"].write("".join(f"line {i}\n" for i in range(25))) + return child + + with patch.object(run.subprocess, "Popen", side_effect=popen): + code, out, err = self.invoke("--json", "run", "--detach") + self.assertEqual((code, out), (7, "")) + self.assertEqual(err.splitlines(), [f"line {i}" for i in range(5, 25)]) + + def test_detach_dry_run_is_rejected(self): + with patch.object(run.subprocess, "Popen") as popen: + code, out, _ = self.invoke("--json", "run", "--detach", "--dry-run") + self.assertEqual( + (code, json.loads(out)), + (1, {"error": "--detach starts a real run; drop --dry-run", "exit_code": 1}), + ) + popen.assert_not_called() + self.suite.make_adapter.assert_not_called() + + def test_already_running_refuses_both_start_modes(self): + record = self.pid_record() + for extra in ([], ["--detach"]): + with self.subTest(extra=extra), patch.object(run.subprocess, "Popen") as popen: + code, out, _ = self.invoke("--json", "run", *extra) + self.assertEqual( + (code, json.loads(out)), + ( + 1, + { + "error": f"run claude-k1 is already running (pid {record['pid']})", + "exit_code": 1, + }, + ), + ) + self.assertEqual(C.run_process(self.root), record) + popen.assert_not_called() + self.suite.make_adapter.assert_not_called() + + def test_run_process_live_stale_invalid_and_permission_denied(self): + self.assertIsNone(C.run_process(self.root)) + record = self.pid_record() + self.assertEqual(C.run_process(self.root), record) + with patch.object(C.os, "kill", side_effect=PermissionError): + self.assertEqual(C.run_process(self.root), record) + with patch.object(C.os, "kill", side_effect=ProcessLookupError): + self.assertIsNone(C.run_process(self.root)) + self.assertFalse((self.root / "run.pid").exists()) + for invalid in ( + "broken", + "[]", + "{}", + json.dumps(record | {"pid": 0}), + json.dumps(record | {"pid": True}), + json.dumps(record | {"argv": [1]}), + ): + with self.subTest(invalid=invalid), patch.object(C.os, "kill") as kill: + (self.root / "run.pid").write_text(invalid) + self.assertIsNone(C.run_process(self.root)) + self.assertFalse((self.root / "run.pid").exists()) + kill.assert_not_called() + + def test_pid_lifecycle_success_failure_and_interrupt(self): + argv = ["-C", str(self.project), "--json", "run"] + for exception, expected in ( + (None, 0), + (RuntimeError("crash"), 1), + (KeyboardInterrupt(), 130), + (events.Stopped("interrupted"), 130), + (events.Stopped("max_calls"), 4), + ): + + def sweep(*args, **kwargs): + record = C.run_process(self.root) + self.assertEqual(record["pid"], os.getpid()) + self.assertEqual(record["argv"], argv) + datetime.fromisoformat(record["started"]) + self.assertTrue((self.root / "run.json").is_file()) + if exception is not None: + raise exception + return {"domains": []} + + with ( + self.subTest(exception=exception), + patch.object(run.schedule, "run_sweep", side_effect=sweep), + patch.object(run, "Proposer", DemoProposer), + ): + code, out, _ = self.invoke("--json", "run") + self.assertEqual(code, expected) + self.assertFalse((self.root / "run.pid").exists()) + if expected in (1, 130): + self.assertEqual( + json.loads(out), + {"error": "crash" if expected == 1 else "interrupted", "exit_code": expected}, + ) + code, _, _ = self.invoke("run", "--dry-run") + self.assertEqual(code, 0) + self.assertFalse((self.root / "run.pid").exists()) + + def test_json_sweep_error(self): + with ( + patch.object( + run.schedule, "run_sweep", return_value={"domains": [{"error": "failed"}]} + ), + patch.object(run, "Proposer", DemoProposer), + ): + code, out, _ = self.invoke("--json", "run") + self.assertEqual((code, json.loads(out)), (1, {"error": "failed", "exit_code": 1})) + + def test_stop_signals_group_and_defaults_to_latest_run(self): + self.pid_record() + (self.root / "run.json").write_text("{}") + with ( + patch("harness2.cli.stop.os.getpgid", return_value=9876), + patch("harness2.cli.stop.os.killpg") as killpg, + patch("harness2.cli.stop.os.kill") as kill, + ): + self.assertEqual(self.invoke("stop")[0], 0) + killpg.assert_not_called() # not a group leader: the launcher's group is left alone + kill.assert_any_call(os.getpid(), signal.SIGINT) + for json_flag in ([], ["--json"]): + with ( + patch("harness2.cli.stop.os.getpgid", return_value=os.getpid()) as getpgid, + patch("harness2.cli.stop.os.killpg") as killpg, + ): + code, out, err = self.invoke(*json_flag, "stop") + self.assertEqual((code, err), (0, "")) + getpgid.assert_called_once_with(os.getpid()) + killpg.assert_called_once_with(os.getpid(), signal.SIGINT) + if json_flag: + self.assertEqual( + json.loads(out), {"name": self.root.name, "pid": os.getpid(), "signalled": True} + ) + else: + self.assertEqual( + out, + f"Asked run claude-k1 (pid {os.getpid()}) to stop. It ends the calls in flight, then exits; run the same command again to resume.\n", + ) + + def test_stop_not_running_and_process_exit_race(self): + self.root.mkdir(parents=True) + (self.root / "run.json").write_text("{}") + self.assertEqual( + self.invoke("stop", self.root.name), (0, "Run claude-k1 is not running.\n", "") + ) + code, out, err = self.invoke("--json", "stop") + self.assertEqual( + (code, json.loads(out), err), + (0, {"name": self.root.name, "pid": None, "signalled": False}, ""), + ) + self.pid_record() + with patch("harness2.cli.stop.os.getpgid", side_effect=ProcessLookupError): + self.assertEqual(self.invoke("stop"), (0, "Run claude-k1 is not running.\n", "")) + + def test_diff_json_and_unchanged_text(self): + before = {"a": "old\n", "d": "removed\n"} + after = {"a": "new\n", "b": "added\n"} + adapter = DemoAdapter(self.project / "diff-fixture") + changes = [["a", "M"], ["b", "A"], ["d", "D"]] + diffs = {rel: _adopt.unified(rel, before, after) for rel, _ in changes} + with ( + patch.object(_adopt, "open_run", return_value=(adapter, self.root)), + patch.object(_adopt, "harness_for", return_value=("task-1", after)), + patch.object(_adopt, "base_tree", return_value=before), + ): + for extra in ([], ["--stat"]): + code, out, err = self.invoke("--json", "diff", *extra) + self.assertEqual( + (code, json.loads(out), err), + (0, {"label": "task-1", "changes": changes, "diffs": diffs}, ""), + ) + self.assertEqual(self.invoke("diff"), (0, "".join(diffs.values()), "")) + self.assertEqual( + self.invoke("diff", "--stat"), + (0, str(diff_summary(adapter.substrate.surface(), before, after)) + "\n", ""), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cli.py b/tests/test_cli.py index 5d0bc03a..7019bbc0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -312,14 +312,14 @@ def test_demo_backed_run_judge_report_diff_for_each_scale(self): code, stdout, _ = self.invoke("--json", "diff", name) self.assertEqual(code, 0) diff = json.loads(stdout) - self.assertEqual(diff["task"], "consolidated") - self.assertIn("-total = sum", diff["diff"]) - self.assertIn("+total = sum", diff["diff"]) + self.assertEqual(diff["label"], "consolidated") + self.assertIn("-total = sum", diff["diffs"]["scripts/summarize.py"]) + self.assertIn("+total = sum", diff["diffs"]["scripts/summarize.py"]) code, stdout, _ = self.invoke( "--json", "diff", name, "--task", "ledger-1", "--mode", "sequential", "--stat" ) self.assertEqual(code, 0) - self.assertIn("scripts/summarize.py", json.loads(stdout)["modified"]) + self.assertIn(["scripts/summarize.py", "M"], json.loads(stdout)["changes"]) self.assertEqual(self.invoke("diff", name, "--task", "missing")[0], 1) self.assertEqual(self.invoke("--json", "judge", name, "--bases")[0], 0) code, stdout, _ = self.invoke("--json", "runs", "show", name) diff --git a/tests/test_consolidate.py b/tests/test_consolidate.py index 9897723f..fc7589d7 100644 --- a/tests/test_consolidate.py +++ b/tests/test_consolidate.py @@ -207,11 +207,11 @@ def test_two_tasks_retry_adoption_eval_and_json(self): self.assertEqual(self.calls.read_text(), "2") self.assertTrue(list((self.run_dir / "consolidated/_audit").glob("*.response.md"))) consolidated = json.loads(self.invoke("--json", "diff", "trial")[0]) - self.assertEqual(consolidated["task"], "consolidated") - self.assertIn("+Use shared procedure.", consolidated["diff"]) + self.assertEqual(consolidated["label"], "consolidated") + self.assertIn("+Use shared procedure.", consolidated["diffs"]["CLAUDE.md"]) task = json.loads(self.invoke("--json", "diff", "trial", "--task", "a-one")[0]) - self.assertEqual(task["task"], "a-one") - self.assertIn("+Task note.", task["diff"]) + self.assertEqual(task["label"], "a-one") + self.assertIn("+Task note.", task["diffs"]["CLAUDE.md"]) applied, _ = self.invoke("apply", "trial", "--dry-run", "--task", "a-one") self.assertIn("Harness adopted for a-one (parallel, run trial)", applied) applied, _ = self.invoke("apply", "trial", "--yes") From 0cae0f75a1b299d0996cc256afb765134dc1425b Mon Sep 17 00:00:00 2001 From: Bae Seongmin <83870598+3ae3ae@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:59:21 +0900 Subject: [PATCH 17/21] =?UTF-8?q?docs:=20describe=20Harness=C2=B2=20CLI;?= =?UTF-8?q?=20status=20shows=20whether=20a=20run=20is=20going=20(#18)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs: describe Harness² CLI as the general-purpose version of Harness² The README now presents the fork as Harness² CLI: what it adds to the original release (a comparison table), the run -> diff -> eval -> apply flow around one final harness, subscriptions as the default way to reach models, and how AI agents drive it (--json, --detach, stop, guide). The configuration guide documents claude-login and chatgpt-login, what is and is not isolated, and usage limits; the command reference covers stop, eval, guide, runs rm, --detach, JSON errors and the new run files; How it works explains the final harness step that the CLI adds to the paper's method. Adds a changelog. Co-Authored-By: Claude Opus 5.5 * feat(status): say whether a run is still going `harness2 status` (and its --json) now reports whether the run's process is alive and its pid, and says when a run stopped without finishing, for example after being killed, so the user knows to resume it. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 55 ++++++++ CONTRIBUTING.md | 5 +- README.md | 172 +++++++++++++++----------- docs/cli.md | 63 +++++++--- docs/configuration.md | 48 +++++-- docs/how-it-works.md | 55 +++++--- harness2/cli/bridge.py | 2 +- harness2/cli/status.py | 9 ++ plugins/harness2-benchmarks/README.md | 4 +- 9 files changed, 296 insertions(+), 117 deletions(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..c170ec19 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,55 @@ +# Changelog + +All notable changes to Harness² CLI are documented here. The format follows +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project uses +[Semantic Versioning](https://semver.org/). + +## [Unreleased] + +The first release of Harness² CLI, a general-purpose version of +[google-research/harness2](https://github.com/google-research/harness2) (forked at +`1cfe254`). The method and its prompts are unchanged; everything around them is new. + +### Added + +- One `harness2` command with subcommands: `init`, `doctor`, `task`, `run`, `status`, + `stop`, `diff`, `eval`, `apply`, `judge`, `report`, `runs`, `config`, `chat`, `guide`, + `setup`, `demo`, `bridge`, `completion`. +- Your own tasks: `.harness2/tasks//task.md` with optional `task.toml`, run in a clean + git worktree of your repository or in a scratch directory, graded optionally by a `check` + command or a `rubric.md`. +- Claude Code and command executors. A project's own harness (CLAUDE.md, `.claude/skills`, + `.claude/agents`, `.claude/scripts`, hooks; or AGENTS.md, `.agents/`, `.codex/hooks.json`) + is imported as the starting harness. +- Subscriptions: `claude-login/` and `chatgpt-login/` use the login of the + installed `claude` or `codex`, with the user's personal setup kept out of the runs. +- Any model provider: Anthropic, OpenAI, OpenAI- or Anthropic-compatible servers, Vertex AI + and Bedrock, through provider profiles; `harness2 bridge` serves any of them to Codex. +- Layered settings: defaults, `~/.config/harness2/config.toml`, `.harness2/config.toml`, + profiles, `HARNESS2_
__` variables and flags, with `config show --origin`. +- One final harness per run, built from every task's adopted harness and lesson; + `harness2 eval` compares it with the starting harness, and `harness2 apply` writes it into + the project, keeping edits made since the run started. +- Run control: `--dry-run` call estimates, `--budget-usd` and `--max-calls` limits, + clean stops at a provider's usage limit, Ctrl-C and `harness2 stop` with resume, + `--detach` background runs, relocatable run directories, `events.jsonl`. +- Use by AI agents: `--json` for every command including errors, flags for every prompt, + and `harness2 guide` (also installable as a Claude Code skill). +- Interactive use: the `init` wizard, `status --watch`, `apply -i`, `--selector human`, + `harness2 chat`. +- CI on Linux and macOS with Python 3.12 and 3.13. + +### Changed + +- The distribution is `harness2-cli`; the command and the import package stay `harness2`. +- The agent CLIs are the ones installed on the system instead of pinned downloads. +- Codex harness scripts live in `.agents/scripts/`, so they never collide with a project's + `scripts/`. + +### Removed + +- The seven `harness2-*` scripts' old flags (`--bench`, `--substrate`); the script names + remain as aliases of the new subcommands. +- The Vertex-only configuration in `.env` from the core package. JobBench, LAB and + WorkBuddy-Bench moved, with that configuration, to the + [harness2-benchmarks plugin](plugins/harness2-benchmarks/). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e4fe97d5..a4e86460 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,8 +1,9 @@ # Contributing -Issues and pull requests are welcome. This is an independent fork of +Issues and pull requests are welcome. Harness² CLI is an independent fork of [google-research/harness2](https://github.com/google-research/harness2); changes here are -not sent upstream. +not sent upstream. Every pull request runs the tests and ruff in CI, and notable changes go +in [CHANGELOG.md](CHANGELOG.md) under "Unreleased". ## Development setup diff --git a/README.md b/README.md index ff77c9e2..b33d4a77 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,10 @@

Harness²
- Tune your coding agent's harness on your own tasks + Harness² CLI: tune your coding agent's harness on your own tasks

+ CI Paper Website License @@ -13,22 +14,24 @@ A coding agent is a model plus a **harness**: the instructions, skills, subagent scripts and hooks it runs with. `harness2` improves the harness of the agent you already use (Claude Code's `CLAUDE.md` and `.claude/`, or Codex's `AGENTS.md` and `.agents/`) by running it on tasks from your own project, studying what happened, and refining the harness from -that evidence. The model stays the same. Your files change only when you apply a result you -have reviewed. +that evidence. The model stays the same. Your files change only when you apply the result. -It is a general-purpose command-line tool built on the method of +Harness² CLI is a general-purpose version of [Harness²: Recursive Agent Harnessing for an Open World](https://arxiv.org/abs/2610.14224). -This repository is an independent fork of -[google-research/harness2](https://github.com/google-research/harness2); the paper's -benchmark setup lives on as a [plugin](plugins/harness2-benchmarks/). +The original release ([google-research/harness2](https://github.com/google-research/harness2)) +reproduces the paper on three fixed benchmarks with Vertex AI. This fork turns the same method +into a command-line tool for any project, any task, Claude Code or Codex (or your own agent), +on your subscription or any model provider. [What changed](#what-changed-from-the-original) +lists the differences; the paper's benchmark setup lives on as a +[plugin](plugins/harness2-benchmarks/). - [How it works](#how-it-works) - [Install](#install) - [Quick start](#quick-start) +- [Driving it from an agent](#driving-it-from-an-agent) - [Commands](#commands) - [Configuration](#configuration) -- [Other agents](#other-agents) -- [The paper's benchmarks](#the-papers-benchmarks) +- [What changed from the original](#what-changed-from-the-original) - [Documentation](#documentation) ## How it works @@ -38,13 +41,18 @@ For every task you give it, `harness2`: 1. runs your agent with its current harness and records what it does; 2. has an improver agent propose harness edits from those trajectories, from two angles: how the agent works, and what it needs to know about this kind of task; -3. runs the agent with each candidate harness, and composes a final one from what the +3. runs the agent with each candidate harness, and composes a refined one from what the differences between the runs show; -4. keeps the adopted harness and a short lesson for the next task in the same area. +4. keeps the refined harness and a short lesson for the next task in the same area. -No step needs a grade or a reference answer, so any task you can describe can be used. -If you can also check the result, with a test command or a rubric, `harness2 judge` measures -how much the adopted harness helped. [How it works](docs/how-it-works.md) has the details. +When all tasks are done, it combines what the tasks taught into **one final harness** for +your project. That is the single result you review with `harness2 diff` and write into the +project with `harness2 apply`. + +No step needs a grade or a reference answer, so any task you can describe can be used. If +you can also check results, with a test command or a rubric, `harness2 eval` measures the +final harness against the one you started with. [How it works](docs/how-it-works.md) has +the details.

@@ -54,117 +62,136 @@ how much the adopted harness helped. [How it works](docs/how-it-works.md) has th ## Install -You need Python 3.12 or 3.13, git, and the agent CLI whose harness you want to tune: +You need Python 3.12 or 3.13, git, and the agent CLI whose harness you want to tune, +installed, on your `PATH` and logged in: [Claude Code](https://docs.claude.com/en/docs/claude-code) (`claude`) or -[Codex](https://github.com/openai/codex) (`codex`), installed and on your `PATH`. +[Codex](https://github.com/openai/codex) (`codex`). ```bash -uv tool install git+https://github.com/3ae3ae/harness2 -# or: pipx install git+https://github.com/3ae3ae/harness2 +uv tool install git+https://github.com/3ae3ae/harness2-cli +# or: pipx install git+https://github.com/3ae3ae/harness2-cli ``` -Runs call models through your provider's API, so set the key for the models you choose, -for example `ANTHROPIC_API_KEY` or `OPENAI_API_KEY`. Model calls cost money; the -[quick start](#quick-start) shows how to see the cost of a run before you start it. +By default runs use the CLI's own login, so a Claude or ChatGPT subscription is enough. You +can use an API key or another provider instead; see [Configuration](#configuration). Runs +make many model calls and use money or subscription quota: the [quick start](#quick-start) +shows how to see the number of calls first and how to cap them. ## Quick start Inside the project whose agent setup you want to improve: ```bash -harness2 init # pick the agent and models, creates .harness2/ +harness2 init # pick the agent, how it logs in, where the harness lives harness2 task new fix-login-redirect # then write the request in .harness2/tasks/fix-login-redirect/task.md -harness2 doctor # check CLIs, keys and tasks -harness2 run --dry-run # the task order and how many model calls a run makes -harness2 run # refine the harness on every task +harness2 doctor # check the CLI, its login and the tasks +harness2 run --dry-run # the tasks and how many model calls a run makes +harness2 run --max-calls 200 # refine the harness on every task, then build the final one ``` A run works in temporary git worktrees, so your working tree is never touched. While it -runs, or after it stops: +runs, from another terminal: ```bash harness2 status --watch # progress, spend, the harness picked per task -harness2 judge && harness2 report # grade the results of tasks that have a check or rubric -harness2 diff # what the adopted harness changes -harness2 apply # review it and write it into the project ``` -Run `harness2 run` again to resume an interrupted run; finished steps are reused. Write a -few varied tasks that look like the work you really do: see [Tasks](docs/tasks.md). +When it has finished: + +```bash +harness2 diff # what the final harness changes +harness2 eval # optional: compare it with your current harness on graded tasks +harness2 apply # review the changes and write them into the project +``` + +Running `harness2 run` again resumes an interrupted or limited run and reuses every finished +step. Write a few varied tasks that look like the work you really do: see +[Tasks](docs/tasks.md). + +## Driving it from an agent + +`harness2` is built to be driven by an AI agent as well as by hand. Every command has +`--json`, errors included; no command waits for input without a terminal, since every prompt +has a flag; and long runs go to the background with `harness2 run --detach`, are followed with +`harness2 status --json` and are stopped with `harness2 stop`. + +`harness2 guide` prints the contract for agents, with your project's current state. Tell your +agent to read it, or install it as a Claude Code skill: + +```bash +harness2 guide --install claude +``` ## Commands | Command | What it does | |---|---| -| `harness2 init` | Set up `.harness2/` in a project, with a short interactive wizard | -| `harness2 doctor` | Check the agent CLIs, provider keys, settings and tasks; `--live` also makes one tiny model call per model | +| `harness2 init` | Set up `.harness2/` in a project, with a short wizard (`--yes` for defaults) | +| `harness2 doctor` | Check the agent CLIs, their login or keys, settings and tasks; `--live` also makes one tiny call per model | | `harness2 task new / ls / import` | Create, list and import tasks | -| `harness2 run` | Refine the harness on the tasks; `--dry-run` previews, `--budget-usd` and `--max-calls` cap spending, `--selector human` lets you pick the final harness yourself | +| `harness2 run` | Refine the harness on the tasks and build the final harness; `--dry-run` previews, `--max-calls` / `--budget-usd` cap spending, `--detach` runs in the background, `--selector human` lets you pick between candidates | | `harness2 status` | A run's progress; `--watch` keeps it updated | -| `harness2 judge` | Grade the result each task adopted | -| `harness2 report` | Summarise grades, per run | -| `harness2 runs ls / show` | List runs and inspect one | -| `harness2 diff` | Show how an adopted harness differs from the starting one | -| `harness2 apply` | Write an adopted harness into the project, after showing the changes; `-i` to accept file by file | +| `harness2 stop` | Stop a running run; running it again resumes | +| `harness2 diff` | What the final harness changes | +| `harness2 eval` | Compare the final harness with the starting one on graded tasks | +| `harness2 apply` | Write the final harness into the project after showing the changes; `-i` to accept file by file | +| `harness2 judge` / `report` | Grade each task's adopted result, and summarise the grades | +| `harness2 runs ls / show / rm` | List, inspect and delete runs | | `harness2 config show / get / set` | Read and change settings; `show --origin` says where each value comes from | | `harness2 chat` | Talk to your agent CLI about tasks, runs and results | +| `harness2 guide` | The guide for AI agents that use harness2 | | `harness2 demo` | A local run with scripted editors and no model calls | -Every command takes `--help`. See the [command reference](docs/cli.md) for all flags, exit +Every command takes `--help`. The [command reference](docs/cli.md) lists every flag, the exit codes and the layout of a run directory. ## Configuration -Settings come from `~/.config/harness2/config.toml`, the project's -`.harness2/config.toml`, `HARNESS2_*` environment variables and flags, in that order of -precedence. Models are written `provider/model`; any OpenAI-compatible or -Anthropic-compatible endpoint, Vertex AI and Bedrock can be used: +Settings come from `~/.config/harness2/config.toml`, the project's `.harness2/config.toml`, +`HARNESS2_*` environment variables and flags, in that order of precedence. Models are +written `provider/model`: ```toml [executor] name = "claude" -model = "anthropic/claude-sonnet-5-5" +model = "claude-login/claude-sonnet-5-5" # your Claude subscription, through Claude Code [improver] channel = "claude" -model = "anthropic/claude-opus-5-5" - -[providers.local] # e.g. a vLLM or LM Studio server -type = "openai" -base_url = "http://localhost:8000/v1" -api_key_env = "LOCAL_API_KEY" +model = "claude-login/claude-opus-5-5" ``` -[Configuration](docs/configuration.md) lists every setting. +`chatgpt-login/` uses your ChatGPT subscription through Codex. API keys and other +endpoints work too: Anthropic, OpenAI, any OpenAI-compatible or Anthropic-compatible +server, Vertex AI and Bedrock. To tune an agent other than Claude Code or Codex, the +`command` executor runs your own command and edits the files you list. +[Configuration](docs/configuration.md) covers every setting. -## Other agents +## What changed from the original -To tune an agent other than Claude Code or Codex, use the `command` executor: give it the -command that runs one task and the files that make up its harness. +| | google-research/harness2 | Harness² CLI | +|---|---|---| +| Tasks | JobBench, LAB, WorkBuddy-Bench | your own tasks, in your repository or from files; the benchmarks as a plugin | +| Agents | OpenCode, Codex, pinned builds | Claude Code, Codex or any command, as installed on your system | +| Models | Vertex AI | your Claude or ChatGPT subscription, or any API provider | +| Result | one adopted harness per task, kept for scoring | one final harness for the project, which you review and apply | +| Interface | seven scripts, settings in `.env` | one `harness2` command, layered TOML settings, JSON output, background runs, resumable limits | -```toml -[executor] -name = "command" -command = "my-agent --task {prompt_file} --workdir {workspace}" -editable = ["prompts/*.md", "agent.yaml"] -``` - -## The paper's benchmarks - -JobBench, LAB and WorkBuddy-Bench, the benchmarks of the paper, are packaged as the -[harness2-benchmarks plugin](plugins/harness2-benchmarks/). Install it next to `harness2` -and they become suites: `harness2 run --suite jb`. +The method itself (propose, probe, compose, and the library of lessons) and its prompts are +the paper's. [How it works](docs/how-it-works.md) describes what was added around it. ## Documentation -- [How it works](docs/how-it-works.md): the method, what a run costs, what it stores +- [How it works](docs/how-it-works.md): the method, the final harness, what a run costs and stores - [Tasks](docs/tasks.md): writing tasks, workspaces, grading -- [Configuration](docs/configuration.md): settings, models and providers +- [Configuration](docs/configuration.md): settings, subscriptions, models and providers - [Command reference](docs/cli.md): every command and flag -- [Contributing](CONTRIBUTING.md) +- [Changelog](CHANGELOG.md) and [contributing](CONTRIBUTING.md) ## Citation +If you use Harness² CLI in research, cite the paper: + ```bibtex @article{xu2026harness2, title = {Harness$^2$: Recursive Agent Harnessing for an Open World}, @@ -178,5 +205,6 @@ and they become suites: `harness2 run --suite jb`. ## License Apache 2.0; see [LICENSE](LICENSE). The method and the original implementation are by the -paper's authors at Google; this fork is not a Google product and is not maintained by -Google. Third-party code in the benchmarks plugin keeps its upstream licenses. +paper's authors at Google; Harness² CLI is an independent fork, not a Google product, and is +not maintained by Google. Third-party code in the benchmarks plugin keeps its upstream +licenses. diff --git a/docs/cli.md b/docs/cli.md index 425cb9f9..983305c3 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -23,7 +23,7 @@ These go before the command: `harness2 -C ../api --json status`. | `-c KEY=VALUE`, `--config KEY=VALUE` | Override one setting for this command; repeatable. See [Configuration](configuration.md). | | `-q`, `--quiet` | Print results only, no progress. | | `-v`, `--verbose` | Print debug details. | -| `--json` | Print results as JSON on stdout; progress still goes to stderr. | +| `--json` | Print results as JSON on stdout; progress still goes to stderr. A failing command prints `{"error": "...", "exit_code": n}` instead of an error line. | | `--version` | Print the version. | ## Commands @@ -31,13 +31,15 @@ These go before the command: `harness2 -C ../api --json status`. ### `init` Set up harness2 in a project: asks which agent runs your tasks (`claude`, `codex` or a -custom `command`), which models to use, and where the harness lives; writes +custom `command`), whether it reaches its models through the CLI's own login (your +subscription) or an API key, which models to use, and where the harness lives; writes `.harness2/config.toml`, `.harness2/.gitignore` and `.harness2/tasks/`. | Option | Meaning | |---|---| -| `--yes`, `-y` | Take every default without asking. | +| `--yes`, `-y` | Take every default without asking. Without a terminal, defaults are taken anyway. | | `--executor {claude,codex,command}` | Skip the executor question. | +| `--auth {login,api-key}` | Skip the login question. | | `--force` | Overwrite an existing config file. | ### `doctor` @@ -63,8 +65,10 @@ makes a files task (with a `files/` directory) instead of a project task. `impor ### `run` -Refine the harness on the tasks. Running the same command again resumes the run and reuses -every finished step. +Refine the harness on the tasks, then combine what the tasks taught into one final harness +per recursion mode (one more improver session each), stored in the run's `consolidated/` +directory. Running the same command again resumes the run and reuses every finished step; +if building the final harness failed, only that step runs again. | Option | Meaning | |---|---| @@ -84,11 +88,19 @@ every finished step. | `--selector human` | Pick the final harness yourself when `k` > 1. | | `--steer TEXT\|@FILE` | Guidance added to every improver session. | | `--dry-run` | Show the tasks and the number of model calls; run nothing. | +| `--detach` | Start the run in the background and return at once with its name, pid and log file (`run.log` in the run directory). | | `--split-file FILE` | Task partition file, for benchmark suites. | A run keeps the settings it started with. Resuming it with a different executor, model, improver or task list is refused, listing what changed; use a new `--name` for a new -experiment. The limits apply to the whole run, across resumes. +experiment. The limits apply to the whole run, across resumes. A run that is already going +is not started twice. + +### `stop [NAME]` + +Stop a running run, whether it was started with `--detach` or in another terminal. It +finishes the model calls in flight and exits; running the same `harness2 run` command again +resumes it. ### `status [NAME]` @@ -113,26 +125,40 @@ rubric or the judge model changes. Average the grades per recursion mode. Several names compare runs. -### `runs ls` and `runs show NAME` +### `runs ls | show NAME | rm NAME` -List runs with their progress and spend, or show one run's settings, totals and tasks. +List runs with their progress and spend, show one run's state and settings, or delete a run +and everything it stored (`--yes` to skip the question). ### `diff [NAME]` -Show how the harness a run adopted for a task differs from the harness the run started -from. +Show how the run's final harness differs from the harness the run started from. | Option | Meaning | |---|---| -| `--task ID` | The task whose adopted harness to show. Default: the last task the run finished. | +| `--task ID` | Show the harness adopted for this one task instead of the final harness. | | `--mode {parallel,sequential}` | The recursion mode to take it from (default parallel). | | `--stat` | List the changed files only. | +### `eval [NAME]` + +Run the agent with the starting harness and with the final harness on the run's tasks that +have a check or rubric, grade both, and compare the pass rates. Costs one agent run per +task, harness and repeat. + +| Option | Meaning | +|---|---| +| `--repeat N` | Runs per task and harness (default 1). More runs, steadier numbers. | +| `--split {all,train,test}` | Compare on one split, for example the `test` tasks the run did not use. | +| `--only-tasks ID,...` | Compare on these tasks only. | +| `--mode {parallel,sequential}` | Which mode's final harness to compare (default parallel). | + ### `apply [NAME]` -Write an adopted harness into the project. Only what the run changed relative to the +Write the run's final harness into the project. Only what the run changed relative to the harness it started from is applied, so edits you made since are kept; a file changed both by the run and in the project is skipped unless `--force`. The changes are listed first. +With `--json`, `--yes` or `--dry-run` is required, since nothing is asked. | Option | Meaning | |---|---| @@ -169,6 +195,12 @@ rubric, or to explain a run's results. |---|---| | `--with {claude,codex}` | Which CLI to start. Default: the improver's channel, if installed. | +### `guide` + +Print the guide for AI agents that drive harness2, followed by the current project's +settings, tasks and runs. `--install claude` installs it as a user-level Claude Code skill +(`~/.claude/skills/harness2/`), which harness2's own runs never load. + ### `setup SUITE [ARGS...]` Run a plugin suite's installer, for example `harness2 setup lab` for the @@ -183,7 +215,7 @@ Run the method on two toy tasks with scripted editors: no model calls, a few sec Serve a configured model as an OpenAI Responses API endpoint through LiteLLM, so the Codex executor or channel can use providers it cannot reach directly. Needs -`pip install 'harness2[bridge]'`. +`pip install 'harness2-cli[bridge]'`. | Option | Meaning | |---|---| @@ -206,7 +238,7 @@ Print a shell completion script; see [below](#shell-completion). | 1 | An error: bad settings, a missing CLI or key, a failed check, a run that could not continue. | | 2 | Invalid command-line usage. | | 3 | `run` finished, but some tasks failed. | -| 4 | `run` stopped at `--budget-usd` or `--max-calls`. Raise the limit and rerun to resume. | +| 4 | `run` or `eval` stopped at `--budget-usd`, `--max-calls` or the provider's usage limit. Rerun to resume. | | 130 | Interrupted with Ctrl-C. Rerun to resume. | ## A run on disk @@ -214,7 +246,10 @@ Print a shell completion script; see [below](#shell-completion). ``` .harness2/runs// run.json settings, task list and schedule the run started with + run.pid, run.log while it runs: its process, and its output when started with --detach events.jsonl one line per agent run, model session, selection and task, with costs + consolidated// the final harness, the improver's notes and what it was built from + eval// results of harness2 eval base_harness/ the project's harness when the run started rollouts// one agent run: prompt.md, stream.jsonl (trajectory), final.md, deliverable.patch or output/, meta.json, judge.json diff --git a/docs/configuration.md b/docs/configuration.md index d576c19f..f2757184 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -42,8 +42,33 @@ parse as TOML is taken as a plain string: `-c executor.model=openai/gpt-6.1-sol` ## Models and providers Every model is written `provider/model`. The provider part names a `[providers.]` -table, which says what kind of endpoint it is and where its key comes from. Two providers -are built in: +table, which says what kind of endpoint it is and how it is reached. + +### Subscriptions + +Two built-in providers use the login your installed CLI already has, so a subscription is +all you need: + +| Model | Runs on | +|---|---| +| `claude-login/`, e.g. `claude-login/claude-sonnet-5-5` | your Claude subscription, through Claude Code's login (`claude auth login`) | +| `chatgpt-login/`, e.g. `chatgpt-login/gpt-6.1-sol` | your ChatGPT subscription, through Codex's login (`codex login`) | + +`harness2 init` picks these by default, and `harness2 doctor` checks that the CLI is logged +in. Runs still keep your personal setup out of the harness being measured: Claude Code runs +with `--setting-sources project,local --strict-mcp-config --no-session-persistence`, so your +own `~/.claude/CLAUDE.md`, skills, plugins, MCP servers and settings are not loaded and no +session history is written; Codex runs with `--ignore-user-config --ignore-rules`. Codex +still reads a global `~/.codex/AGENTS.md` if you have one, and `doctor` says so. + +Subscriptions have usage limits. When a run reaches one, it stops cleanly (exit code 4), +keeps everything finished so far, and the same `harness2 run` command resumes once the +limit resets. With a subscription, keep `run.phase_width` and `run.proposer_concurrency` +low, and cap runs with `--max-calls`. + +### API keys and other endpoints + +Two API-key providers are built in: ```toml [providers.anthropic] @@ -88,6 +113,8 @@ Each agent CLI speaks to some provider types and not others: | Provider type | Claude Code (`claude`) | Codex (`codex`) | OpenCode (`opencode`) | |---|---|---|---| +| `claude-login` | yes | no | no | +| `chatgpt-login` | no | yes | no | | `anthropic` | yes | via `harness2 bridge` | yes | | `openai` | no | yes (Responses API) | yes (Chat Completions) | | `vertex` | yes | via `harness2 bridge` | yes | @@ -95,7 +122,7 @@ Each agent CLI speaks to some provider types and not others: `harness2 doctor` reports a model whose provider its CLI cannot use. To run Codex on a model it cannot reach directly, serve that model as a Responses API endpoint with the bridge -(`pip install 'harness2[bridge]'`), then point an `openai`-type provider at it: +(`pip install 'harness2-cli[bridge]'`), then point an `openai`-type provider at it: ```bash harness2 bridge --model anthropic/claude-sonnet-5-5 @@ -104,12 +131,10 @@ harness2 bridge --model anthropic/claude-sonnet-5-5 The agent CLIs are the ones installed on your system (`claude`, `codex`, `opencode` on `PATH`). Set `executor.bin` or `improver.bin` to use another binary or a full command. -**Logins are not used.** Every run gives the agent CLI a fresh, empty configuration -directory (`CLAUDE_CONFIG_DIR` for Claude Code, `CODEX_HOME` for Codex), so your personal -instructions, skills, plugins and settings never leak into the harness being measured. -That also means the CLI does not see your interactive login: models are reached with the -provider's API key. `harness2 chat` is the exception: it starts your own CLI as you normally -use it. +With an API-key provider, every run also gets a fresh, empty configuration directory +(`CLAUDE_CONFIG_DIR` for Claude Code, `CODEX_HOME` for Codex), the strictest isolation. +`harness2 chat` is different from runs: it starts your own CLI exactly as you normally use +it. ## Reference @@ -169,8 +194,9 @@ person. | `max_calls` | `0` | Stop the run after this many model calls (rollouts plus sessions). `0` means no limit. | Spending is counted from the costs the CLIs report. Claude Code and OpenCode report dollars; -Codex reports tokens only, so `budget_usd` does not see Codex spending. Use `max_calls` to -bound a Codex run. +Codex reports tokens only, so `budget_usd` does not see Codex spending, and with a +subscription the reported dollars are an estimate rather than a charge. Use `max_calls` to +bound those runs. A call already in flight when a limit is reached still finishes. ### `[suite]` diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 106e7680..cc32aee2 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -5,13 +5,16 @@ files the agent reads and runs around every task. For Claude Code those are `CLA skills, subagents, helper scripts and hooks; for Codex, `AGENTS.md`, skills, scripts and hooks. The method is described in the paper [Harness²: Recursive Agent Harnessing for an Open World](https://arxiv.org/abs/2610.14224). -This page explains what a run does, what it costs, and what you get back. +This page explains what a run does, what it costs, and what you get back, including the +parts Harness² CLI adds to the paper's method: one final harness per run, `eval` and +`apply`. - [The harness components](#the-harness-components) - [One task: propose, probe, compose](#one-task-propose-probe-compose) - [Scaling with k](#scaling-with-k) -- [Picking the final harness](#picking-the-final-harness) +- [Picking each task's harness](#picking-each-tasks-harness) - [Carrying lessons forward](#carrying-lessons-forward) +- [One harness for the project](#one-harness-for-the-project) - [What a run gives you](#what-a-run-gives-you) ## The harness components @@ -42,8 +45,8 @@ For each task, a run: works on *what a practitioner knows* about this kind of task. Each writes a candidate harness. 3. **Probes** every candidate by running the agent with it on the same task. -4. **Composes** a final harness from the contrast between all those runs: which edits changed - behaviour, and how. The final harness is run too. +4. **Composes** a refined harness from the contrast between all those runs: which edits + changed behaviour, and how. The refined harness is run too. No step looks at a grade. The improvers see trajectories and results, and decide from the differences between runs what helped. This is what lets Harness² work on tasks that have no @@ -67,15 +70,16 @@ otherwise dropped. Per task and mode, a run makes **4k agent runs** (including the `k` runs of the current harness, which both modes share) and **3k improver sessions**, plus one reflector session that writes the task's lesson and, when `k` is more than 1, one selector session that picks -the final. `harness2 run --dry-run` prints the exact counts for your tasks before anything is -spent, and `--budget-usd` / `--max-calls` stop a run at a limit. +the task's harness. Building the final harness adds one session per run and mode. +`harness2 run --dry-run` prints the exact counts for your tasks before anything is spent, +and `--budget-usd` / `--max-calls` stop a run at a limit. -## Picking the final harness +## Picking each task's harness -With `k` = 1 there is one final. With more, a **selector** compares the finals' results -blind: it sees the task and each result under an anonymous id, never which harness produced -it or how that harness was built, and picks the best. That pick is the task's **adopted** -harness. +With `k` = 1 there is one composed harness per task. With more, a **selector** compares the +composed harnesses' results blind: it sees the task and each result under an anonymous id, +never which harness produced it or how that harness was built, and picks the best. That pick +is the task's **adopted** harness. You can make the pick yourself with `harness2 run --selector human`: the run stops at each selection and shows you the anonymous candidates in the terminal. @@ -91,6 +95,26 @@ the harness that improves harnesses gets better as tasks go by. Tasks in a domain run one after another so the library can grow; separate domains run in parallel (`--workers`). Each run has its own library, so experiments never mix. +## One harness for the project + +In the paper, every task ends with its own adopted harness: the method adapts the agent to +the task in front of it, and the adopted harnesses are kept for scoring and as examples for +later tasks. A project needs one harness that it can keep. So when all tasks are done, +`harness2 run` adds one step of its own: an improver session reads the starting harness, +every task's request, the harness adopted for it and the lesson recorded, and composes **one +final harness** for the project's next tasks. It keeps what helped on several tasks or what +a lesson supports as general, and drops facts tied to a single task. The result is checked +against the same rules as every other proposal. + +That final harness is what you work with: + +- `harness2 diff` shows how it differs from the harness you started with; +- `harness2 eval` runs the agent with both on the tasks that have a check or rubric and + compares the pass rates, the evidence for whether it actually helps; run it on `test` + tasks the run did not use for the fairest comparison; +- `harness2 apply` writes it into the project, applying only what the run changed, so edits + you made meanwhile are kept, and touching nothing but the harness files. + ## What a run gives you Everything is stored under `.harness2/runs//`: @@ -101,8 +125,9 @@ Everything is stored under `.harness2/runs//`: lesson. - `rollouts/`: per agent run, the trajectory, the result and the final message. - `library/`: the adopted harnesses and lessons per domain. +- `consolidated/`: the final harness and the improver's notes on why each change is there. -Use `harness2 status` while it runs, `harness2 judge` and `harness2 report` to grade the -results, `harness2 diff` to read what changed in the harness, and `harness2 apply` to write an -adopted harness into your project. Running the same `harness2 run` command again resumes -where it stopped, reusing every finished step. +Use `harness2 status` while it runs, `harness2 diff`, `harness2 eval` and `harness2 apply` on +the final harness, and `harness2 judge` / `harness2 report` to grade each task's adopted +result. Running the same `harness2 run` command again resumes where it stopped, reusing every +finished step. diff --git a/harness2/cli/bridge.py b/harness2/cli/bridge.py index 7a0879f7..f82222e4 100644 --- a/harness2/cli/bridge.py +++ b/harness2/cli/bridge.py @@ -68,7 +68,7 @@ def add_parser(subparsers): help="serve a model as a Responses API endpoint for Codex", description="Serve any configured provider/model as an OpenAI Responses API " "endpoint through LiteLLM, so Codex can run on it. Needs the bridge extra: " - "pip install 'harness2[bridge]'.", + "pip install 'harness2-cli[bridge]'.", ) ) diff --git a/harness2/cli/status.py b/harness2/cli/status.py index 7059e902..7a55b273 100644 --- a/harness2/cli/status.py +++ b/harness2/cli/status.py @@ -106,7 +106,10 @@ def snapshot(run_dir: Path) -> dict: if row["state"] == "running": row["state"] = "stopped" plan = meta.get("schedule", {}) + process = C.run_process(run_dir) return { + "active": process is not None, + "pid": process["pid"] if process else None, "run": meta.get("name", run_dir.name), "suite": meta.get("suite", ""), "executor": meta.get("executor", ""), @@ -126,6 +129,8 @@ def render(snap: dict) -> str: head = f"{snap['run']} · {snap['suite']} suite · {snap['executor']}" if snap["k"]: head += f" · k={snap['k']} {snap['modes']}" + if snap.get("active"): + head += f" · running (pid {snap['pid']})" last = snap["last_event"] lines = [ head, @@ -156,6 +161,10 @@ def render(snap: dict) -> str: lines.append(f"run {'finished' if status == 'ok' else 'ended: ' + status}{why}{tail}") elif last: lines.append(f"latest: {_describe(last)}") + if not snap.get("active"): + lines.append( + "not running: it stopped without finishing; run the same command to resume" + ) return "\n".join(lines) diff --git a/plugins/harness2-benchmarks/README.md b/plugins/harness2-benchmarks/README.md index b8532135..df458aec 100644 --- a/plugins/harness2-benchmarks/README.md +++ b/plugins/harness2-benchmarks/README.md @@ -2,7 +2,7 @@ The three benchmarks of the paper [Harness²: Recursive Agent Harnessing for an Open World](https://arxiv.org/abs/2610.14224), -packaged as suites for the `harness2` command: +packaged as suites for the `harness2` command of [Harness² CLI](../../README.md): | Suite | Benchmark | Work | |---|---|---| @@ -15,7 +15,7 @@ compatibility patches of the paper, and grade with each benchmark's own judge. ## Install -From a checkout of this repository, next to `harness2`: +From a checkout of this repository, next to `harness2-cli`: ```bash uv pip install -e . -e 'plugins/harness2-benchmarks[render]' From 1dd141fdd0ac93cccf5fb487f02d65155f940fc5 Mon Sep 17 00:00:00 2001 From: Bae Seongmin <83870598+3ae3ae@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:09:10 +0900 Subject: [PATCH 18/21] fix: clearer output found in a live run on a ChatGPT subscription (#19) A live run with Codex on a ChatGPT subscription (one task, k=1, nine model calls, all without an API key) went through run, status, diff, judge, eval and apply. Fixes from reading its output: - Codex and subscriptions report no money, so run and status say "spend not reported" instead of "$0.00 spent", which read as free. - eval and apply call the run's result the final harness, as the docs do, and eval's table is aligned. - init --yes no longer prints stray blank lines. Co-authored-by: Claude Opus 5.5 --- harness2/cli/apply.py | 6 +++--- harness2/cli/eval.py | 26 ++++++++++++++++---------- harness2/cli/init.py | 3 +-- harness2/cli/run.py | 4 +++- harness2/cli/status.py | 3 ++- tests/test_consolidate.py | 10 +++++----- 6 files changed, 30 insertions(+), 22 deletions(-) diff --git a/harness2/cli/apply.py b/harness2/cli/apply.py index 227938af..659463e4 100644 --- a/harness2/cli/apply.py +++ b/harness2/cli/apply.py @@ -91,11 +91,11 @@ def _apply(args, ctx) -> tuple[int, dict]: todo = [(rel, kind) for rel, kind in delta if args.force or rel not in moved] label = ( - "consolidated from the run's tasks" + f"Final harness of run {run_dir.name} ({args.mode})" if args.task is None and task == "consolidated" - else f"adopted for {task}" + else f"Harness adopted for {task} (run {run_dir.name}, {args.mode})" ) - print(f"Harness {label} ({args.mode}, run {run_dir.name}) -> {where}\n") + print(f"{label} -> {where}\n") for rel, kind in delta: mark = "!" if rel in moved else kind print(f" {mark} {rel:<48} {_delta(rel, current, adopted)}") diff --git a/harness2/cli/eval.py b/harness2/cli/eval.py index 2e44c836..eac094d6 100644 --- a/harness2/cli/eval.py +++ b/harness2/cli/eval.py @@ -82,22 +82,28 @@ def _text(data): def percent(value): return "-" if value is None else f"{value * 100:g}%" - lines = [ - f"Comparing the consolidated harness with the starting one on {len(data['tasks'])} " - f"task(s), {data['repeat']} run(s) each.", - "TASK START CONSOLIDATED", - ] - for task, row in data["tasks"].items(): - lines.append( - f"{task} {percent(row['start']['mean'])} {percent(row['consolidated']['mean'])}" - ) start, consolidated = data["mean"]["start"], data["mean"]["consolidated"] delta = ( f"{(consolidated - start) * 100:+g}" if start is not None and consolidated is not None else "-" ) - lines.append(f"mean {percent(start)} {percent(consolidated)} ({delta} points)") + rows = [("TASK", "START", "FINAL")] + rows += [ + (task, percent(row["start"]["mean"]), percent(row["consolidated"]["mean"])) + for task, row in data["tasks"].items() + ] + rows.append(("mean", percent(start), percent(consolidated))) + width = max(len(r[0]) for r in rows) + table = [f" {a:<{width}} {b:>5} {c:>5}" for a, b, c in rows] + table[-1] += f" ({delta} points)" + lines = [ + f"Comparing the final harness with the starting one on {len(data['tasks'])} " + f"task(s), {data['repeat']} run(s) each.", + "", + *table, + "", + ] if data["skipped"]: lines.append( f"Skipped {len(data['skipped'])} task(s) with no check or rubric: " diff --git a/harness2/cli/init.py b/harness2/cli/init.py index 34f9e4dd..bb8b3f80 100644 --- a/harness2/cli/init.py +++ b/harness2/cli/init.py @@ -107,7 +107,7 @@ def _run(args) -> tuple[int, dict]: print(message, file=sys.stderr) return 1, {"error": message, "exit_code": 1} ask = _asker(interactive=not args.yes and sys.stdin.isatty()) - print(f"Setting up Harness² in {root}\n") + print(f"Setting up Harness² in {root}") interactive = not args.yes and sys.stdin.isatty() found = {name: _version(name) for name in ("claude", "codex")} @@ -159,7 +159,6 @@ def _run(args) -> tuple[int, dict]: ignore.write_text("runs/\n.env\n", encoding="utf-8") (config.parent / "tasks").mkdir(exist_ok=True) - print() print(_harness_summary(root / harness, executor, editable)) print(_key_summary(root, (model, improver))) print(f"\nWrote {config.relative_to(root)}") diff --git a/harness2/cli/run.py b/harness2/cli/run.py index 31680703..fac49701 100644 --- a/harness2/cli/run.py +++ b/harness2/cli/run.py @@ -117,7 +117,9 @@ def _estimate(tasks: int, k: int, modes: tuple[str, ...], totals: dict) -> dict: def _spend(totals: dict) -> str: - return f" · ${totals['spent_usd']:.2f} spent · {totals['calls']} model calls" + # Codex and subscriptions report no money; $0.00 would read as "free". + money = f"${totals['spent_usd']:.2f} spent" if totals["spent_usd"] else "spend not reported" + return f" · {money} · {totals['calls']} model calls" def _stopped(name: str, reason: str, totals: dict) -> str: diff --git a/harness2/cli/status.py b/harness2/cli/status.py index 7a55b273..5ccbbb27 100644 --- a/harness2/cli/status.py +++ b/harness2/cli/status.py @@ -136,7 +136,8 @@ def render(snap: dict) -> str: head, f"started {_when(snap['created'])}" + (f" · last activity {_ago(last.get('t'))}" if last else "") - + f" · ${snap['spent_usd']:.2f} spent · {snap['calls']} model calls", + + (f" · ${snap['spent_usd']:.2f} spent" if snap["spent_usd"] else " · spend not reported") + + f" · {snap['calls']} model calls", "", ] if rows: diff --git a/tests/test_consolidate.py b/tests/test_consolidate.py index fc7589d7..b1ffd7f8 100644 --- a/tests/test_consolidate.py +++ b/tests/test_consolidate.py @@ -213,9 +213,9 @@ def test_two_tasks_retry_adoption_eval_and_json(self): self.assertEqual(task["label"], "a-one") self.assertIn("+Task note.", task["diffs"]["CLAUDE.md"]) applied, _ = self.invoke("apply", "trial", "--dry-run", "--task", "a-one") - self.assertIn("Harness adopted for a-one (parallel, run trial)", applied) + self.assertIn("Harness adopted for a-one (run trial, parallel)", applied) applied, _ = self.invoke("apply", "trial", "--yes") - self.assertIn("Harness consolidated from the run's tasks (parallel, run trial)", applied) + self.assertIn("Final harness of run trial (parallel)", applied) self.assertEqual((self.project / "CLAUDE.md").read_text(), "Use shared procedure.\n") text, _ = self.invoke("eval", "trial", "--repeat", "2") self.assertIn( @@ -277,8 +277,8 @@ def test_eval_filters_skipped_failed_grades_and_limits(self): self.assertEqual(json.loads(Path(data["path"]).read_text()), data) with patch.object(GenericAdapter, "score", return_value=None): text, _ = self.invoke("eval", "trial", "--only-tasks", "a-one", code=3) - self.assertIn("a-one - -", text) - self.assertIn("mean - - (- points)", text) + self.assertRegex(text, r"a-one +- +-") + self.assertRegex(text, r"mean +- +- \(- points\)") data = json.loads((self.run_dir / "eval/parallel/results.json").read_text()) self.assertEqual(data["tasks"]["a-one"]["start"], {"runs": [None], "mean": None}) text, _ = self.invoke("-c", "run.max_calls=1", "eval", "trial", code=4) @@ -429,7 +429,7 @@ def test_rollout_grades_and_optional_cost(self): "path": "results.json", "mean": {"start": 1.0, "consolidated": 0.0}, } - self.assertIn("mean 100% 0% (-100 points)", eval._text(data)) + self.assertRegex(eval._text(data), r"mean +100% +0% \(-100 points\)") def test_command_order_and_help(self): i = COMMANDS.index("diff") From fc8231a9607e51ebfb567713a64bf2abd5b820dd Mon Sep 17 00:00:00 2001 From: Bae Seongmin <83870598+3ae3ae@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:14:05 +0900 Subject: [PATCH 19/21] test: match the eval output changed in the previous commit (#20) The previous commit renamed eval's column to FINAL and aligned the table, but three assertions in tests/test_consolidate.py still expected the old text, so the suite failed on general-cli. They now match the current output, with the column spacing left flexible. Co-authored-by: Claude Opus 5.5 --- tests/test_consolidate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_consolidate.py b/tests/test_consolidate.py index b1ffd7f8..0c28b556 100644 --- a/tests/test_consolidate.py +++ b/tests/test_consolidate.py @@ -219,11 +219,11 @@ def test_two_tasks_retry_adoption_eval_and_json(self): self.assertEqual((self.project / "CLAUDE.md").read_text(), "Use shared procedure.\n") text, _ = self.invoke("eval", "trial", "--repeat", "2") self.assertIn( - "Comparing the consolidated harness with the starting one on 2 task(s), 2 run(s) each.", + "Comparing the final harness with the starting one on 2 task(s), 2 run(s) each.", text, ) - self.assertIn("TASK START CONSOLIDATED\na-one 0% 100%\nb-two 0% 100%", text) - self.assertIn("mean 0% 100% (+100 points)", text) + self.assertRegex(text, r"TASK +START +FINAL\n +a-one +0% +100%\n +b-two +0% +100%") + self.assertRegex(text, r"mean +0% +100% \(\+100 points\)") self.assertIn(f"Saved to {self.run_dir / 'eval/parallel/results.json'}", text) results = json.loads((self.run_dir / "eval/parallel/results.json").read_text()) self.assertEqual(results["mean"], {"start": 0.0, "consolidated": 1.0}) From 3a62c9ec359bdef5ca869cfbba2368276a79ad85 Mon Sep 17 00:00:00 2001 From: Bae Seongmin <83870598+3ae3ae@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:18:26 +0900 Subject: [PATCH 20/21] test: skip the shell syntax check when that shell is not installed (#21) GitHub's Ubuntu runners have no zsh, so the completion test failed there on /bin/zsh. It now finds bash and zsh on PATH and only checks the generated script's text for a shell that is missing. Co-authored-by: Claude Opus 5.5 --- tests/test_cli.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 7019bbc0..78c9b2fc 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -20,6 +20,7 @@ import io import json import os +import shutil import subprocess import sys import tempfile @@ -501,9 +502,10 @@ def test_completion_aliases_and_module_entry(self): for command in main.COMMANDS: self.assertIn(":" + command, script) self.assertIn("--only-tasks", script) - result = subprocess.run( - ["/bin/" + shell, "-n"], input=script, text=True, capture_output=True - ) + binary = shutil.which(shell) + if binary is None: # e.g. no zsh on a Linux CI runner: only the text is checked + continue + result = subprocess.run([binary, "-n"], input=script, text=True, capture_output=True) self.assertEqual(result.returncode, 0, result.stderr) if shell == "bash": probe = ( @@ -516,7 +518,7 @@ def test_completion_aliases_and_module_entry(self): + script + "\nwords=(harness2 run --on); CURRENT=3; _harness2_complete" ) - result = subprocess.run(["/bin/" + shell], input=probe, text=True, capture_output=True) + result = subprocess.run([binary], input=probe, text=True, capture_output=True) self.assertEqual(result.returncode, 0, result.stderr) self.assertIn("--only-tasks", result.stdout) for executable, command in main.ALIASES.items(): From 9cc2c31c0915c8cd7ba6d1d1ce0f79988cb866e9 Mon Sep 17 00:00:00 2001 From: Bae Seongmin <83870598+3ae3ae@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:36:14 +0900 Subject: [PATCH 21/21] fix: keep personal Codex hooks and AGENTS.md out of subscription runs (#22) Codex has no flag to skip a personal ~/.codex/hooks.json or ~/.codex/AGENTS.md: with --ignore-user-config it still runs and reads them (verified with marker-writing hooks), so every chatgpt-login rollout and improver session fired the user's own hooks. Claude Code's flags, by contrast, do keep user hooks and plugins out (verified the same way). chatgpt-login now runs Codex on a home kept for harness2, ~/.local/state/harness2/codex-home, which the user signs in to ChatGPT once with the new `harness2 login codex`; that is a separate login, so the user's own ~/.codex is never read or changed and its refresh token is never used. `providers.chatgpt-login.home` points runs at another home instead, and doctor then lists the personal files every run will include. `harness2 login` shows whether both subscription logins are ready, and init and the agent guide point at it. Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 7 +- README.md | 8 +- docs/cli.md | 7 ++ docs/configuration.md | 28 ++++-- harness2/cli/doctor.py | 27 +++--- harness2/cli/guide.py | 5 +- harness2/cli/init.py | 13 ++- harness2/cli/login.py | 102 ++++++++++++++++++++ harness2/cli/main.py | 1 + harness2/providers.py | 60 ++++++++---- harness2/settings.py | 6 +- harness2/substrate/codex.py | 1 + tests/test_settings.py | 10 ++ tests/test_subscriptions.py | 179 +++++++++++++++++++++++++++++------- tests/test_ui.py | 19 ++++ 15 files changed, 394 insertions(+), 79 deletions(-) create mode 100644 harness2/cli/login.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c170ec19..ce151ad7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ The first release of Harness² CLI, a general-purpose version of ### Added - One `harness2` command with subcommands: `init`, `doctor`, `task`, `run`, `status`, - `stop`, `diff`, `eval`, `apply`, `judge`, `report`, `runs`, `config`, `chat`, `guide`, + `stop`, `diff`, `eval`, `apply`, `judge`, `report`, `runs`, `config`, `chat`, `guide`, `login`, `setup`, `demo`, `bridge`, `completion`. - Your own tasks: `.harness2/tasks//task.md` with optional `task.toml`, run in a clean git worktree of your repository or in a scratch directory, graded optionally by a `check` @@ -21,8 +21,9 @@ The first release of Harness² CLI, a general-purpose version of - Claude Code and command executors. A project's own harness (CLAUDE.md, `.claude/skills`, `.claude/agents`, `.claude/scripts`, hooks; or AGENTS.md, `.agents/`, `.codex/hooks.json`) is imported as the starting harness. -- Subscriptions: `claude-login/` and `chatgpt-login/` use the login of the - installed `claude` or `codex`, with the user's personal setup kept out of the runs. +- Subscriptions: `claude-login/` runs on Claude Code's own login and + `chatgpt-login/` on a Codex login kept for harness2 (`harness2 login codex`), with + personal settings, hooks, plugins and instructions kept out of the runs. - Any model provider: Anthropic, OpenAI, OpenAI- or Anthropic-compatible servers, Vertex AI and Bedrock, through provider profiles; `harness2 bridge` serves any of them to Codex. - Layered settings: defaults, `~/.config/harness2/config.toml`, `.harness2/config.toml`, diff --git a/README.md b/README.md index b33d4a77..633145bd 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,9 @@ uv tool install git+https://github.com/3ae3ae/harness2-cli # or: pipx install git+https://github.com/3ae3ae/harness2-cli ``` -By default runs use the CLI's own login, so a Claude or ChatGPT subscription is enough. You +By default runs use your subscription, so a Claude or ChatGPT plan is enough: Claude Code +runs on its own login, and Codex on a login kept for harness2 that you make once with +`harness2 login codex`. You can use an API key or another provider instead; see [Configuration](#configuration). Runs make many model calls and use money or subscription quota: the [quick start](#quick-start) shows how to see the number of calls first and how to cap them. @@ -138,6 +140,7 @@ harness2 guide --install claude | `harness2 judge` / `report` | Grade each task's adopted result, and summarise the grades | | `harness2 runs ls / show / rm` | List, inspect and delete runs | | `harness2 config show / get / set` | Read and change settings; `show --origin` says where each value comes from | +| `harness2 login` | Check the subscription logins; `login codex` signs Codex in for harness2 | | `harness2 chat` | Talk to your agent CLI about tasks, runs and results | | `harness2 guide` | The guide for AI agents that use harness2 | | `harness2 demo` | A local run with scripted editors and no model calls | @@ -161,7 +164,8 @@ channel = "claude" model = "claude-login/claude-opus-5-5" ``` -`chatgpt-login/` uses your ChatGPT subscription through Codex. API keys and other +`chatgpt-login/` uses your ChatGPT subscription through Codex (after +`harness2 login codex`). API keys and other endpoints work too: Anthropic, OpenAI, any OpenAI-compatible or Anthropic-compatible server, Vertex AI and Bedrock. To tune an agent other than Claude Code or Codex, the `command` executor runs your own command and edits the files you list. diff --git a/docs/cli.md b/docs/cli.md index 983305c3..c5f9319d 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -195,6 +195,13 @@ rubric, or to explain a run's results. |---|---| | `--with {claude,codex}` | Which CLI to start. Default: the improver's channel, if installed. | +### `login [codex]` + +Without an argument, show whether `claude-login` (Claude Code's own login) and +`chatgpt-login` (the Codex login kept for harness2) are ready. `harness2 login codex` signs +Codex in to ChatGPT for harness2, in its own home, apart from your `~/.codex`; `--device` +uses a device code on machines without a browser. + ### `guide` Print the guide for AI agents that drive harness2, followed by the current project's diff --git a/docs/configuration.md b/docs/configuration.md index f2757184..7679b621 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -52,14 +52,26 @@ all you need: | Model | Runs on | |---|---| | `claude-login/`, e.g. `claude-login/claude-sonnet-5-5` | your Claude subscription, through Claude Code's login (`claude auth login`) | -| `chatgpt-login/`, e.g. `chatgpt-login/gpt-6.1-sol` | your ChatGPT subscription, through Codex's login (`codex login`) | - -`harness2 init` picks these by default, and `harness2 doctor` checks that the CLI is logged -in. Runs still keep your personal setup out of the harness being measured: Claude Code runs -with `--setting-sources project,local --strict-mcp-config --no-session-persistence`, so your -own `~/.claude/CLAUDE.md`, skills, plugins, MCP servers and settings are not loaded and no -session history is written; Codex runs with `--ignore-user-config --ignore-rules`. Codex -still reads a global `~/.codex/AGENTS.md` if you have one, and `doctor` says so. +| `chatgpt-login/`, e.g. `chatgpt-login/gpt-6.1-sol` | your ChatGPT subscription, through a Codex login kept for harness2 (`harness2 login codex`, once) | + +`harness2 init` picks these by default, and `harness2 login` and `harness2 doctor` say +whether they are ready. Runs keep your personal setup out of the harness being measured: + +- **Claude Code** runs on its own login, with `--setting-sources project,local + --strict-mcp-config --no-session-persistence`: your `~/.claude/CLAUDE.md`, skills, plugins, + hooks, MCP servers and settings are not loaded, and no session history is written. +- **Codex** cannot switch off a personal `~/.codex/hooks.json` or `~/.codex/AGENTS.md` with + a flag, so harness2 gives it a home of its own, `~/.local/state/harness2/codex-home`. + `harness2 login codex` signs that home in to ChatGPT once. It is a separate login: your own + `~/.codex` is never read or changed. Runs also pass `--ignore-user-config --ignore-rules`. + +To run Codex on your own home anyway, with whatever hooks and instructions it holds, set +its path; `harness2 doctor` then lists the personal files every run will include: + +```toml +[providers.chatgpt-login] +home = "~/.codex" +``` Subscriptions have usage limits. When a run reaches one, it stops cleanly (exit code 4), keeps everything finished so far, and the same `harness2 run` command resumes once the diff --git a/harness2/cli/doctor.py b/harness2/cli/doctor.py index 5da655ca..cae3c435 100644 --- a/harness2/cli/doctor.py +++ b/harness2/cli/doctor.py @@ -16,9 +16,7 @@ from __future__ import annotations -import os import subprocess -from pathlib import Path from .. import providers, registry from ..core.proposer import Proposer @@ -130,16 +128,21 @@ def binary_check(binary=binary, channel=channel): "detail": "; ".join(problems) or detail, } ) - if any(config.provider(name).type == "chatgpt-login" for name in referenced): - agents = Path(os.environ.get("CODEX_HOME") or Path.home() / ".codex") / "AGENTS.md" - if agents.is_file() and (text := agents.read_text(encoding="utf-8")): - rows.append( - { - "check": "Codex global AGENTS.md", - "ok": True, - "detail": f"your global AGENTS.md ({len(text)} characters) is part of every Codex run", - } - ) + if provider.type == "chatgpt-login": + home = providers.codex_login_home(provider) + files = [ + str(path) + for name in ("AGENTS.md", "hooks.json") + if (path := home / name).is_file() and path.stat().st_size + ] + if files: + rows.append( + { + "check": "Codex home", + "ok": True, + "detail": " and ".join(files) + " are part of every Codex run", + } + ) if args.live: for (channel, model), spec in sorted(models.items()): diff --git a/harness2/cli/guide.py b/harness2/cli/guide.py index 240adff7..24b09160 100644 --- a/harness2/cli/guide.py +++ b/harness2/cli/guide.py @@ -47,8 +47,9 @@ ## Workflow 1. `harness2 init --yes [--executor claude|codex] [--auth login|api-key]` creates - .harness2/. `--auth login` uses the agent CLI's own login (a Claude or ChatGPT - subscription). Change any choice afterwards with `harness2 config set KEY VALUE`, for + .harness2/. `--auth login` uses a Claude or ChatGPT subscription: Claude Code's own login + as it is, or for Codex a separate login the person makes once with `harness2 login codex` + (it opens a browser; ask them to run it). `harness2 login --json` shows which are ready. Change any choice afterwards with `harness2 config set KEY VALUE`, for example `executor.model`. 2. Tasks. `harness2 task new [--files] [--check ""]`, then write the request in .harness2/tasks//task.md the way the person would really ask for it. Good tasks are diff --git a/harness2/cli/init.py b/harness2/cli/init.py index bb8b3f80..b39aeebc 100644 --- a/harness2/cli/init.py +++ b/harness2/cli/init.py @@ -37,7 +37,7 @@ PROVIDERS = {"claude": ("claude-login", "anthropic"), "codex": ("chatgpt-login", "openai")} AUTH = { "claude": ("your Claude subscription, through Claude Code's own login", "ANTHROPIC_API_KEY"), - "codex": ("your ChatGPT subscription, through Codex's own login", "OPENAI_API_KEY"), + "codex": ("your ChatGPT subscription, through a Codex login for harness2", "OPENAI_API_KEY"), } CONFIG = """\ @@ -264,6 +264,16 @@ def _key_summary(root: Path, models: tuple[str, ...]) -> str: loaded = S.load(None) lines = [] for name in dict.fromkeys(providers.split_model(m)[0] for m in models if "/" in m): + if name == "chatgpt-login": + home = providers.codex_login_home(loaded.provider(name)) + ok = providers.codex_logged_in(home) + lines.append( + "Codex is signed in for harness2." + if ok + else "Codex is not signed in for harness2 yet: run `harness2 login codex` once " + "(a separate login, so your own ~/.codex settings and hooks stay out of runs)." + ) + continue if name in LOGIN_CHECKS: argv, cli, fix = LOGIN_CHECKS[name] ok = shutil.which(argv[0]) and _quiet(argv) @@ -289,7 +299,6 @@ def _q(value) -> str: LOGIN_CHECKS = { "claude-login": (["claude", "auth", "status"], "Claude Code", "claude auth login"), - "chatgpt-login": (["codex", "login", "status"], "Codex", "codex login"), } diff --git a/harness2/cli/login.py b/harness2/cli/login.py new file mode 100644 index 00000000..3cc39362 --- /dev/null +++ b/harness2/cli/login.py @@ -0,0 +1,102 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Show whether the subscription logins harness2 uses are ready, and set up Codex's. + +claude-login uses Claude Code's own login as it is: runs switch off your personal settings, +hooks and plugins with flags. Codex has no such switch for its hooks.json and AGENTS.md, so +chatgpt-login runs use a Codex home of their own, signed in once with `harness2 login codex`. +""" + +from __future__ import annotations + +import argparse +import os +import shutil +import subprocess +import sys + +from .. import providers + + +def add_parser(sub) -> argparse.ArgumentParser: + p = sub.add_parser( + "login", + help="check the subscription logins, or sign Codex in for harness2", + description="Without an argument, show whether claude-login and chatgpt-login are " + "ready. `harness2 login codex` signs in to ChatGPT in a Codex home kept for harness2, " + "apart from your own ~/.codex.", + ) + p.add_argument("service", nargs="?", choices=("codex",), help="sign this CLI in for harness2") + p.add_argument( + "--device", + action="store_true", + help="sign in with a device code, for machines without a browser", + ) + return p + + +def run(args, ctx) -> int: + home = providers.codex_login_home(ctx.settings.provider("chatgpt-login")) + if args.service is None: + rows = _status(home) + ctx.out(rows, lambda data: "\n".join(f"{r['provider']:<14} {r['detail']}" for r in data)) + return 0 + if shutil.which("codex") is None: + print("`codex` is not on PATH; install Codex first.", file=sys.stderr) + return 1 + if not sys.stdin.isatty(): + print("Signing in needs a terminal and a browser (or --device).", file=sys.stderr) + return 1 + home.mkdir(parents=True, exist_ok=True) + print( + f"Signing Codex in to ChatGPT for harness2. This login lives in {home}, apart from " + "your own ~/.codex, so your personal hooks and AGENTS.md stay out of harness2's runs.\n" + ) + env = {k: v for k, v in os.environ.items() if k not in ("OPENAI_API_KEY", "CODEX_API_KEY")} + subprocess.run( + ["codex", "login", *(["--device-auth"] if args.device else [])], + env={**env, "CODEX_HOME": str(home)}, + ) + if not providers.codex_logged_in(home): + print("Codex did not finish signing in.", file=sys.stderr) + return 1 + print("Signed in. Models written chatgpt-login/ now run on this login.") + return 0 + + +def _status(home) -> list[dict]: + claude = shutil.which("claude") and _ok(["claude", "auth", "status"]) + codex = providers.codex_logged_in(home) + return [ + { + "provider": "claude-login", + "ready": bool(claude), + "detail": "ready (Claude Code's own login)" + if claude + else "Claude Code is not logged in; run `claude auth login`", + }, + { + "provider": "chatgpt-login", + "ready": codex, + "detail": f"ready ({home})" if codex else "not signed in; run `harness2 login codex`", + }, + ] + + +def _ok(argv: list[str]) -> bool: + try: + return subprocess.run(argv, capture_output=True, timeout=20).returncode == 0 + except (OSError, subprocess.TimeoutExpired): + return False diff --git a/harness2/cli/main.py b/harness2/cli/main.py index aec4bc86..400c0237 100644 --- a/harness2/cli/main.py +++ b/harness2/cli/main.py @@ -39,6 +39,7 @@ "config", "chat", "guide", + "login", "setup", "demo", "bridge", diff --git a/harness2/providers.py b/harness2/providers.py index 96053697..f17b7ba3 100644 --- a/harness2/providers.py +++ b/harness2/providers.py @@ -21,9 +21,10 @@ import re import subprocess from dataclasses import dataclass +from pathlib import Path from urllib.parse import urlsplit -from .settings import ConfigError +from .settings import ConfigError, state_dir PROVIDER_TYPES = ("anthropic", "openai", "vertex", "bedrock", "claude-login", "chatgpt-login") USAGE_LIMIT = re.compile( @@ -53,6 +54,35 @@ class Provider: api_key_env: str = "" project: str = "" region: str = "" + home: str = "" + + +def codex_login_home(p: Provider) -> Path: + return Path(p.home).expanduser() if p.home else state_dir() / "codex-home" + + +def codex_logged_in(home: Path) -> bool: + try: + return ( + subprocess.run( + ["codex", "login", "status"], + env={ + **{ + k: v + for k, v in os.environ.items() + if k not in (*STRIP_ENV, "OPENAI_API_KEY", "CODEX_API_KEY") + }, + "CODEX_HOME": str(home), + }, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=10, + ).returncode + == 0 + ) + except Exception: + return False def split_model(ref: str) -> tuple[str, str]: @@ -182,22 +212,20 @@ def litellm_params(p: Provider, model: str) -> dict: def check(p: Provider) -> list[str]: - if p.type in ("claude-login", "chatgpt-login"): - argv = ( - ["claude", "auth", "status"] - if p.type == "claude-login" - else ["codex", "login", "status"] - ) - login = "claude auth login" if p.type == "claude-login" else "codex login" + if p.type == "chatgpt-login": + home = codex_login_home(p) + if codex_logged_in(home): + return [] + if p.home: + return [ + f"chatgpt-login: not logged in at {home}; run `codex login` with CODEX_HOME={home}" + ] + return ["chatgpt-login: not logged in for harness2; run `harness2 login codex`"] + if p.type == "claude-login": try: result = subprocess.run( - argv, - env={ - k: v - for k, v in os.environ.items() - if k not in STRIP_ENV - and (p.type != "chatgpt-login" or k not in ("OPENAI_API_KEY", "CODEX_API_KEY")) - }, + ["claude", "auth", "status"], + env={k: v for k, v in os.environ.items() if k not in STRIP_ENV}, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, @@ -207,7 +235,7 @@ def check(p: Provider) -> list[str]: return [] except (OSError, subprocess.SubprocessError): pass - return [f"{p.type}: not logged in; run `{login}`"] + return ["claude-login: not logged in; run `claude auth login`"] problems = [] if p.api_key_env and not os.environ.get(p.api_key_env): problems.append(f"{p.name}: set {p.api_key_env}") diff --git a/harness2/settings.py b/harness2/settings.py index 90f5d9a2..5aa93fe0 100644 --- a/harness2/settings.py +++ b/harness2/settings.py @@ -103,6 +103,10 @@ def cache_dir() -> Path: return Path(os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache") / "harness2" +def state_dir() -> Path: + return Path(os.environ.get("XDG_STATE_HOME") or Path.home() / ".local/state") / "harness2" + + def parse_value(raw: str) -> Any: """Parse a TOML literal, falling back to an unquoted string.""" try: @@ -241,7 +245,7 @@ def provider(self, name: str) -> providers.Provider: configured = self.section("providers") if name not in configured: raise ConfigError(f"unknown provider {name!r}; configure providers.{name}") - fields = ("type", "base_url", "api_key_env", "project", "region") + fields = ("type", "base_url", "api_key_env", "project", "region", "home") return providers.Provider( name, **{k: v for k, v in configured[name].items() if k in fields} ) diff --git a/harness2/substrate/codex.py b/harness2/substrate/codex.py index 2b8518fb..591d6254 100644 --- a/harness2/substrate/codex.py +++ b/harness2/substrate/codex.py @@ -248,6 +248,7 @@ def env( stripped = providers.STRIP_ENV if provider is not None and provider.type == "chatgpt-login": stripped += ("OPENAI_API_KEY", "CODEX_API_KEY") + home = providers.codex_login_home(provider) return { **{k: v for k, v in base_env.items() if k not in stripped}, **({"CODEX_HOME": str(home)} if home is not None else {}), diff --git a/tests/test_settings.py b/tests/test_settings.py index 25b3fbc8..978c30c0 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -65,6 +65,16 @@ def test_defaults_access_and_copy_isolation(self): self.assertEqual(resolved.get("executor.editable"), []) self.assertEqual(settings.load(None).section("executor"), settings.DEFAULTS["executor"]) + def test_state_dir_uses_xdg_or_home_without_creating_it(self): + with patch.dict(os.environ, {"HOME": str(self.root)}, clear=True): + self.assertEqual(settings.state_dir(), self.root / ".local/state/harness2") + self.assertFalse(settings.state_dir().exists()) + os.environ["XDG_STATE_HOME"] = str(self.root / "state") + self.assertEqual(settings.state_dir(), self.root / "state/harness2") + self.assertFalse(settings.state_dir().exists()) + os.environ["XDG_STATE_HOME"] = "" + self.assertEqual(settings.state_dir(), self.root / ".local/state/harness2") + def test_precedence_and_origins_across_all_six_layers(self): self.write(self.global_file, "[run]\nk = 2\nworkers = 2\n") self.write( diff --git a/tests/test_subscriptions.py b/tests/test_subscriptions.py index 4adfcc1f..976e41e1 100644 --- a/tests/test_subscriptions.py +++ b/tests/test_subscriptions.py @@ -51,6 +51,8 @@ config = home / "config.toml" record = {"kind": kind, "argv": argv, "env": dict(os.environ), "cwd": str(cwd), "stdin": "" if status else sys.stdin.read(), + "agents_exists": (home / "AGENTS.md").exists(), + "hooks_exists": (home / "hooks.json").exists(), "config_exists": config.exists(), "config": config.read_text() if config.exists() else None} with open(os.environ["FAKE_RECORD"], "a") as log: @@ -123,12 +125,15 @@ def setUp(self): path.mkdir(parents=True) (path / "config.toml").write_text("private user config") (path / "auth.json").write_text("private user auth") + (self.user_home / ".codex/AGENTS.md").write_text("Private user instructions.\n") + (self.user_home / ".codex/hooks.json").write_text('{"hooks": {}}\n') environment = patch.dict( os.environ, { "PATH": f"{binary}{os.pathsep}{os.defpath}", "HOME": str(self.user_home), "XDG_CONFIG_HOME": str(self.root / "config"), + "XDG_STATE_HOME": str(self.root / "state"), "FAKE_RECORD": str(self.record), "CLAUDE_CODE_OAUTH_TOKEN": "fixture-oauth", "ANTHROPIC_API_KEY": "ambient-anthropic", @@ -294,7 +299,12 @@ def test_chatgpt_subscription_executor_and_improver_launches(self): ], ) for record in self.records(): - self.assertNotIn("CODEX_HOME", record["env"]) + self.assertEqual( + record["env"]["CODEX_HOME"], str(self.root / "state/harness2/codex-home") + ) + self.assertFalse(record["config_exists"]) + self.assertFalse(record["agents_exists"]) + self.assertFalse(record["hooks_exists"]) self.assertTrue( all( key not in record["env"] @@ -302,8 +312,48 @@ def test_chatgpt_subscription_executor_and_improver_launches(self): ) ) self.assertEqual({p.name: p.read_bytes() for p in home.iterdir()}, before) + self.assertFalse((settings.state_dir() / "codex-home").exists()) - def test_inherited_login_homes_are_preserved(self): + def test_configured_chatgpt_home_is_used_without_writes(self): + for populated in (False, True): + with self.subTest(populated=populated): + home = self.user_home / f"custom codex-{populated}" + if populated: + home.mkdir() + for name in ("auth.json", "config.toml", "AGENTS.md", "hooks.json"): + (home / name).write_text("fixture " + name) + before = {p.name: p.read_bytes() for p in home.iterdir()} if populated else {} + adapter = self.adapter( + f"custom-{populated}", + **{ + "executor.name": "codex", + "executor.model": "chatgpt-login/model", + "improver.channel": "codex", + "improver.model": "chatgpt-login/model", + "providers.chatgpt-login.home": f"~/{home.name}", + }, + ) + with patch.dict(os.environ, {"CODEX_HOME": str(self.user_home / ".codex")}): + result = adapter.execute("one", None, "custom") + self.assertFalse((result / "codex-home").exists()) + context = self.root / f"context-{populated}" + context.mkdir() + Proposer(adapter.spec.channels["proposer"]).call("mission", context, "custom") + for record in self.records()[-2:]: + self.assertEqual(record["env"]["CODEX_HOME"], str(home)) + self.assertEqual(record["config_exists"], populated) + self.assertEqual(record["agents_exists"], populated) + self.assertEqual(record["hooks_exists"], populated) + self.assertNotIn("OPENAI_API_KEY", record["env"]) + self.assertNotIn("CODEX_API_KEY", record["env"]) + self.assertIn("--ignore-user-config", record["argv"]) + self.assertIn("--ignore-rules", record["argv"]) + self.assertIn("--ephemeral", record["argv"]) + after = {p.name: p.read_bytes() for p in home.iterdir()} if home.exists() else {} + self.assertEqual(after, before) + self.assertEqual(home.exists(), populated) + + def test_inherited_claude_home_is_preserved_and_codex_home_is_overridden(self): context = self.root / "context" context.mkdir() for kind, provider_name, variable in ( @@ -324,14 +374,19 @@ def test_inherited_login_homes_are_preserved(self): }, ) adapter.execute("one", None, "inherited") - self.assertEqual(self.records()[-1]["env"][variable], os.environ[variable]) + expected = ( + os.environ[variable] + if kind == "claude" + else str(settings.state_dir() / "codex-home") + ) + self.assertEqual(self.records()[-1]["env"][variable], expected) Proposer(adapter.spec.channels["proposer"]).call("mission", context, "inherited") - self.assertEqual(self.records()[-1]["env"][variable], os.environ[variable]) + self.assertEqual(self.records()[-1]["env"][variable], expected) def test_login_status_success_failure_missing_cli_and_timeout(self): for kind, argv, login in ( ("claude-login", ["auth", "status"], "claude auth login"), - ("chatgpt-login", ["login", "status"], "codex login"), + ("chatgpt-login", ["login", "status"], "harness2 login codex"), ): with self.subTest(provider=kind): provider = providers.Provider(kind, kind) @@ -339,9 +394,17 @@ def test_login_status_success_failure_missing_cli_and_timeout(self): record = self.records()[-1] self.assertEqual(record["argv"], argv) self.assertNotIn("CLAUDE_CONFIG_DIR", record["env"]) - self.assertNotIn("CODEX_HOME", record["env"]) + if kind == "chatgpt-login": + self.assertEqual( + record["env"]["CODEX_HOME"], str(settings.state_dir() / "codex-home") + ) + self.assertNotIn("OPENAI_API_KEY", record["env"]) + self.assertNotIn("CODEX_API_KEY", record["env"]) + else: + self.assertNotIn("CODEX_HOME", record["env"]) self.assertTrue(all(key not in record["env"] for key in providers.STRIP_ENV)) - problem = [f"{kind}: not logged in; run `{login}`"] + where = " for harness2" if kind == "chatgpt-login" else "" + problem = [f"{kind}: not logged in{where}; run `{login}`"] with patch.dict(os.environ, {"FAKE_LOGIN_RC": "1"}): self.assertEqual(providers.check(provider), problem) with patch.dict(os.environ, {"PATH": ""}): @@ -359,6 +422,33 @@ def test_login_status_success_failure_missing_cli_and_timeout(self): self.assertEqual(args[3], 'model"\nnext\x7f') self.assertEqual(tomllib.loads(args[5]), {"model_reasoning_effort": 'effort"\nnext'}) + def test_codex_logged_in_checks_only_the_requested_home(self): + home = self.root / "login home" + with patch.dict(os.environ, {"CODEX_HOME": str(self.user_home / ".codex")}): + self.assertTrue(providers.codex_logged_in(home)) + record = self.records()[-1] + self.assertEqual(record["argv"], ["login", "status"]) + self.assertEqual(record["env"]["CODEX_HOME"], str(home)) + self.assertFalse(record["config_exists"]) + self.assertFalse(record["agents_exists"]) + self.assertFalse(record["hooks_exists"]) + self.assertTrue( + all( + key not in record["env"] + for key in (*providers.STRIP_ENV, "OPENAI_API_KEY", "CODEX_API_KEY") + ) + ) + with patch.dict(os.environ, {"FAKE_LOGIN_RC": "1"}): + self.assertFalse(providers.codex_logged_in(home)) + for error in (FileNotFoundError(), subprocess.TimeoutExpired("codex", 10), ValueError()): + with ( + self.subTest(error=type(error).__name__), + patch("harness2.providers.subprocess.run", side_effect=error) as run, + ): + self.assertFalse(providers.codex_logged_in(home)) + self.assertEqual(run.call_args.kwargs["timeout"], 10) + self.assertFalse(home.exists()) + def test_api_provider_homes_and_ambient_oauth_are_unchanged(self): for kind, provider_name in (("claude", "anthropic"), ("codex", "openai")): with self.subTest(kind=kind): @@ -453,13 +543,15 @@ def doctor(self, adapter): output = io.StringIO() argv = ["-C", str(self.project), "--json"] for key, value in adapter.settings.flat().items(): - if key.startswith(("executor.", "improver.")) and not isinstance(value, list): + if key.startswith(("executor.", "improver.", "providers.")) and not isinstance( + value, list + ): argv += ["-c", f"{key}={json.dumps(value)}"] with contextlib.redirect_stdout(output): code = main([*argv, "doctor"]) return code, {row["check"]: row for row in json.loads(output.getvalue())} - def test_doctor_login_status_and_global_codex_instructions(self): + def test_doctor_login_status_uses_dedicated_codex_home(self): adapter = self.adapter( **{ "executor.name": "codex", @@ -468,24 +560,15 @@ def test_doctor_login_status_and_global_codex_instructions(self): "improver.model": "chatgpt-login/model", } ) - agents = self.user_home / ".codex/AGENTS.md" - agents.write_text("Global rules. 한글\n") code, rows = self.doctor(adapter) self.assertEqual(code, 0) - self.assertEqual( - rows["Codex global AGENTS.md"], - { - "check": "Codex global AGENTS.md", - "ok": True, - "detail": "your global AGENTS.md (17 characters) is part of every Codex run", - }, - ) + self.assertNotIn("Codex home", rows) with patch.dict(os.environ, {"FAKE_LOGIN_RC": "1"}): code, rows = self.doctor(adapter) self.assertEqual(code, 1) self.assertEqual( rows["provider chatgpt-login"]["detail"], - "chatgpt-login: not logged in; run `codex login`", + "chatgpt-login: not logged in for harness2; run `harness2 login codex`", ) code, rows = self.doctor(self.adapter()) self.assertEqual(code, 1) @@ -494,19 +577,8 @@ def test_doctor_login_status_and_global_codex_instructions(self): "claude-login: not logged in; run `claude auth login`", ) _, rows = self.doctor(self.adapter()) - self.assertNotIn("Codex global AGENTS.md", rows) - agents.write_text("") - _, rows = self.doctor(adapter) - self.assertNotIn("Codex global AGENTS.md", rows) - other = self.root / "custom-codex" - other.mkdir() - (other / "AGENTS.md").write_text("custom") - with patch.dict(os.environ, {"CODEX_HOME": str(other)}): - _, rows = self.doctor(adapter) - self.assertEqual( - rows["Codex global AGENTS.md"]["detail"], - "your global AGENTS.md (6 characters) is part of every Codex run", - ) + self.assertNotIn("Codex home", rows) + self.assertFalse((settings.state_dir() / "codex-home").exists()) self.assertTrue( all( record["argv"] in (["auth", "status"], ["login", "status"]) @@ -514,6 +586,47 @@ def test_doctor_login_status_and_global_codex_instructions(self): ) ) + def test_doctor_custom_codex_home_login_and_instruction_files(self): + home = self.user_home / "custom-codex" + home.mkdir() + adapter = self.adapter( + **{ + "executor.name": "codex", + "executor.model": "chatgpt-login/model", + "improver.channel": "codex", + "improver.model": "chatgpt-login/model", + "providers.chatgpt-login.home": "~/custom-codex", + } + ) + with patch.dict(os.environ, {"FAKE_LOGIN_RC": "1"}): + code, rows = self.doctor(adapter) + self.assertEqual(code, 1) + self.assertEqual( + rows["provider chatgpt-login"]["detail"], + f"chatgpt-login: not logged in at {home}; run `codex login` with CODEX_HOME={home}", + ) + self.assertEqual(self.records()[-1]["env"]["CODEX_HOME"], str(home)) + for files in ((), ("hooks.json",), ("AGENTS.md",), ("AGENTS.md", "hooks.json")): + with self.subTest(files=files): + for name in ("AGENTS.md", "hooks.json"): + (home / name).write_text("fixture" if name in files else "") + before = {p.name: p.read_bytes() for p in home.iterdir()} + code, rows = self.doctor(adapter) + self.assertEqual(code, 0) + if files: + self.assertEqual( + rows["Codex home"], + { + "check": "Codex home", + "ok": True, + "detail": " and ".join(str(home / name) for name in files) + + " are part of every Codex run", + }, + ) + else: + self.assertNotIn("Codex home", rows) + self.assertEqual({p.name: p.read_bytes() for p in home.iterdir()}, before) + def test_executor_usage_limits_stop_and_resume_entire_run(self): adapter = self.adapter() self.stop_and_resume(adapter, "executor") diff --git a/tests/test_ui.py b/tests/test_ui.py index 53a2549e..021f6190 100644 --- a/tests/test_ui.py +++ b/tests/test_ui.py @@ -114,6 +114,25 @@ def test_apply_json_reports_the_plan_and_refuses_to_ask(self): self.assertEqual(plan["changes"][0]["path"], "CLAUDE.md") self.assertFalse(plan["changes"][0]["applied"]) + def test_login_status_uses_the_harness2_codex_home(self): + from harness2.cli import login + + homes = [] + ctx = _Ctx(self.root) + ctx.settings = type("S", (), {"provider": lambda _self, name: None})() + with ( + patch.object(login.providers, "codex_login_home", return_value=self.root / "h"), + patch.object(login.providers, "codex_logged_in", side_effect=homes.append), + patch.object(login, "_ok", return_value=True), + patch.object(login.shutil, "which", return_value="/bin/claude"), + patch("sys.stdout", io.StringIO()) as out, + ): + args = argparse.Namespace(service=None, device=False) + self.assertEqual(login.run(args, ctx), 0) + self.assertEqual(homes, [self.root / "h"]) + self.assertIn("claude-login", out.getvalue()) + self.assertIn("harness2 login codex", out.getvalue()) + def test_guide_mentions_every_agent_contract(self): from harness2.cli import guide

VZXazi< zE*n7xTTo6kh72P3{ZhmiAs{HszoQ?4*CD+#5RkbY=Nn!T63G>VW?>~`1CzWrm?ww> z0Qm%~Qa092!Y2{rf;H7Pyx`X}(jV|$Www-xY1K47OGFt;Udo{Flc8~?3Sm;oxFVw> z`}}~i{1ydwDW6I72Yfxfiw!*-(C2ognwF)XA*T~-E=Ld{6Dy1HnK_GlQa{-du6|%C z{~YSj+oe%^)u)`v_{?V}&Qlt$y#4JNckO3-4wAr@Ya_Gn@|e3k>Z-tj>iE&Iw~j@2 zPaT`?_-W_+ozwg7@iE8tkMoPed9UoA&99E-S4Z<|BGOdlR6#U{M2uE9I4rA^0T+4F zA6?Qgb!A$dmZw5f{n7kwvzDi0mZ$%2-ig+~WHuGwb$Yl-_x5G_H=8$XFVX*-5+mXZ z+%w3~k#P`;NGh!yg@#`f9bv%@jLP}U$owsGU?{8>5KWB_60gDF57VvEzE>;OUNwc zhs;@NSF1`%hK!U~4*a7=Wn&H>_Dm|W{{ULu{|#Ctih1)ts#V9=)s9(rvYxqE=zc>J zUh%X}Sp(_w>THvUON}#;)CGv7 z3=m1>1&=f{6N*fsR6X3lG;dUL`x_#Hjr{K5BC3e3KxDx;t#5&JN=acdss<)Rl5U~d zb}5YT%iP7Hqf;Hh)0&JaO$25Nn9h(mjQmbB`co&s{)Gr*r8Q8&B79Uyiq)}uA3SIF zoeA0dst?vb z3&e)VLw!3FY&rv--~hZeS(or_3+ofBQW*^q*bp>g7W;gcm?J=Hlu74weBIJ+4 zu)pX0g+PM<(+oN*Ub`X^)TDvZZG4s9pn$NMM9OnTEXX8@B&CqvMBue#2>(dr$8a5F ziR4%+D;_Ry<8#u~r`RxB@PjxudK9gnvr8eAQ$_v=8A=KHz0N4*kDbNgqDWQTSu?hC z&Ql1Xl{07Z*la;{te`rYUjrI1f66q~6V2Teb8d#L4Z^x-AgpuekWEmuL=)D{xHoI0 zZ|Q7aRV=S6n!6(MY$Oml2jZ}P7m|%lx(7C*=*6uSkcO8p%*mh`l)JEzGhis zSw;9LVpS_)4^rU$rQVQ}`_RU@^24S#9j`kg&;GzYwf=r_<6}D-t=pZ*E4j7x=GF-R z$}`d2N>*87WnJV7Vht-2E4-1NQ|0q!eNO#ndLsyShaxDQu~$#6pAN)px2SdJmVity zL@D7I8~Jhmqui3&-0E0v^=xi^EVq7oeKfZ|p1bLxQJ1s%Z}kRL*2v`)&2Qoyxnuhk z5urwBV$C!omMe@}eLRAPu0RJ9@T`SufAwPv<@ z{bmZq*n(RYqL`tx^$M)WibS+jf;nRnikSCn zT5yBx%L*l~z`Xmq7quwjDr!acxjkB82!N^PHrxh=%ke(ScFA3q) zGXpXKDvfMx1;I4biqM+LLR{YjVeK%)w6>Jbh7mp+_TbUf31>Hk5Md}lkg1ZCOkoIF zq(WQZdzyeCgQFCM0M|rV7;7xAe4%HsLGd8M06E3AXfwsuQ$Q-mPyQF4piL`(Cb;4Z ztAwdSm8uo~IYqdnJQvAb@dQeOq$R2^ouxuq*^35MRsDrXQy?D&r&uI8CsFe zR9DKMTVbLPQkvm{50nyd*5uWAPPQV(2#PdOk*gk6HY4zEJSR&@WZOrFl}L;+?xBan zy#@BOudknMiL8kne{0KZ<)&EWrn`AFmD}#++d`Rm@Wr>6r0&r#hzFX4mhC;a~j@#hiHZ)XisR z^H#<3R!yz@d0yi~djT77PUFG!`3aerEkQl>__Sb) z9^9OuoxXG#oQaez2+nU-Tair06nQk;k976Yn(WM^3Q@rBXYCPc%fdz~kOKWOE}&Af zvWgNcZ2Fx6BC6R#a|ETK@!b@?0$B2-aztdO%YoX^%8@+{Nj?@m>WBcTpxb9^O53RY zF>Ta`Y}xEj0L4-}0g$mOLvh8yJ|rcKLCTs;yOEF`eWRU)2P(s7CY&ih(sJcnW|+1z zk=LVV6`5&D9JQZ84T>k}lWX9w4}F7Cy!3>=z6Q!E>l(1$ReiO0FRVoVEZ<=y3^VgU zjaq`Ob=I7;aGCSY8wdU;*=P3Qt4GVMJ@SY7WtQ5tw=LgZ!zrRNvpt zddf0Rp602L@l8|vW_;6xOMynU)o~|$s%`K-fIp+U6V-OvEg@uQwpz`de!oz(s18T= zIzo=r?H^fZ^@V~btcy-`!Cb(A8rW4)AqT`Uo?!bS@(dQzUh+&IM$Em!_AjQHeL}at zU&h^F(65nLq=ZbjC8tz4RN)TW*rnX(KSyH=+Jo@8bCl9 z6+IYn?mg_r0BVBW0+z0pAzGCV*Yt!;ETAdU=yEC4=7V~TUHgNs(j{uDe4Cl}2K96U z&?Yp@T!@2uT+Xz>9FqXlIQ_GNeKQT=BW#J_RiKd z$LgBzRX(WO`Re7jTEBQi#O?P=s`xj|RXpx!0Vj~UHVXs;Rw-%jXUGJ7a@7K<*ZHujK`%wR2 z?Y`U3-l_UR`p>8z=H~2bE}1pV^q$@l<`$e`IToSbFeWRXVg$dLr5Gdb6c-yJuNY^< zs6O&Y6WP5K%;H@78Kp9skYKywlTO-EMxH6Ul~?f-`q-g9cAJgYl|dUWBx@9!@GUCM zWKXnaQjCIw6cB}|c%>0ZDU-dZMkZ3^TZ#b6qV6r}FOj2*rmY<(^K)okayi(JwQ64I zPqs7r7@qCxzc(_u?z_*YxW0s|@|~6`!`r(fTcQom+|%D3y0?>AfYrv_wNY0cRIAPg zXjZK@XjaK@e#L$Gz>7b=c<17Ky=om~F($;^LL>)fV^LQXWHu`fkj%z;khHQWYvb~l z%;tIT=Xoa%y*GOI*gf&X*65+5@y26gI};IaXXEwV zW33-8B4kOpRes-IPPA!gA{5^7z+Lf9aJuI1x(7AS%z5BmN8) z&gX%EC4V}Y@=>#q^Rv~PV%3|X)vX^E&F(%K+kG zJeOBE8F*#$TyDYSxmRlE3Ly!5r3WA5g#a7~;(2Wk?It^2H{@cglDeE=u4HMP4iSQQ z$$pUjyC+gMIU&B)U>;mEjyC`BY;n{gbGv-r(FYQm0t ziGN9#>G_qmqnprtz%ssRP&SBKR3r;EFKSYrY~3rpK%r1xDIyxVDn$JWR%ic z&G?qC32lEB@q?Y$6XBB9hh-2E9L7c{5RW>B$K=yAglazOx2U^KdKLXbFLMob5eR)v zr}>#fl>^iTS876C&OwywXJe(vKbPS6$)ZDP!Ez5O*|w zQ29asy<;Dqi$8TFzV>L$arB|x>@dx9=9F9I0>o@bx7f)d7AW&nrCD2C@}ix^v{kCe zUqT6~I9pn_Zz9A`x8XZ3T13N(2D-pNxyqvl4EJfE8+h*Pb#So&0nbF%>h66-cijVF zmtj=TSZ_`EmhK5yIH0hU2OvBUarK>%G>c^LfV6KhL6jlHuIs`1V$4zy3^ZH~jM**) z30a}aOOJSQNi3$?w~SmGkxyQIWN)kH0h6~K!l^@s40~UY-&xptl@Y+aK#aw%I05sN zX)~IJSzzVJN?_5W)}Xenz9BHIW|y02nqOFXrbnp3s*pW>7eR*4sm2;*l+nJ?c;N(+ zf=$XAr!P)P#cUAAS%U`-!a-n|_z2SV)$j3jcl#t5CXy(dhC5mK2(gc)jMX%|20lwwi%8#>%QV_*xjHnrjCCXTGm#_4jWOO?%??VcJn z7vO4sh=-(vCsWZk^uMORz7(5gJr)fce^fWhr*5N3{lawW+LL?}gjd(lW6<6t9;A`V zjKT!);{%MaOY+@GU2lI1`l>L>$g&2DMfiP4a!()w__CB~f>^syGkmJ*>VMpJr|rGn z58S8k?0%~nWXHH`q1Qhl1O%}A1&zUQ7z+d+*qzQ!R5R5gA?CM!%7L2%pr(cx*3(~y~q|io68kxdI`kz$t5tTHcW*K1* zTr~d`eSMz3@@JF$kkS#0x`7^JtZ3C{O|oI0{sqm|3IwOcaH7) z*y))pxYn5{TsBu$0i+dpdozZ({@oFh3YW*rwmvi)2!e=-`Hnp5!@6ol{$-+(a)t600gE|{}*Ck7Le`gNjTcjErJaP+12Fz zr?H0^k|wrAqI{Aa3xcnVw_!E1uM|@eI?KB#9Y=Lxx{vDmdkR?VRx*vH*0Z#@oXVzD z{H$)mL0{U?#S0J_pbWeDBk0kpNZsI@@iG(A4&wlG7mRQ8egIp8$fG zrjT+G9oR@s|{PNC81#cHK9#ut*6g zwkp?)KP?p@75_N-4-i6n!1E86nRvr5Jse;87o7c%IqRQr<$uC0Xa9;nGUEcz$-MDh zFMMar0JxlAcE79@a4^bw=1rVc7&GFd&OTW+!x#MzT_ImL&mnlE=XAyM^z-3bN?kL< z7tHHCyc?H&&+F-3u|j#T+orttwzKy~#?nuXNT&)4S&2v0Jb!|JWHIx_kMi^Rryn)w z_z+KC-#y z(cw-WvtcQlH?ydPD_T8oWlb8`L)5tTMdO o-NMp&E5#s}SvGH{n1d>%+!!A!awGOA$lH0-XG`;WpplEZ6g$Yl9+!)+O| z?)ST+qaHmS4WD)Wo^1GWkVo)ui{lTDdM~yebw7<@zH7j;>8~c;)ePAjob--0am(8q z%peIBch^?s1vN2O7eKtIp`^jlP}87~#LF6N4VH$gg5%aLX;B)bA*IqB2Lk>lBunA) zUsgP=AdA|6zLM19(poQhDJ$)CJ>tq<52Ah4Dk=SdZd8i?oOmN;C%l6Iy!K34r=Gr8 z#sP%axyg;dN=1l^gfDjKov01!F=`@*Ul(uKr(tg6yD39SOzoQKF4aGeL nKMNS+Uw0Ai5)p%?1+p6&|G!Vw- diff --git a/harness2/substrate/__pycache__/_native.cpython-312.pyc b/harness2/substrate/__pycache__/_native.cpython-312.pyc deleted file mode 100644 index b860f0bb77385a23719770c83c9bc8cae1174994..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5433 zcma)ATW}NC89sZHR$ARGncy2(fC&-7Hg`zyz<>>eOKcJoC?Zr9+O;iNmNaKqz9_4X zCesHiu{^LOCD^f8MQ!Y?Tzl6edBqBc&DUps+Beb8U zNSTQ;enx+?epY{Seh!{&)HGuDo56+?O;LWt;D zmuQ8jQ>+sOc)Bb|bX(BMKhY!Fe#ZIh#Cp*VR_+PLw zQkg*tMqotLgg&79k&o@gWNE2pV=KIu!?7Tq@WL*@7Qqg_>U~0%=i@X^9*Roi@JK|% zie{3BV6USZH#QUw4Pi5}VKq~^yJi-pfnYMKXlxJ%bhz$lX!G|$NtW-9qq=-<-r^IFmD%$wIPIvVpW`|g?U?>{lOCBN@vzUfrKaXQbRF52qzf_FyA9nKw@ z8HZw?^)A5-=ot)4wZSACRq9@Ge(V}apnRVqkN{` zl%>6W6b=fU=Ij$f7d}SgLPJt$SaXDe2_=c8Ks>1=l8Ru06F44%L)93ZjA`b+e%-4yXMeRte^J2~29`qDNir8?8Y7K|s*u8N z^2(x56#y{cR;ahZPacMh;J`C;EO)#hv}JlqG{gC+MczH_$U5?CU(VH&>>bGN?*5$3 zpZn1x{=&Z&ZJulXg*Bb?YdZ77_HPkIx)5|6xOeP+*XwhG`2)TAroMvXO#a-3Jb&Te zC5D(hmDj+0zgV{o-Dy0$j`@u99ByDfYv7>#3{t9WaUaZ=BV|>#sIdg4QkYW#8^bTb zJ_t3w5SM#8O4AB)NB6)ywXTQnMgGoMtXZ#_6EK3#7Hz8 z8wRkLHTt!)nmrcBBf)4mB?Yh))vV`&(WF$y25}6HO9ZhZYu2zFsB|?riN*1_0{DVn zEE%8VU`l!sCMv_mro%=O1H=Nq3~k%sXCTLFTd+3GTbpvb3f9(4_ZNa=W>d~{b?c&| zA>Vkg;5d}$4;2Mxrf1PrKhyud<+}5lbCNAOJu_nV9ViIS=|kB=4~3RR_qx3Iz`gE! zd-CfK7u-kkwj(7os`Hk3WOHRs;d+?#8IX`8)ahg+u<&)Hq6oZLnR?JPJ(&-YB@=%-h~9dYWLY?)UhT9W`z)IgrhnIr;sP2iZGhLP|5XBZ4VJ z{KH7qpk@VRT}OZ<_6L#3O`BFAkxRpwRgh>-lNt?3TL6smDqo>7TBTw_EhMUx${R>Y zTU1M=DvFFH)#wN}Eh9Qhq$-1ImNjBDa1^6e%OI7us#aLZ46H;@h38Qob^y%@t6@~f z{YGdbQE5e5M!R(@nxJZU9e5?Dn)PvxDMY*IDMF}VMFRifj9tMSp4qMH`^efAouDeR zB6r7R^i)}u)BPt6vXPp~s7!o6bO19rKaJX| z34uUjA`}D~4FsNYNoAmYzh=f#A_^||5v}o3EEE^Ru|drw5#22N?7H{pRI8bQwZ$+7 zQ5naIBx)QUjK(i(W_V4!219s4ng9&0Yx#f@*Z80uNW|svc-eV7iGc-6K{236`7jL)Ud*v^O{(NYD>&c?4q3CX2aBrD+Z@JAD-0chQz4Pw91^52R?xNE@BV|=E zSwS4lV0t@LXx_Qd+%@0a1tRBMAm7|oX#U9zGt+&QFV;2A*KLM=*PU0Kv&=(R+czH6 z)CN(^-B@a>P{}%jO0L?v4*?y(9vF#)1wEjR>r?^(sRA>@fIJP?03Gt2))a6pgg#JJ zVG{Vj0m)5hnOh*0F+l%5JyoX%LS_)cApDkvH&_E zC~P&#F?!>*&@QYP8me$lrtef&rs12bYUBb=&EBcHJ=`Pt$SNeGnnkMS z`e4OHsuWXO73@>Z*%cPm!uaz~;Q6PQg6o1YF6`bb`uY5y`$jY-d=XL7E0VFZ@GYkQ z`7H;CL{Lql`PZOGb)f)42#AaC#jqTNUnSWA$j zV>qlpfG0QGs07^M6Et>Mnvit@!-vQ?9V9zUvL47ZQ+YUz8yP0tW+fv?Iz3^+44Nq! zBMR4NDbo|D>jRCw40qwOzyL|B49U2UjCh7*yGcffNhU$A%0LwbD~%_BJu5Qf0vKE( zmUVEjAT##7`l-pP>&rbaH)crku1>2eVJ$<)Bll|AW<&@d}g4;K5_vQEW zWsc@;XNnG2=D40MHmuJjZ@qKlotx=L4eezeQgnjO&8EuIwkz8;bucI1irt9a9h?j2 zVukf*@@FsJUw;vFNs+fr3t3@-Z=B~FXWQ=9=lRA5{9f|fnRVvOx3^3>AMiViypvqT zX5@!_Gm#}7-#L)$g&FaB_)7Tw;n|~qIQhGizdt?y!uChb9UyyFO#kzPMVo8-RQA-= z>qVPqx;NXK+x5`aTCyP5&XN^1wBFt~FYL;*yT1Rn71`gQbeZ#Eea}7?uA<~lsWI>m zh4RJHPpT#?!Ob?7vP1z07oqJ^#SoL`njyYOtl8dpOacQEW-!rB<(Q(I5(E7mw3Ar$ z)bZNcj{dR~?N3M{T~IzX!^^WIZs_LZ{}w)~*Jmw9FLI8{#y$qslv7D2q7qrzHg88q z$0dB6ymeKV>s7sEQOm~Tmkn`_j7Vm|=c^?$bE;kphxG5==B*BO$>-6`fq)nf1p=Bi z5Ew}6m;L}Iw2sNY1Tdj2+yoi%A!R5UzN~+;#)jfpDobXfu{BmU!sjmCXU*$Vsj6W~KiR%Eg-2~hfOyNlYbuj#+}L&<9Z diff --git a/harness2/substrate/__pycache__/base.cpython-312.pyc b/harness2/substrate/__pycache__/base.cpython-312.pyc deleted file mode 100644 index 4605aa7db2d54b533d5ddedb931d95ccb2bbeb1a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3794 zcma)9-H#L36`!%k-`)?0rI3cCHz7^qhP8o^mI4V~NJvp(p=?mAiO}rbnQQN4$1~2o z<7MqAQqs_;zVL=B^`Yuh5qZd8(U-nZK_xm;)vcTA11Flq(YxyGZ zFvYsexb>6cv(hq1pRbb>P9&?2QZQ3Yz?QhjlUXH4dwK)(y+fc~f2v?|jL4^v|32@)$$Pc*S3v zj~SowN;mR$(VGS9S$3&ksn?Rym~#d{Aqr~sf`E-9WQT{4LvAERLm}N8<24ujHL=Y6 zG?i$Th#5^|p%9)LdEcTScSocPE7K=y80OGI`zTn1>2%@wF_8>&k2D(Yah;Y- zA%{Dhy~Ng$>mV`&vrh1M6$g)1Z7gbFZiCpA2wIBHW0DX`3M!?lL2ayfW7PEu#)rgc zAEuDHq|xNFTg@+*zz#xF$5>g)I>sk-nGeCd>u{zY2yk>)NVcpU=cnvNG$586&|FVsX-gX0m>up0?+g1z@%9Ic7){r6K?x>2ABo~a-Ln;|WT=K^u!loY{h z%j3;)ZRl-@6zN`RN-tp z#)kMe5rVib%3QXU6XQ(RrAVU(HtA=VZTvKrToK?m38N;X`#TEJ(T#gZEx-up5t31g z6Nnu+0o4xbfXSeWRz~M8r@_(T8@R{0RQSe5X}Edtj^4)=L4ae`FDPtlx%cWvKNmXKCF{A>SBLH{~>nt@TiMEH|o%V z$K&nCS}-p{Mxd6_>h`rEVT6w$fNl{Kq}WxDL}&^sUFN6=I!+h0PEr~>Bq@!3A&CNZ zgFHiJ1Tf?82z|v5p4^GtT|=*9`3~s+ljeT;Bf$e?n69TN)d8n~k3-QX9z`03@ku5* ziW2x8zAth3GoS0H{Ks`E#$Nq97fDZzo{}eU=BsDAaSr;u9wKU4aJONJUemIJM zStkcf&}FHMn=tI`p(3P@irQo;yqR zLG_GQAh8zVXKe6aulAQ{?GX~kNSq)+waM-~{{`wJ(Zs*{IRxGLZ*P{TJ~}_N%-kh& zn9t>^ThrOx&&&t=pW4dm#tZYK#_@$wd&pL9t{*9UYX=s=iI!6Nje zW=B#lLA8tY#lc^#q>|#Plxtp-Z(9k;S7YIx zvGA#J@L}G_RBk?h-`w}_OwK&DWk3wGMrQvOeLkF>9Ier4Z9H1rnl-9(H|^i-yE|9^ y-LX%s+b8bMz4FJ(_dY0YVb?vMXPCKQ-YgCcdegE!Te|CC=S-vW>W2mG@&5qg?)1w5 diff --git a/harness2/substrate/__pycache__/claude.cpython-312.pyc b/harness2/substrate/__pycache__/claude.cpython-312.pyc deleted file mode 100644 index b632409b037821dfedd3f7582581873add745b03..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 17276 zcma)jYj7LKncxiGZ-4;!0AGUmkO+w&C0TD%QYey=C|MLOQjQ5(f*}k@fdT<~W+;gW znB``DwNy@fB4bw(uE=XR>s08q>cZ^0YL(jTMRu|kX1-- zms9@S_w`@~AWbnV+x;wMpzk9`jjMB`BWrV`_v@Y_%x8KBHA&XPe*Xo5&f9KX8>5kXd}ik zlg~t8UBosW)Yfuy+$*3#sR65 zDTlOzsf4tOaY0(m)IeIx)InO$Y=P8G;H^vpz>SO&(k7-E(rwUV<5h~;{#W#SuuAwq zX$#|lbO*uS$+QCOC2$+l{u=$CXkQbv%h$~8_HAR1FneBA`L;7hnP(txVU96-A@?Y# zlN7V>Rf^gFo~pCMY%qBSM>+@tB`BY2FAz3(UH$^uJEph zey@k2Er*rtM06~`UPFB7JOsIMK%B?GcqoXkU9sqJc%+AAW9(bL&Z9rL~oSi1CfYFDH!~N z=P&ei`A?i36x2AtxS;d%p_{xvHY}*3fw7RF36kX#^nu8Yz$E97MLqNreb;lH7rJ}e z#u&jYm;1*8H^X1LOHqQeSQZS7`(YJf3F8;#M(!}@j>G~?h;iSz8iTzENBNjLbUnmQ zy15X?!J4*`9Vpp4_l*D>g?+gagT30jt993|UGCu302>JMu(pDpyB3Z_{7jgAVrdhn z+)j!(!75ha*ZA3q2&@Qn!LqO?9O4Tmo(+$TgjoMon1(rO}$HKf&9)uV8O24>FT#yZq^W30N!G@3?{2@S{ z^vgRc7(@)R)K=-E3atc(xaR(72+n9>4FxsFPh8>Nr95h~l59PEX>pgxDst>TEG(D% z3-LS<5l7aEJq&fh0!IOmqBQWQq$qj~*cfnQUam_}e1S($l*BEj=mb5gChgWG?-kxS zMVK0(^c1g^sz-GaRLr3dWAROb82X+<;&oC!rQjV>eL}%IC0LpdXW`4FvIPB2ic!5O z_whA4p%|bVDc&Su@D>RgRy0v;#guYGYh^q60BjDf{{;zT+GCg#2#h=dvU>49^7hu!fS_t*r-yMwVP9|%X? z0aEUcUBScPF|s%vLGK^v8652GJ2@a|LN~)4&j~7WqSZ*&K#f51u>ZhF1j7q~$V7G2nN-~kXN@h>-8rN6&a=0ky?rESEt}Ej z6-xC3pmHjrzsPWPR3kC~G-9qkOEPOck@iyL*?7?p2E`IhZ_=Za9*z zKAN>0OY4s1jF#!szk95w%$0XdX=C$mDVp?FK-|B`Er!z79a+nf2hXH+Cw`w-V*L}Y z5@z*IbEk*;z+*jbQGRf+?6_X}pY*H`v69@Du)gRG^O!mdG*7c!I&%P(aiQ z>^VUv)C^TkAq=xcj6X12sXe29Q(5TY540%u&uDODMvIK;P1OM9(e=gaU+VOFy)UuH zUW&G~?KtMS3~w#R4wHgJlP6ww8SmD^7u_%Mmv#V#r-?=O71xkrw?G@>254dsU*l@> z;?WBVHYBLW0(|hQKo1D?SwX{v0xZ0#N7&fJINlp@7h#6?m=Yu~kA_7W5L8!U%%or( zW`RjT2Ot3mdUC(x-OPR&P}mnC;qbU3&v3ih=dhj8FW%YMoP7yO@Q(u-8;RZSoNt+H zN!N7TXMTMAC*2=(|7_1M_WpG52WPVO{^>IxYwfGfiuvBT-gNz*#WO2=dop`_9t5&` zPyMXn7fnBH$~w=@=sz|(ldiY6ys>4WEbH32;@X{Y?Ot|0lXFzgoPoNWbIXdeIpb_j z1+vbTg+t#x_U^Il&W?<;W2Pr(uUoMKG zN9x4F-i2+c_KaiqV)uQ=edprojN@q9bTnsmq@6ovudP&U&s1z*ptBV_($<}enniwb z`2O*9+p&!G*n_4AUs*Z%V&>$F=^xFi$AmqhpDLttKiG>nHeZC?aT4(g6vaJ&paG!36hB z?BW_Aj3OpY_(#rUneNG%Y|}lfRn6~27ny9;vny4dnX1l@s=98OW|T8uhI{|@wiWx% zjD6>laVN+S=01A0qI!OGZZuVyt?}Yx4aQ~Sf9{B!&2M4mxp846} z@~-o%6%4MmM#Kb^0kT-|iOmNJ#i zHqUgVRh7ScTn@7V;e<@HF!Yi8!TT4|4PDua?uU(O+gTJ&pk5SC{-M0frTCFc-*r&^ zqlU7sXVgD>Mh$QfR!M#eq5{3P2u1T$i1Gm&@hKP@tS1VeDuh-OutG4(SJhxQsbEx~ z$!nQPMh&?R?Iob^S1~%!_w|g6(L<>LY!n8_jZ6(=gxmzy3KQgJJxKMX|Cl<)4CNNa z!dUOpr8Oc9_6-~KYh|`DWq@Y`ONSluGG;5|fZRT*@iYilQ7HxONt7hx2i9q+aO^;= zjzvJmWZdH+)(a}Xdn6LO5{RHo4Z8gV$e*C$wy{;P0PEy7q-ernkq+jOaw$Bg;xkFT zOQWGdc_ke6C4ww!{o!bs_xs~j8|K+2)x3&rkQSyXI4Co!ztFLl z;BB3Pg*=_i1jT)jAR!f>0g8fGIvivXov*@^30ic{4Aof?urg zG0J-~kCr&`luAz}sLdo!UY0XQWreLc1>PpX30mgcsxWm=ElIO9RiZSic=q=1v{tGq<_Q&}y{D6ANmE~=WHnOXCm4go z9OOq8Exv!l78h;gj^w{8`yH#uLZz+Y|zG|}HIeY7Da@(Tn zk!fGfRF+pm=@Sl*{|CCx%HwY8Lw7@$LW!(OCz_B#jLhZm+{)DbEnr-w+G$b#1DOV- zmiG!g0Zbk;%|eYrI-66fA_XT@qq1BGV-%jE1}9OBD05D!6Y9;VSklHg`nD4=WSW-x zLv2jz>5DYNDGg|F>QR~Mi|s*91FdXI3$?ODTC7c|h?IL8YMsSW31T#)_E}z*Cr4$< zmT;&N16V>UtrWYpbOjR{seDxC1jY7@meJkQOBe}dLQAB|y9vcKefuroH+4l^34$IA z9EqlGgUrcqk!$4jn$j_bgzh@UUf{RNwNQq#E#DxONaJ#LzF8{cw@DDJo^c&7_eeFP zJ0z%><9K5KVVW`BGfRCl7NQlG>Km=aRuY6?olt@XZ0++Xi-t>LsALa8r9kgsUxqxc zZ|^+W(>DkrEPDmsS!@ZW{?K>?{0r|Y*dzF^5&0GoJmZzJ+`L{|W|2041!}_JZ2*z< z9K3oeS+pC742xLbG#r@vLyVwiQLYfQVbR1a7(tvRz6?Jf6LbO2KOW=4H$hSWW(fQ> z06q*JIMl>2@L6^m`b6s+Sau_^8zF}6!U7FAFF?;GhOnYwEy~%)a2B12rDi6^#zl$T zqbJ;uJ%yFGF(Fd6reM|W#gZ_#(FX(37?}S85kV)}+lUlG*klQ-437~{26ETWv&xdn zdFYip3dauRi0V~Sb+S9%FqnP;*`m!bqe7lopJdWad$VQxW;Cl#U~+GDywS0s%+|Di z(VVR+XRlbXH)QM$sl8cy%Zhz>#=bjie+HEA>gE*lo$%Y?)Wt=8 zy6W&u@258Bym8K$yuM^>&N-{*&&-`k?OSrTpn2fz+}U^97Cp<(j$b#lE%1v2*@iv3}K2m%N&~k*RNg-?{AA56ZwhL#Z?Ab}%iT%vPLA+fMx&W@eZ( zBu}PZTo}nT?@!kpct9`Pjz1X8*m_}XbJ^?lD`h(}@ZYo(hPF9pjH2`<$xuke$ZO%9 zUISm{I{sYbeMDzhfrgFv zS{gPy;6Rma3N#f3{+-Y>G@(&_LPsRQt$)BWpl1~Klrj%c5gwo_8rre15{*~^TC}Dp zMS%&OkS?K6_8WjJ5~q$uHBh7%Q3oB5jf-mPUt$Fz8oUQQ29Z#ZJVa{239wf&K_OpI z#v+WMhr~}5X!dO^)Qp8#lqU@EA`^-zX|sOz5|QV9K;UK|0Sc*XN)2W!cYq$M zo|#y6ZAfx=4climvp43fIcs^cYRS5FwQ>7*4!?bPVfg;@%Z*@X$~2ysRlgq2*xaku znq+sXdBKsjdeb_uc-Bx|ksjP>a@I}*0Fpj6DxW9`g`&bl6B?DyXtAQ?HohlIyfg)x zlA@2D1U!#|tP=Yz5k_4_(4(_}1K(O`o#JF7#>Rq%AFtggqm|Zv2;FcEkbsb;Gp*>V zGP6xZ*vM@gBJ!&U#O!bybgaz3Zt9niQ$)GQ2$Pi(wr${o2i}^AhzWz>T$j z71(!S2)MiOppYdHj3}Ohq>ExJRFL)176fhfuK|t2Vd0>O=2BS(NB`{S*F=)gvGyOa zu?LdUHF2)EJQfus1_7U|vuJDe>tzK_zf*+sYy~8mU zgz-2mlkEAA>%CrZ7hj2iRViR0s(hI)MaWAO&V`1X5=Hg{uZ{ce5ak%?g=Bm6AArocx=FJFMD6>0nMaV{(Yx89Y*(q~H-hq&lGTZaz2vvZ8 zD_@CF6=inhT?kQTy+Pc06--KcWH#_Wp#+FRxfvUPD*(N%1@=J`wy+tQNJ4x!bw_na z9aM(F$Z`n;UFp(FsCaqo%?!=gX%PQi!2Z>*DQ?%PsGxF6J*E0KUA(e3b88hZw{OEa zu-XSu-lygx9M7^xrmshB4hMc?Q zJC?UC%kEvvmgh6>U2mw8baMA%=MRQfj`U}a^e^r0pLu0=>n%R*-nC?TKCgr}zqVAa zSQ;{xhSb5Vr8TdHB0P`ocN{6-Uv2q~7T+nA-thl^X?dQ*#a*cGY}b9DR73h%8dd|O zW&<2}4x$bwqCr{VM1CeKoNjn2RXEW!4hoJ6*k<8G0B2BuxDWU6l|oN~9>QAu7-_<> z$mGxxAqE2#0yku$6`2N!%wc~5m@LvBX|s`Dt)hFJ4WLgw#!kk!Y+ULM2>${-an+Da zQy|vb>T;&a)S>(Ohog^C)&&!QL{tUh08DbPigzn2iR(XGx-(>Jo?r$8NB~SW;3Wsq z9h4S4J?wvlr1X|#{~Hv<>o(4IUE9yF4QkFPm)9mQX04vI&hz=JA9n`+=(Rs)db%J) zyZ9$KhW$BqkDpn%q}VjTpEq_&`2P#hrP)*_jah3`TGu4brc~6x{U@{81;Dz+qQ3a% zL2(1l47p8!m*Y@`m$PUyEj5fnAXsQP5+1qA$6Xs2P_Fvl*a*c%SUg?XY!iAUlcyf( zn#qj8>e>KxFw8h)><#tRhoH2}Okd1|j@@_HSi~jIEbP zgt`6$)-7Z5Um%IwB@W27^%r;{`$McHos;I4;?!{wV00t_5dmphP=c@{67~vKAV**c zgw4erQI)iLda0luhFBTA{s|8GRtH4ifdmS1YJLdKL;ER&s{w5C`(K#|L<%)pzaN5P z{C?3DZtCkC?0uof-`_cSYJg>N7897@olk^5QT)S8kwwEIi&_EuAtwKX2~sqnA$AF& zgP5Q@fJlJs7($qQ75=ydNM5BrR&{=AbU;LmW6Sh&k5#m~a!o_4>(?|!wLM==outpu z|4!AYook0p>bA97T78;Egh5CxYD3)YX=KTXC|5R^UQv=?bg;wvtftnvfvQ7L5occP)hAd+a0`<>> zcW85!17wX7WpOhpgHeHlw!m(u=n-&XLxh+fhTHJq!s7#m$HN#V0uF4WSOVTdlLQs> zKi+{2QpygHlw?;yNguN77|d}U-~t9S7Q~d{F((hY0j3m;ro?%tb;OwgViAQ5APpunSxUR(_A;5IsdPhgXin7j!IFq2+vu4Dr@h$j~W zW)fT}sIax57T-`F038ROf|>*J7Ya!PmBssO4gSb!2QddCy^iJ}&?|Z?ni^RQ&qTA9 zFRfTkWh|#2R-~UF$XW(xlpv~DY||%lHpjeW&hnAXJ)?ql`tnsU51pPno#eC5rkNhN zx*;koba(KPu_0Hmqc=Y?x$|bKY%56BCj0bR^tqZG7!Ys*B`G^esIik+@bu? zOauI(Lyc*T{`hwFhudgK*&jg@_Q#OO1OgWzgX!`j=-9QFxp8Uslq; znlYoL|7oMl(ziWN8Ao3ADIKi29n>>hk%cC7MGKz9jyVPN)}PRAy627&*^W4+FX=~L zWNK1BtcG!dm9ZQd;s#t4(a0m=k}07hLXMwo6M2EU7j#(I>RF67~y*> z$Qe7CFfx_kJL!VlNlFwM*(DZz6BUbVEJPdy7Y2@_C3ZOM+UKN0dSz}aWz zd0mEa>d1VCivH`(d{>*#lBo~NyTNR^=ayD;%KVfQwe>0eG%$@)Tlk8mlCLm>0$#YL zF``jg2l$Fd@D~k zV>s3A(wxb8+b``hc{}oy8F@-pWAhm}Hyycz2f)Y?6>$99zgJPrj?LoPiek%Vv22nM zG-XLxIC^qtF#;hWuWUkbts5?+oe2vA5g!o4x&ze;zOF!M)M&i~74yyeiGwC|uT}ac z>;>);3#j(qKK57e#(oM(ytZXH0MP;rgs#D_djgQl`uJft`!9eZ&>nD?O%nGRgtc5D z@`?(>YvPp`Pj#N}>lqlh1ixJg^Pnsd1Kkl86_dEy9lXIDfrx+`3~FcGM1&V~;A$h1 z4?Bk_x{E#M&!0Pg$sIRc>>L>A9T@EF8@%KOu}B{PYa-Zb!B?l6fIxS46sMxXV0us% z$owBe3Hw_}pa5ZrMS$O~Tm!?dpqD!n^yhmzA*3Io^x1z$B>G}pamfw1@M{uKr9m7D zN5SZM0xTM)P!ytbLjC|342R?Ti=ADAz32KaxeL*@*dnfgRYAE(Fp>_$&x)6wJ3i2J z{squM&h@#;Cke)qqPX_r1*nIf&Oy_7V@ovVCejx936>~v!3vhFy$7a4M?E$ohp{1O zMMOIm<6u~nb=jiKhYBEycA%FoC76T(gifUg4wR{<1mJ^LJvf+EGD8J^-~-b z6Ofj{P7cBh*vTq}N|&Q?Benlq)h>|dPtyUFiOKB&wd z=uPiG{jfRRe_^G6DAPZb_W85@FQ?B1(sfssZNZ$$xv{tEWouKevUYxQZZcH`exs$z z;JvGb{;IMEPG{TBK+(gAUrhdVGCg!9doDPuDh!;wc~8B-EcRwwj;6OCOV@qrk*zaV zRx$6Ib0rThmo-UfiF-YZ&G&WRYkzPk+j=&=voBqDZrRokB5lrIIq#YCBzqwcdPWOV zOINmLjIHmVzJKFqs^z_B35sie=iJU@c)^~vx6Wu+jpef=ce!O_W6r&8QJbken65Z9 zb8^+{N^XZ>N^1+WSuL-f=|LxI=$*l3#}0J*y6WF@z2Qpj&$@QZoQ5Fk*{>{F8&dJ~ zzTV9C-duD0;{K)PLvz>ee`Qu#SdE3l_bZlm9(%jufiGLxU;OA-QjU~5b$+u?FHhDi zl{GC?FUFTVovW2K^YOWOs`)#fw>=9d7td!K4nCMzt~^`pYI33cyRLU#i~H~I&w7rh zTe=?X`ElrBcl!KG*%O!2J(tsU{ztZ#b7kepif`1XPA__wnvUhXd%ipL?$Ba<*?T-w z;dy9=35(|-DbGyaKv(v_nQYs055w7(7t`B^(sjO%Y!^v?uVa7PmYWVrjc>d&v2cO( zd_C(0?@f05>GZZU>AL4WvYq|y7OJ8y-%i<^@_jV;lF8{U&sw*vSerA}=B#!5^r?@H zj+}Mxs@5=b@Xw}ZCzrJDoV7A(NV?K+TA7S>$35SQ_h`m@^v8||FaD@5{k$*J>09<* z%vdj`br*ABW;pkGTOI0eL`7kNnBcd#^u|DR2tS7!d_e0?fiMO?71>nOlCL;MNv{9q zqK*G4Eo3)0kFy|;BwnzZgpyIBkPBho9ZzXr9Fc_jfDZrs9~@KG=6PLVNLgrt_OFx( zno35E<8S({a)^$!HV|^J+APFLx~!q^w-Tx;`u4ZLf}r?Q_}LEr`~g2&W|E&`_v;EG{t)bcLS#y1AXV&a}f5-g`KOFl0(1XeBk>{5OzWivr>BWKYVI zscl`U-Ib}`wdl&$KAUxPKO-1ok@gO*f z&#>%gW6dS!OFq4#g3hSKTt!W)FhJs0(3otcV1ZxIf;&O7!jDM(f)@*JOYo<15OMSw zqLUk*91>zi%|RJ>rtxU7-^H9xiSqXop>I;3;>wR%UpL`A*!Le^`kN` z+@|Y}0{nt6=X8lb-f7D^+uPAy@ z!OD)-aq`LX7;ytJWFyWcutFCre}esyrFtks|0qMH!x!6pK`B}9c6JqdK$G+E1h^8w zU{qoEwGycQuHUEP$|BV?kiTk>Xv zEL3?@-inZovODu-2-&HMt$7DRPAn@&sDi59ny*Bt3NGfn3!!QXLd$9ps-?=Rf2%O) z4Ud(kGEKe?%l1=XWnp~IAIYyMe;Iw)6t{s=-xDWf@m44`RCzMIzA6w@Y* z2WXnsuW1alb*o8JwFpW_IU$qG+3`91Xh+O$!ainy z->t6dmT2r`{@B90b-V8S^?kqZyWhS3fzxT@@Z92mIAA)+aX+UY>SI+r{K9v6j=RQ* zTsJ53qAtu2>biA2`SoFax1PNX-3InHb{p|FgiV9yZZnHx3|j`R-B$QbqB(3Ew0GN? z-x78VI=fxWZ-w98?O}dfxMN<+CEX>1rQM~2W!+_i%et2hmUox4FlV@8 zu(G>yu&TRiu)4dN=X6}qD>jEX(IvXytc)Hh8jUpx&Cj*9cXc)liW_VVfxciQhEgM<(7@uLHbbGHD8!;Hr}I(itUN>o zQQ8f{`Tn336iz8ogBOCmg1<%Bbza_cN;ngWh_Ve;9L&}tFBU?>fj;CNL7RJ#4;9E- zLv4po3I;@fC>9(P5MdzL8;eRKf)tF1C{Ajy9Sur;F*p>Cjtru{f*3k;1_6;?^hp5q zITr{I!yAeS2b5Mf1TP>Td4muMX8W=?9FXOp++h2VW`d7T+XInEG!}@3q7k_rzYRwN zv3|VGhXX@Hp-A6{G-dGHIwVEUg-{Qf0;S=|U_d%c@ncdjh_}5{8HvuJVDE=CJ$=Ts zwYNXmd$tEdnzq2{2}phC(vIMTp{Nw=ITH#8<+MG5qR$0;D3`P|JDN(EPoFkt+mW^$ z3Jgb(o0PV73`aVGFQc8Yv>_5045p226w=l}_My=O3RA@uA@ zj!T#1zV!x%deG}~G!hSKiG&dKBK1F}!1?}YnC20tLfi4Sj+cbkuoO{-P8t7zD2Ay2 zFwI6VZm1~cGct$?GTIo4;cFl!Y~HkC`?ejz=@Epbt@7DWINT$Kq(`pot%|ccEqGmUtdt_=|`2~llwv!sHD><1J#K1VdZ2a2;mUsKk}ik zr!G>4qo-&>Yt7ss1RMGqgqlzF2P48Rnul^IHq7P%<|6ulRYmB>EX7#u*?^fCzyeVg zOSZm^O2&c(<&=6+G22RTVcC1f8%z+fnrF2X_|46*d;TrcfKIC`%~nI3Yl6dH=j zUFk9@Nb|HOh}0uJ+PqHNlo)J^xwRovQl*V@3}n@3@af_eOTrpE+L~Jrw_zmX|7zDN zzhCIx-Xw^@7*!||l4GG>p+6WN0v*684M&JppBfs8^+zL{b2CFJrg0a|xjhX-Bc~dK zW>FM~9%G^g(Zm={%B4kX@f!rce~%5dE5VrdWanG1Vt=VrY%5hP9PGnt6G2mng0iTf z(U`87+=1SH0aW{Z1mPfxAyD`upK5317lbiaC$u529HkbGY?aW2J|Bwp3uh$I?7=_` z{UDs8NlM+1;cYtgG$Y{XiU@+T+=eZF#>Gx3m2XfoJ;|!3wa$~BfLhA}7eP6d#_h;5 zM6BX7!QPQxkRUcs(KD>TXar;>%*cAIALIoyQx*=icOGbMW8pE(P$3+R4s8G@WnXA9 zgoR#4?`dTZ(fVZLa-_WtY%3hxkY#>(D`JDt5)KTDL4ONUvl<98==`=|K<|3RS)M+R)HYGkOyLMo~Q> zFKdh(i;w1@#X}MZpp3%BU~h<)T1`{U>2S37Y|Vxmg`(pF%S$*D42lYDPy&S0f!?z! zZ)g3YjY+vSDTzYMx#bY>X=Agk36yYH4xGlI5HdIeVka+}PD?V$PAQE>vMKda^Wj7J z^Qvjd#3Clt(e}cz108Lx!rDkQKd!JlOPe(gBTCVsp`hq{S~x|kuIUsdkFq%$2s5HQ zI4s8mHZE*BfR`xvqCq%9bBR)|&zso{Q3OQro>)lEAt5%Ol{tSpDvnSoj4MPVSd287 z*`CxhG9Nc*dm?U9=DN?ozDrf0tfrR^PkRiCk6&=0!>@5cuA2we*NHr^y{_93 zBurl|8bl+o<#N&Vnz`Eq%xng>G>a=m3*Hvdig4R?-Q(Dlz5rX>fk-J1@>m>4TqQaX zW)t@m=MItW0Z zph=>$;3NvLCgc)iAh{u}>%GvPCN!r%9g6g3g#nJx8u|4c$1e!*T;mdag6n}#c@jGO zqFC%1;coKnK3&?{1IoZanY0rDfNKUh7(SD>Ecz@XS56RwM~54U&B=|M1Hs0Ip@1BW zHqzQ5;3qe3JU1+zyRdObGC2rnbowQU-&G^xqo$c^)oCANX z8ZTlvFbKK<_Ia@*@@KbW*;K@+W7#9>3dej~C+fefS7pUl`Iu2nkgkJCq_$xayAl`A{Jzs-q;FY5}G5_77V4N$)>c|_walbT9>j~c98f*UAW5*0u5 zYUwQyTA-E*Lx{U&{zkT!crHdHhdl{HY`N+mSfToI?_BSQHjxkU3H`vTT&U{%M%L5H zT`^uU_3A@hukK_e$9)>TG-@0*C5!{LYTVpgw5x5E_&3TE#)Roj-5b>gu2(;58Z~~6 zzvWPC+{x8)u@W_99Y!2&An44VfpXQCdnXK{<(5?)C7yegJI`Ivzr>yAd!ZjuN;uE? zobjswrF}smcmbe+L>3aKiLNlgSOE5AQudjsBm@vJgcyaSV8#R>b%tac);D{=q!eLQ zkYhn0kJG{4z_1)-CA1tm08&I)RnAEugvmfN8q&601>(*uiem6xh_@1tfsC4~ClltW zcm!yq0b+`*H3<~3Nm!H1b4`4`nhQ0FpkkJ|Kw1hyF@TzDVv#TI3W$NBn7{XepTvDX zmcMC83Z4mFxCw#xG-y{BQG`%mBq{}aAe@0%r5*XTjvEP2HGwADp?-ky(Z{3@JCp+u z-57F7n-t;P=amj&4AZvbB)zv$W)cyFv~GV_+8mOZXe_nR7Yh^vLzHpaJa|^5A8C75 znr8B|l{j%$lm_Ywwq58AG6aw|$^FBzP&jRp1}UaQD=e++I4rf3$4D9n86pG8Dl&l+ zpPA7&l!(}@>INBteOlieyMWAS`iN=%Y??nO6Zyyvzd{J@z!*U(6bJ)0#4B4CF|3BX zu!~42uK)$T#AQsJ!*kzWJ#DX^bj;W{%zLZnyelUS_l1qq!p7vvEg6pA`V^d^jSG5x zu`9#rovw!#uB3dz^5&i~%e=ejO8j#Cx__#C*1ZX-CoL24q^~Qv{8-9+Jn1+-UsU>q z{`+P1(`EI^hCQ=o&!mc;9W!Ng2CH`tB|EC89o5%6-ud)fpT6b4Q=VGB_wK&r3!OjdQi9K>j&OFaO2>t_wc(1-)fy` zo_P88%im6XD{=R^d)p6j^S;fea&hI@{!bB>~n5nuoJf`=<^=Ve;y@9)~s%5&c@uV}6Je1DS>?qBjXt+ime z{h_{oFk_DbY-yqe&tT=o7*E!?3sG*O{-B2Z!dKwA#`SSA;$G}Y7%%WL;1+0amd2|g z^cFSB{}t$4uks20Ds-)Q<4-i;&zLZP3z)u4zs>lw;LnOb8~*J0b6m~mJsPh-i1SGy zot)1VpXmZ+!n#v*T>0v@Q$VME=dz+#5CB)j9I|kFI22|uC?pBd^AV^Gv$Ip!580md zGl0Rt2!VG+BnU!#!=OA9XPy?G?>y2D;UWmMat={~LlAd@63T$yjKmBm&(s`Spz;Bx z5s1dgF{od5x3(Q`KX&L41Kg(w$@Byx=T7BPNTuk|w1Gn6X1|}w9&w$&H(t)XSyfbS z7lRo5;&fgNZ`u$KX`5`ad)Jw$D+0;twfyjI& zg-n1tlJ+8P)%2ukBWsToRO8Dg_>_KA$b^;k4h>_xd5>Z{`vCTV6 z$9q2?o#=Y!#kXFZJfCtlX7u>{z~Rjp@dg=j7H7=lvvAI`jFo&g&gsqA$>-plMHwgg zT%3DNa?SQ+{f;|)($SQ0(?<{ItjiRU&&xS?W{SyI!Z{lNoSt#(>+PB4^nC^AEYGZ@?}GY$6@BN5tTqMpsjH0MMC{qRtcM4vj~md| zB|c;y0CJtm*|9eG_#b}E+Q5_xL0>^S;@@(KP^P_DAkjgr07c(RLN}HH6d;3wf`RuR zIot-7GB|QT=nF?rV`&^23Pzd_2*)}O397a-H|N1C^{5g=Kf=su3o%ey7An*4M*(b3 z$fiwK#;8W@LHB3%j50o}fUqQ=}) zVb-2q`1Pdrs8eV_69tqE$jt+^hAYM_psmOPTEi%a{D3AZiUR7)Q7IZnYe$mOdQC>ZF^|rJk=AeldGqSm1Xwpg;M1G>jfJE zWjkv7l@)uvruXYs>@7EGD-NG?Gn6{%NrW!MbPogS%%*hNCnuKO;#DDu=b(+ywMz#O z$ERZ*D!rokiPlIG9GTX$N>d!*TA`sHW!EWQwS@8H2mU8YS_22_1B>InrF_~_KH;5s zX{zhKfA_S1_nlD6zdvayzh^nHcrhYL4r_e|Kc3W7Q2K(+9LN$H%1C|GMV8P(s$JB4 z>2rt!l0%b;wOu-_d|<@ZoRvzHo}l71@~R0ew6*nESE$kq1##q8m1<((9Ci8EIow_)?^r&4m)iE0zqgQUC@09C^rl;DB3u ztL}T(PJ7o*mEYcar#|Iv9y5c6T(MoYO>CK}8?((?8t1Ieap$yk`F)FT+JaSd2VC8E z&(iuhEzubJ?;-PR91+Mu2=N;Tr3d;b{S`x&eM9HP^ildzLqeYeHian6IbP=9sGyMs z3t!GOQif6e*D$I%-5Wqqmf45+uVH;)J#FpB zr6E)?yT+vR@JnALN8^}F7M64lp>gk$P6u}UM|_t%;Lx(tdu(&U+ONCbb(zSUiw_J z=M={|-1sW?8VXJ5Ji?wo1vQ0sjLKvWh*Nf2f&kGF;aMau>D%M)F78<2JhX*a@ED9? z#mWO(ujnv|G^#2=p#3dHemV_m(@zF3FtUJR5F5R0Gff^I8iM|mwp-T11_Rm#Y=$Wt z!M**_P%k!Aa|?n@f6hgRp;N^!4JqpUY!eb2mWuI>56VJIEFhnS4&V&4u;~eja@rgV z4F*v$teIVCR@%`q9HCE&qL8G6v@te31f8HBtXf@JH}GBh*W`SSoIilGWM!qzBt#9z zf{*Q2WWPV8G=$W!!bm-LWzXe3N&BjK?~060SF~n9Zy-IK!RZ2vg5>qeoil7?|h6oWL?K8N%&`6(j>i&4)gtebrcYN50f zsl=nykE~Ea$0*B_5Y-sFA(eqddvQwHOX9o4P%n$GQ~}i-)uNIhANOb(s!e3&nD=)z zbU~a)DMXc|?~s#AO3O(3MzP~1xl*+3PJ&@d6pb~FqNNSgg{T5s$J7r?B1nIRZ$6v! zyIQhb1_V}>ejsd<>k;YON}4ZfX}$zs+^gatx#4J5?~~WPOR`|an)13cml@Stxd4rq@atP5dUN3!xe1AyJACvPQIq#CwOwKdp zY*QT)QRE$1iLY@#GHicfcVB9qD_(i&P{yD$F1suJ6NlG=g*Pr=FqIo^nP+)!)tb5D z(vRFLjkX7#;*1e*knxi8jG27c(kRYY$>-pzR%D#yb8)Vs48@03VJluJA|Ey#4)N{$ zM|!idVXoYlVeh6r%DZhpdq1!)TQDLVYSZWVgZw{nym3FjV6qz<7Akq;^YHS-s^zsC>4K9^vu60zm9UkOgdC!?g^B`&w* z5s^X_hY$y%)7eT4xGN6>X?Y}g3X@=y(TUtd#IEFG#5LJ7K=fzHQ#hA62=OfqImM9% z+|$@D?SL=MqX4|Yr}+q&BP~o|5||#t20E>WRV&s?SI4pTmY#iwyV54?IALcGM4p$W z?<4LabZI0D;34U6@!dnbF=%kkT6 z8Eby^+!u<+UwE@@V&mtl=G>k!%beXY)-e_s+xms3`^9zB#dSC2sf}Mfcbk7Vo-D3Q z6>plhZ%P_AvB6%%N;OpRdDMW75kJD`k5EoHQpZ8*;YD$Xnz2WVBa6CL?&Ki0{0YU9 zu(RaJVUK+)NVQUU&|2Jew>hm$3o>|M+EbnR10?GbRhCt8#BloLIT(bPoblskbm+!9nn>FvR`77V`BK-U{Q!>9=x{{wT zujCiCi>5AX*QWe-VbEX!gp8FZH(9goC|cSMi&{az%4sA@^B3~lBBk){lUq@WSV}AE z_I#)I_6s*&oCpi!3Y zG4DV$6v0UAqo)UQ$^>Z!VT(7FSajy#*bXOu6%j6R56rep7w6nX^X8)Q+AqknW)K_i zm`(W^n>APexTtBYdCpUO&3x57!M|x6Yo2#kOsu-zIqP0E8TfkNyL~r8bIZy<4?_g{ zEZ%6H`1C#Zy7`joi5K2HIL?!5e*4w!U)(uYRsH$>88_#B8cf#X&6P0cF1|8yd1RW6 z>UU~q>-MC|52f6Pl9oercIV^t%!rC(s`3lh;kgE*#=Jq)I7LBDgDw19+8zuBU!wxK z!DpsN8m%MKqfwn`D5M600i~`F=0z`I`+rpTimo&JKF_PBM?Q1>iyd@qfUJy2w1fdO zZ7mbThZ;m!J1`s}Q(Ww$lN~qC%)nAR8y1yt-h__wkVwUj#mF!+6^CKxY9rk%Y{?tQ zbfqCAHbHEcao_?cSzwckd9wi)%@PO-cEf`Ki6)US6vgodq9$iy71JP%qvxc*goCMk z6))&^6-ypF?ZS(ST@O~QfvlCKze9|6a6;;qbk@*@QI#W|iYU%lqzC#-Y;H1p8b$30 z6AjYzr1etZe3}m~*$K>xF8ycZ?59cj0z4#i>y5kSUEc96uMgaJ)l9o;rk=ZVAnB?} zx%Q3eu^;59o@hzgg``3F)k7oaDE=jY*RHvW`u8^9e)&(fO?#S>?x*irp8j7MJ-$DZ zff=}0t9{%}YqMSd$F63R{(X*z_kEKQ?jl;DO)e8Kz>{__pzqb#%qZW-jbdj4Y=tj0 z=~R*rRtby*U8JFq23d{8!V9ocQB(cM-Y0BEYH$YBcpDalHn|4?c9uRlwGp-|HTIP- z#yo0FmBAsmld$J2R=?-oqETgIqo%w(OpoUw6(vjsbI`0HBiPdzH7CrUtxA{^`g5FQ zg?%9D-pv&#khDKsZ>KRD+o zflk|63Zu1al~*e#_P)7dY%dnPr*fj^x;X2uMFeoGE9Wks8$a{%O2u{R=jTF3NrcGuW5U}fXM%Za(wYsO9E!xNoXKWA}Wab9+gpT1|Q$gJkvD>L<+!+YuQuO3!Fxp&`PJMFFoeX*AQ2zArr{Ydtr z&F`-$ZCR)L{<`8;&iMUjc(|Iy8A8F8vP^0pD9Ud{M?e`M`m9CY-po_?6;uz4+FDQK z*x8fzri=;W+g#2{nOVAF;Fdn3wiaQ(TewuSw%*MNlW3v3!gkJp9L=Kj96xIL#CqG* z9C9k^Q7cOgnM>3~45B?@rrM6$P=bT?h9O~zCt5zFo96WZogYc^`nWA^r#Mk`c$C^aPP1NNo8mop)eRy?7e#d*DB&$WN? z*sz+Gu$ngI2`?I~8MXflOE3M6OD|)kcRit>mZ|-Ok%vGA-si~cFGq|_L)eqoUk+)5 zSYF7oU95nBVdbafMpz0;Q}3S4uo_ zObYNvm8TDAd`s;c*z}jFezk0ANf!nB`XuEAVsZXA6g*;m27qQm3hB^>Abq$M5q}(P3zCU;)@b!TG+pL(#SLz5UJhpVDTBULiqC#!p)>ifKbezz^QuJ+JQy|rSGo3%m0n>edFCT-*rp~119KU31`1W| z(eA&zfBg7FN6H~wYI|UIBt5k==Gu8z@p$XC{a5!-yp$?gGgX;#Z2>m4IIlP^JCe)R zOz9_wlcoMyOT!!hXZh9giM6Ss+A-5yNjXb&B30rWv(6PQo7gd3R6A4DGG)C}kzBiX z%%o+UT>ez5WaI6fDc5e~yiAxpFulxw)qDHo-JN&WCpWhvGkeka$r*di&CPS(PmQ%? zOCL{s%k=L4JA0G0&)kvj*4{0>d+u(3vh>KTk)>E zuk5_M^Npu}vvj1HA+ut0$}P;e>nHaDZ=(w0_10PYs&~b?;__=vSDRoO$(F=|5$^@g zP~sYEWv1s;vt*TjN}M{Ltk^P(;~h%F%roX?YA?iZte!G_({j^tyCSvr={sGin!U+Y zEy>c>SxcJ|J?U9LZC<~q2~C%qUf*@!UNdd4nXN^4Y{@n!S=IQS@pkW>ovEkVl8yV4 z70=B&_G9bE>c$d<9P-lTm)SYBil){guY4ZOK)eZr8kb?9S%9*3{PLlUoiZOAp<%9M0%DOR=J>%2F}R zVxDFu{lPVGVIxUb@2)o4n%ZV6tSUWVPC)u6veBRJ^oaski}A)rWKj zAT|5~?O34VxT&Q&Dn>Nee30S9V%;V-GO5`;p)(R4tGbFmxe@t*G^pb^Z*h(V$6uZ(o-j^y z+%Qh|PHj!m0jo{P(#`iQTk-}F?baT`|EwQ=4<7Wx$Lhfg>3?!t6B^R1%wvk^t(>MN z|D*#$pxLQ}e$vM@q=gBjV==SiMWH@JyrY@U;z%&%#L@ih4#5TiH&+ZRXMM1tFqC)m z4^9zDxP1n<=isb7T}}Z5Ae<@Gjs`-(*C5%bzYWAe+OZi=hte4zkeuYBDaM9~bvXu( z#)%CPGb357?z1W5Lp79EA>NnpC(}MJQ$hbu{Il`TCGMM7Oq*BCu56q&KgC8XxorKd zw%fJ0M^YP_lk4{;OIu(i_zR0?ymZ1esh@o5o~7|)W-g7!Zz*|?7ha4xc^ld?ZFWFg zmK^~|f7y|e%Ak4E^mhZA^r+1zE~k@Pl9#57(()>T-VNQf^vUVnB7A~Q+ZL8;G`9nDi(>+m=Mz_}4RW-dNn1OfP^v?n<)fV#0qrq#q|o}j#EtWBoYu6Y1QP;fmq2I~nDVh_Q0`L+mW-Xt2beaBjORQ9LQmTkJ=hNnANQceGvYGKu+FBM-iDTJZw|3b0;D>-Duswic1w3ESYqKFC= zwUZq<+uc>jX&$1+B}YKiWNx3|emz)cT&0MONb>uX z?CWr_?*iNVk}C%<9|U>Tr5v@;-l{m%U3TT-<%<(fr`&5)5ZE$n7oeY)QUwBIWyZa_z}v>1Xa)dgiRIq<8I%b?uz1^vc=GXMxgPwSdDc@hdli zxAx!e{Kny#4bR@`yz5KVA4;x1oGfj>XE}m|X%A_iozOlL!Tz(-eO26FR~7Hur2p%U zJY0#em~;z{Cg&4LBhY|1zt9BIKu1qT4Y}RYJXWR=NtGbCVsW4RD)a$R9~YMP^T@3U z%34Kry-dV8xPtLDIUh_1$E6kfiYLU)Q#9w8X~JAU8SBo2MP{7A!@+FT9u`|Zr=O6& z#&*D@G-jQ!6wKd=9%K)g)6)Duk9J!bdH&oJO41bGqt?8VtOZIsg>5~JVX9@qj>KL_ zahk9e>|>4jJ|Wzxj_wn}U7rx{R>N<3)LvG<@Wc!%(H}+UIC)K8TMN$yP2HaBi5Tsw zuqV%kM~7V7@>u341$O0kSp9UPW+9vrWe>2GE0^(hSs3kw5KySvh~1saS=67x{glWIC!+kh3sS`3nZR(E?oojT zR?AcnqY!}TGkGyY zs7k|=>}pRM1Vz6Jg>fu;Hi%OlxY^-i2*#qgQ7|a(zSuiTD(~GFqoY)=qJ7?dQ5vO7 zU@$g4N~(0R5+4^;<$ZK2=u3^WOv@k(&$0n9X5vF>zBkQBB@-%-%Mk6X`bv`}+-A%% zr#-ZR+GaVyl;(hOWUnv%N9vzi(#QuxPykQPZZm8rWwuY zk4eiY5k^uJrGG>57^GvMjc{GsC=L$}$%+_E`cg%ELEG)pU&HY^rC*Sr!MNMxBkiaH z?H1s}W_v6*O41gTmDU3f)4_dq^R1XRi^}zO?DSI}v{Sd^zeO36^F9*G{{RO-&cl^g zUTT|nRa~!|h`qh;zH7Cz`+ofL@$r2VdsEJp*n+n^$CuC8SIm`eyj}C{`ft_WsYq>V zNp5Vt+ng-kH{;m%V9`dqrudt8luwjQ3{Qq`SI##BSj$XFX0j;)3w-CaND zDZS>n>d5H$(z>74t$TOhT;zGwQ5>CX8-cmG}8?9t8-s($D{Ht(&R*!0e} zx3*2LdHb1^cm0gF^VWqsx^E=l`EKlc@$bYxsQL3(J`g{Ulh1TQf16dg?zqu5RXY_+ z)on@EZcSEfn{{j_3xO7%bJu6~@pLc$TEp5}vJcUPIJ3hXi$p3q+rhoB?rI@Mb5dVIDIR$%L!?)VF zk=fbD^p;FG3&&qadA!Xql%fOV*7ysD$gnUHjriFWyUG=_>aG(rRBmFt9+#^^l_)Vv zo3$;*{bw+Y$Mt86*p6bBtPs2h7&JwK{sH-jOQoIc5TMG0(q1yBQtrT2ONH^$6Uj9r z%}L6P=ra_SM0#N=i%cXUwo17qCbQC(9(GnvrhFL@{|BPTlo-lMyYo`(5;|I#EKZ7F z?|-*{>RhUR+nvUl`U7*`GIm^STgqEA)~qU1#|>o8RET=s zv)5@dE2QPg=!|>oeA$YL^Qp2m<9b}?cx~s^onL%ZaVO zqU|i=I~U)&m|Xu%YUQ)3vSuyGmng~STQjSONY&tIxY$UtVPHwx(2qEzL|c!F+Jk9`zfC_GLu3UphL|hz z&!|X~!Xa5q-2In-h>L~W^1|`|8_0&Uk!7VXr*BL0wB)79#mv=%)i|3wO~&7@#n*Qnp&uptYepUrqbPw()W;x&SNRK5eKjIlnqlOy-2=J3Wu5~)*lX?R;+SO&L$lZ&I@p8CtV6Qzz!KemwubG_kt(~CW%zATV9lVemwN^ zd2&eiAd#RZkrFp;#nni0fJA_p7W(Z~&ir;9Insp_?>PA0)~OsAV~4p25Gh*+e@ni5 zwOe(QqG$Ul|w y2z(0<6LtXrfI32f%FL^k?m+&4&bN(L{Yd~iJ diff --git a/harness2/substrate/__pycache__/command.cpython-312.pyc b/harness2/substrate/__pycache__/command.cpython-312.pyc deleted file mode 100644 index 2c13e0ee5319d4ab5bf8650ea725089d75396f32..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6789 zcmb_hTWlQHc|J2cv-b;cB9{~ytuCRoinxqqyOtu6F)7)WEQ?Z1TCP{6$D^GgIpplc zXJ$omyGm78ZD}AuEF=hN*n!nlKxs%V^i(KMb<;i==*w=_E;nNeHIN3V-!`EOr+P{M z|IE%VNzp|M^icbsf6o2?OTuh2AcPHJD3%V!mO?p{dQTL_&Nk8yz#iIw(!DNW>Uf{#Y z2;+TvOS(1L3cOzl=xyotWP7?J*^%Cr+{Ma*dS|*T*~R#f9!+;AyE(#>=ZO-&NtDQ4 z-d=ICXVDdJDZDTeGiFp>A4p}>X*r|B)CDz_H?uT0ubDG3IRgbbT}x@Ez8F&#&6Ka| zYRpj0SavFwquE(CWyYp7T{RM)xn=!`VTeu{fGmv{O$hn-Bnf@GC zfZsHqNy~H&o0wEpArFk_>6DyO$8&1xa}JxitRO5POHGMQVg)iV@HJJ!4pzvpo7;6b z zla0C9e3s4`IS^&S^3)rr>icI>K)Z!+ZexbYqR1No69vwa!riU&Z*oPhNTfzhk%zCk z1rcI;8a?=%q{yEkck2aCe`sCz5JDC~n`4DRXXy-XGu@t;mjmDOQ?}h5wdhgycZlJPvw9-4eBz)rS-hf_Yi!NKB zfLY*KV3x^d>r;5er3i`$o+q$Bj{?YAB(n|6!_n-dcBFAezu!K(uE< zE@F{FIAP3bx^4{QGkHT*tWfrvN-4ODAt@StE*`dojGR_2pRCWzi-wfVSfK_hrR4>y zqxqSVa}rf$0~-0*usDVlu~{5OqIs0PNww)|l}a;O#@fu723e2vR9kzhozZH~-s;}{)z1EE?>DO5yQ_VB zknTTJ?b?n1cJFzA=$)Zz?CW3jMOyqd67l)zUKk_M~ug zPs8G)CK@SA3cqv9cln?24&1!SnWEFa2~M+Kr>vPPu6u&h%gL_`J~XmGvz5a^X~^N5 zd9eYLML{F?#Jlyia>R@{t%`!#>hQB@ncGrx48^_k{xr@Rcbx1eW`{Gt0Yc$mHxV*e znngF>mWl#%kGk7%Gjd_bFJ*ObfS%G>mz>fcbmS&Lweo%w}i+|~Y6_|n;L!~*2ju#q*nV(%qsW}Ws zEVnU}H#OaI(=>L9i_9%xBKTlVCKwQTSgw@00BuchNyR{e0+e)cGLVMjmTI!D8ET<- zyT}roYM+6Q#@_+CL8^hsTA;Ta=)E2JMPLwsv)Xq4qcdwSPL^MsES-P7)FN-V`1Wv( z@S*S*ZqnBIqnWkN{&Hu3DKYYK=V+z%^pd;Ai@vsMuw^aSTMqWFOsw?{l=}u&XDWS9 zKj{4MYAM+Jaq!%Fd(X-r+&T6)!+$k=|4?QB$p@K_l(jLbJSLTfUaz#vrJ($~^+@k; z3GZuL54Qc_!dn+s`tJ*$1fQw~TWcbeJ~GhK|Is@V9_=DO@7gzd!i7Pcm#P@*sY=Sh zV8oJ!D}M_NVzIi*dHj&ozMQ`|5`+@UD^x>-_O41E#F zaU|Gin>A8|=N+W5$xAL_uWtYWD{SjrK62~G4--qC^~mm(Gj|5=dmeOD_6`4Y=%eY< zD_1Jd{81$`S@KTWa&Z(5eF+A7OfD|~^Hp-uAOzc4@lBMw*FZB40>TgYF$ypb2Eca0 zlnOkaRhF+2!q#IoJ5C8gAgtvi`x;Q_d&~hhS5Lq=#y%jhcW-db8!daID{U*Uu1>5C z43!6l9%z+;vn6lz6YsgLrT|?C`aRq7YruTfmcMGdvjPsWH95yjk%z-#ix}udXbEP3 z+GH}Y0th~UV+cx)CwS`kNtsH~Y8#-`fL6e~><&V4kQb|8fsRHT$Y!B*taZoB-SO3f zYlnx*hleWN!yo!emtJDB8MkGFzV0OSQYM>!w~uhL(-FQC0yN!WcJp}kaZL4-_T||o!Vxe;xVdL zh_e=sV1RKHqQrT4xpMOC9*dNICtk2Ja-kC ziRBO(SRPYLt61PtGuPrlc7L(lQ&|c}Eq2t^1xtJ*pEXs>1yflf#0n6R3)!5SvHTEN z!#Y!NU$b2B-e9@MC(c}adBPGDEoEAMs+xI<>rmrf20Qc*VJG$m@qGgQkEXClL4uOU zh2e$_glF(TmKZis9ZCuvk3kZQlY|=MAAsB-wHDIewLEZZV6Cmc+}6L|xqEr;*4&-_ zx3#}cR638YNBiCnyc4){dNoiv@MIP(57Fe(*|5U@|jd4LO_T@9ArrGU0?u3 zxt$Kw=6l0Vn1Q#!rgvQzi@YMvodIpQi@*oKf_q4YD~i|MO^kqG+KsOP#m12fQHE!X zUV&B6Ng(k0q{7QdO8tPR2=JEMjzTFqW+8m#;^_FZFI|TF^Tn~r7~GwG_#6hO4qg*M z`*6!r7fgy_e%xc9R{+^)7E%sgyDT5PT-egEd~ku3QV<{;l;T7J_5>k54a{S~0uWFYMh0{d$d;HO1R=i+7ZmVv z2-yo?drW3q>#_ScfxrP6>G_}J1u^l`b2S%_S6SY<%GQw^`rC_ z1o(B^c*ZXNaW_4KZEkqJY7w9)hZ(=2$EYKR&YPZH{yU+qR921*P#)`4A zY(`~`A9>DBBqs3gU&#!!gUmJ4DW8?65vE7%5?u(zMtjXh}rtFV;*af3tWw98l@ zlrt<8ZLPUbx?`y&PU%_MjJMz;8hZkgEWd>JWq1jaBr7OMZ{%gYUgMD@C7Y5Y`gI&S zhJ=OxXOY8OjU5Jl2f3G#=tvMg>7OF`GbDJ}GqbP{9(-b;NcO?c_igv4j-7Fi^x<`fVg$EacBlSU#*s0A#DoTP#ogN?~8ag4h=C zV)oUGJuZkCCM+<_XR_X=1j~EM*7z{}F?7W6)ffR%b8#H^o39b>JKV3yzF(2xrzG?# sY5kN$9+K!o((`K)dr0~olKuZihRS5EbcQYMuD1{mo;+5i9m diff --git a/harness2/substrate/__pycache__/opencode.cpython-312.pyc b/harness2/substrate/__pycache__/opencode.cpython-312.pyc deleted file mode 100644 index 83cda9bebec246a1f905b013e4d353a7f57eff6a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19071 zcmbV!dvqJudFKq??*{>r67?FAqDV-5L=Vf7EX$!p>Sc;lL|PBp4g_I_Bq$J|XFxq* z&{bWZExC4&$k?seM%mIyPeZNMhG}yeWw*Ogx82s~w7UfbI)W$4R&BcHG^hX2rn=j@ z|LpI(g8@K_N_uu|-nqEX?|%2Y-~C>rzjV7D9G<)UkB0wzAIJTae&~-^%kawwd5*iq ziCjM?@}eQmj~Mz5JmrmXW51ETP5mbJHusy^+tP2r+Z4Bs*!pd(tvPNVar8Tow}{rb zbHvr}W_cU(o_;UO+vC0wf4?7jhvYr)uK&ob!Qa1Ltm&^2H}r21H}-EtFVpY=eTk*F zxalLTlAiv}Vo2Qlj;Vi(xLw?Ww;=8i1-xs;X0aCUIskR6uA@!u2?>n7^IF4 zf-oXn0z!vIBEZ#{0)&htuQ0SkQl*+kMYt4E06;I6K5TV)j33NwT&;Smv~Cl4cO zKi1jZ*V~GM$S9UC3Oj^W9S?19DwkvNxYEpejiXA5%CXUu!Ui2mj!0oKCWnV(iQx!M zL(e%HG#pKij3yOHY#Wu4OSKN~Lg$FUiW7vNk;rHfBeWS204rcp-jQxWQ| zbmO<_bR;!|xAhdz0F3{PsGNSwV+q1GH32zDcsqMF%Ih7KqCcYp6f&sx=#Uh>9FE3g zstrjvA`e~(8CB~Ejr>$wS7Z$PCdsPf%vj=#^v0N^q*R*@WyqBFD%X{iG*ZCVi1>wj za4aH=awHa4usdmgS28i!C&?qHI#2bSxj;pAw5%}bnr#ZLq>Ysfj>W`CA}XOx?EdQ< zmp%x31}UbJm!*UNAS8x7DFw>88cPjnWJxCxl#{%g5Rzl5(J>Izm^dhz)=Y{~~LE1#@?ikx5Kp0a528V#-A;u-;P_qzEMzphAoQ1V$t&{!Om2^{! zz=nF0Ff{(`=8itqG8!Knj3q**kRk0a(0JRK&i0N|orLJ*!P=GH498$d zipfIHxo+$~Sk1Lm2`WHeHlqel8{-W?fHvTOj852oU_iJ?{PaL^?u(3#T`VKT16Xlk zDLxIkWy|<2Z5jIxlr93TjRJt8DMdKZ-Fu>=lhGxEJ)TUCVmMZiOvJAXaiVDa1lx;7 z5`vTfcg0&w-2Jl{j+x zqQGzqml4es5!OVyTQyN%)vjGO;bcN}7qd9R*VJI?XEZVz#zhA{soEK6#MD)pC_U4`=<1}G`j7a2<>^^$>Dc|DzfwMr*T4E@F!V8)#Tl2IKR+SwG2p=9#15@x;4 zN($F~jQaxY)Tjp5xUf{u$_q#Jl^SVxMB4x6o9t4S$D-_77#~l&UVHPkH#4uj*?_#eAI8GOp#`z2{@|U0IIN6&r7n*URA;W*2 z6OG?6>h0dJy&{X4Eng^#*9(kTsE^gV;nAs zeWmssL{7cO*A1d|6p`W+o2bP;}Q817OaUZ3f-9O!tKS$iKbqGFgbw3f@6&K zI54Sk8^cb0n|^q~(u%@`ju4X2wM{LsoEyd;duQVZ}5EHx2+8{e#NB)~=>=O|cM99Ov}f-7KO z3D8=oRd_`q8CwCf1&?RzYbH4boJLey)2|oTrPHZKDxKXOr8}3j0-XVs5vb6#dee(? ztfq>Dx7BT&m8aR>gYV9I?hJoV^XIqCuK;b5*u2zgb$xV z6Ub;%1e$|t9+Z<~qpC#)46#wwhJ&CGu0tlsr3IN>)!g7R8_PpOM_)mMomzf73JRS*#A_Y};?N=dHGh_P4J* z@~m4j@ZJsi4I5|bmpH@btxHCuyJ6XA_Bxh0v)!@e;=I)lJ)0Lin-{n2oZmJ7M$WVU z#<52>&rSEFdulLe+kB(_Q>(2okT37^-?C0yXZZIW6K^eX#?6&WMw7d0*=V5wEf~%entUne-1yMhu;6T1+_rDgxnCQ={iS82ky;wj(#W|EECYOQ?F(fFHMh7_@j4kU zULw>-dqi9hBCd@ZE~OW7!?Y8tp5g_8Ye(La;n%uS#NwqhY+uzKKXu05%Za95E@bXb zSAMnZSRZ&z+1}K;BeXSk9$;!L(bojNS=J^LGOGrtz0DX-uT#JcHVq`^7*Nq?;d#Q+_2!>kaKRznhs=}4j`5HVu)xtV(Bh6(tRFBrX2ma zhf%)z$?*EECGziZcMZB(W85Gbhl}@msh4Pahszi`is;mP6H3#^dcd6OT1e}GR9uF+ zI!zmbmZ0&(g6><8=Pr&iX*zL{F`2r$))n+b^b?>3wIXwi!!kxkN?3q+X9ShyewT;dB+Oai4H*#{4(`?nd2G!ukVMT!;2iK7^SP9 zwN?U=iZ2No&!B$8-6H+#oAPz;8>ZjjZ}B|$-U}w~y6N|fS9xrcu|>F;HeQ4nza6vR z;Z=)7>=Xt7ll*0rFigorrK%kgu^1*D>5f4)w0=o$N9hXswjYGXEL|Iw52fpW4hHo` zze@dIMDh>ZKZEUYhDyt)uSK%w&QH`&L~agE4&5A?9GM8u99a~0&ewi_$9H$!zc9Ze zx2q>7oX&3f%A)Pe!fTO#QHV0%eyN@3AthUsqznyEH4Vm-msHbm5*jR(rl1dssTm@V3wEV4a}Q%RK(Z`Kw&Wo&MLuV=+joBsyxM zWl_E=sQ4x8Z%Jq^$R3byVO)xk;Spw-g2A(w*)4UsTCbg?rxuXUnfUC$@qr3aFqjprzu@e!l`)Zhq*dsIyj2nfe}T@@$_8Y-NInW2 z!t}(zZrk3b8wJuQ>xK5S?I&RP?CKO)X|JP;>|RWQCWMun83ZUcnzURpoiXcyq#SUh zDtXYH|9R&+h*-T~Dq@Op@ikpD0Xm%5jP*A)HeC&X)c@Gr$ zb_Mu0M|@j;2^~Yc#+y$dU(SW)mr;t~Vu7bXP1UA2x+yL6k}b_lTPJ@vxz&aTs)XN2A(XHJ~%>vgnWfkl>?fIt)% z-_j-Eu-*-~OWM#Zq}#~QtKqI-78IUtT(tXLY)xAoj+T}dha==veXCs>auq;Vtuf75 zA|J;unH)NdYpWK`axZsL$tg;@DQQ9yvdBGnsr-P-r&Kcpf|Q~fW8yW{S^!uf9!-R; zUk#p>$dq9~=HA1fQVULcgL~rRD(W8kcRcX#ST>v7mcRAym}t-Aw)o2A$%oD@3(hTz z&f2-N^N~CK51S7!G#~yrlx;qoYwmm4+`rJ=pKE?C+xXh-8%RDkb9P7GzhlYFnITwX ztS1i6xnX8+wtjEU@%)V=pIRMxoBO6~(v|o0XJ7qlwm+P8U3_9R*d0$Sg_Z|$ju*;X zLTPZ<=j*r4HY^!=cj$=`^7<1S=dD}_w$GmZ_NyPhI=^|b;mf&Td(P8-F5{@%1k*84d0ipBPP$`W*oCSG#_>Ytdi7Xxf^0u6yX*wBXz{ zBjuc1v!<>8^7&3IfuZyLcD}=EiUMiHhp>!?Uw(k}7AJB25QO>-A`fSx0gl8VtU+ct z&?-gCJ61RptHAF~a41%bHoUE(9S%Md`*Ykj+qqx#pSVtR!Vzba?4nC_-!?qmUR*DF z(8BR_jhz!~L@#D_l9TZ^|8xs+gIIySE^(t6Ksz_weVbH&&uU*^`V}f-e!fc*tcE2= zTd%Gy67!arA5SwJ#0zc`fpiK1Hvgi#t3??b9gW8%u~juBquug0;E{ySOR+>z4WO%@ z_zgrIepx^UQZvMeFml?HG2mBGK)KFoyan!?us*RIK}$$cCiu?=RNIQ;l_Hca3~4!* z3Ww7*Pp!OFuloaxrhwVu;N;iWPMF^F6FD>;Gxtfxt z+LkeAY$QRC+cWmz&3fz7d$_ig*K?u`t8(g?#;WYY^?F_Dow16pyY5n}_mUW}DDqHoWcIX8er;atuz{W1Y zRW?Sm-oErMLS(z^FYD8$x5yOt19&it6%6_SgWi>K{eNTBwI)U#D>0ga(4;*XbBc^+ z>=`EQY}HdbOJXo%it%?Vb(-wu>bcaH^yXVRc|Ej}5>>B_krlXjo4d+gGhX1JK38?8 z-36JKxpIW`E@IH+TZ4r*sfSLGmxp*c`D=?@o_IHORYgcQ3QZIbaxE+;(eaWLhgTcI zb5fvd;#%=u2&E6MHMlPFlSxt_BUMQoATJV`d=;26LTP(MjEtsQlF=5GZ^1QTLCrwy zS|74*c(;_WbPX=taPo56OdSuvS^|BUtR)JJriHpM%wws6mVM|0J1U@tRamC8p!%P3 zyr+GuJ?}JAY%r0OrLbmXz$K$wVAJ+uScK{-$0;rzC#mn4iFwGJa1X$xJ}Q&T%>rYD z98p3W_qS&>e+DN5ht~Tq)Ws!cs zQZO`@ip9w?497KEs%E$qMikWn7_eKUVyJE`VI&fdrQrsY;;Ip*a{0?Ng&D)ZPw2QR zqZ6}~Fi}xHOO+;yOHqw55yBQd!k{1;R^5kTR_?qOl}5=yCzIbwHJmvmUqvS5W;Yuk z1t@I!CIulDsxg|nMgzjjL$a)VgpwDirx6e_!;DNst{PHc!h{LtkHzYy1!vQvijDb-EwiSXYuUyN zj{;k>6%E;jE=ZjIZXR~xfCqMBw+C`(aNUgU{X-MBM_&KU^kjOmX8YXnMelReKd@nH zdv@dbtnUKMDAdD%9u}^m>Vu(&RofP-w$0n`ZM|1@?~QD0`(oAMod3v#Wyw%&uXyCD zxY;w=Gjn`);9kckuJ*^9gx|jKn-~7WtJC%=<5V96j9Z7M56w1y64?HzX5+(}?F*Q7 z_g!WF`HwPlZ)JBJ%GJCybtJ!T^K9TZT~qD(VCAjj)5mXhPj}DlxwmzydolPDm32*b z%?9VKQ(d0~_boZO%^?`beZg#X;{$KwqYa_CZMh9Qr)-ZaH)J>OT&Ucc-E(xI^5~Qy zAE>-_VEVuZ2OkA%ZoNGH^6waD4}Z%#SC!q>u@LH540clasp(U*+W^Tz-Jy@gPlCOV ze6_Ppi@qKCs*N*mE>vxwdv&2|-;^=$3*2%{J0ALk1)ne%yl2Y#ghk(rRP34d%v_sq zoboLC4nS@2RbxGO4}YioPWQcix#o_K6WOm`%ASfsS@1!(fX498Pwbp;)91)C1fH5a zHM4Er^oi@ayvx62M(Gy{amN2yBHDs+lnp*uc&3&%x7eE%-7~ zK|wQ6rQiY~awgd^s>@VlsiQw8Q~f?Nx46MFr$EMbjaOW_SGcuoVekwWWSws!{BRn^ z|7nCCPL~@)i(5di)_uV1l8;caEZqE;1~= zSXYNo2sR*#p%b5qJY&}YwE%Ox62oby3FQHDVjAq`Z)=1<#OS_=C{*}|5Ej55iWZ0F=(MK0vfm=G=unp?FOgABm% zXeRjsYX9q${01erD4~s4O@ttYL|7eOU8S}GVipp%ma3rF{S+OQKScuTu;B7J9=R*0 zqF+nS^!@hv-#kB?&bfCj8BzS0$Q5rm&fI||E9GpQJGf*=4hO;Qo3g*#z2u~F7w4{8 za#PO3xhs~ul=D%UpK=vc7NA^^b5}1_Qm%@|tVV9a^jQt(^5e|NsQunu!qQ< zlw@{X9wUS3TabRqi&hGO5RDlF1a}DiqA_NKiPHq(dL0SYs4VaPG%r3-8_cm{YY4&?8m%;INo>y)&7Jz#Xl zdr$Oqv!hoadVykhT4agB0u&tZ2yB+g0R$&#(SJnom;~{BlGLj5+f|b&C9Z?n(T=5! zZQHl271%>bPDBuWsD~q|fi-4Tw-pOJ&jXkfrx0w~-WBe7rSJ4Bec`^Impi+|r`pel z%Zio_Lkq5r`MS_V=TzU+(V643+L#usxUeR!lfF z9i8qF{UJ@69grzxp5+p};Vh9g?h!1D9u|*O$ZI89S|avLxqe0`mGQaCs` zl!bc=1a0h2-+oG}60Rb&5b;?M1re16TY4e#KtK>nxv@vPS&HT!80sOw;|^f4IL!9L zVu6Gv#L(c90o{%vt{QIDp(Mgt5IdTpus3mqPor?0s}=X(n2N+0`C2>(YPevMINn0yks)$K*H9- zd*wf*qVFML{y9c^1-jEhC8V|Ju28js=xm0n!(|`PSrHk6cx=w(m9Ev;JZ0U)6s+_?OL#u5;N77a>N1g_Eqq2~C)uf3~Dqt*_t1XcF9^um3kaKOCTYoQ>?fYs;DAZ0^`Kh4w0XzIMX%yvH`9*RX z4|fG{!bFsiL57{9AXN+@S8lC??k5ezV}nDf^oFNbQmkyFHITeRaT5Nit+$${n`Taa zVryhDYlcgcETKW#+od0*jL?lRsWNE>K>)Ip8j}+;u?~0x-o=aUg#2sBec>;JyT6M4 zK>SroeqH`EYC#%Qs6zfa2$PWnakJsD>I#S77>mRUHP&!gOh&_D&7n0wgT^TN8YRDt1n!?z*P}d7 zb@wRw6H5M+l37Z~9K<+{Oh#^&5Gs}3pq_WQPfg*+PVbG5C6mEieP8}3IO(gGZM<1n zwlu-za*^k%Hsv?hn4PB|Cl+?4Zh6&shl zl=D%UpK=vc7NA^^^Z1u&G>DXrz;ZR^`gpE(XFjy!Gq>LycwAAxWX8KN3*{W#hRsV( z%DK38jZ4%DvpCi*dnvcgX?En-ZCEnWds~b4-q)_ZyL#CBu|2SCM*Y%8?i}CGe`YkJ z?S?H&?A-?WnSQ@?SbLxDW$(v!*RmPqPdvQ2Ny8_ha;yQ#-`)7#rX6o4#_2m8SrPv3 z#s|1pNPB^71a``j|64VNOu&YrbG9?Gu!vQg5|TJ9FF_cVHg!J9QFi^^jH zl)&+@6R(1(rg^P@1qC4WzB8|M9|<2lg%2q--3!d0S6K6MXRT_c4`fE=MfBPW0#&|> z3=AwD??i0UaJzTb|Mjy70JPaDDr{=oOkK{lsTdIWL15}D?^n(2{Mv@R*EeCyJ6#iJ zCL$BheQ@AmVCzC)>+dLYJHK^hp8qhN4Q$N?b}cw}Wlg)Z4JB)i{GX5%J=27D($w(g zm(Rm7qG=nV@wQ>yRJL11F^D+?in~?X#kGP{$dz~3Qrtt!AU6&Le68KkjolsGl=1c6 zf}x(Sj#ZD|jv4Zaqpf;M4gfd~w!wAra~&TUyDrF$J5rVUP#H(6O3&-CW3*El$C|E+ zqS1ZaSvH3A=`rL{82Z<8cDQsXU_+zLP9lKhDSMr{pkz zTh&v;jk_|o;i5C8)LQhWw&=eyF43c7Wo4Mpd;hC!{cGdOr>|iR?D(I7-(sHxpK*-4 zGwwC9^Q_)}eMTMTaZkpRF!(0gDv{J`_t^9A%)s))s%3HngOl+(x8w)b$GiF?0*6P`GUbC{5SUv75!|>X>u|e;%CSCo4KANSDcQbi5RED_};Ee@P+~Jz0a7Xv22#$&l%^ei4&XFbYT_) z5r5vPAFs+(Dg5RT*k_F#TYeQ;6(el^(|QFwD(nnC~7WHI8!M$gFV9) zdam@wSSFskK)C(~y-#RI`olAt%t*=}c{%Zs?M2;4Es%}+qHwaer@Kr0>YS7?`s$oT zHIg-4f$Ryh$soRw$A`!TV5ujD+wccK4~azT+plUp~=&3^JUF;-6V4G@!(E@+cO| zBtK}lZ&3ailJv%=0mQ@sM!0cF6NhxHE9(aYl@E2R{F|zoc|l>}i=-l|bp#$0nB`;5 z5ChY;S3A#~={fVdpqeN^4bxtSDKw9vRV({Y6+RaJ)wJPt0iUs$$k7Tr4t@HJ#j19e zL%4bz8X^i?=p$H7G?#x$Ad-NidZ_9|u}16Pjo425i~(Lj7B!t3Q&?CL+@J}>bTWg! zqFOWyEGg2;SUhzUc(DTK5#&`DFh2$lZ$yd4Vrl!U?MM1xYsEtNrgZStp2NMJXU?`$ zkx*D++EZTox*&txxpY0bvaxi0#6#bfYW5OPKp7%KauzK^m6`&|bkyUtFl$dQGgQH6 zi|D8neS4{Bp@F}Q>VheiK9p8eFHu>CMDF)Oml$XW%ZGDE{*N@65uba@bUVxEC?Ppr z)1o580V;tACtU4gMy zmQ~}3GAREIMnrfbZMPyl(r}aOH1B@o zshHaH?(joT-GZlX?%2H(2no!2j!qb%vaYK^EWF2mGclRKCrA^9yxV*8{N(v}`;mpG zcYJburgza>n|B21Xz3^fBL*(Use){>D{kcHH z=SCF#eW05Bt~1g1Uz*)Bdn6lZTC&0$g!Q>AZgx#}&FuS~&bj(I$4 zV)jS1JLV35=jffI^Lsz){^O3t+V%h4bzg?Z+ZViLo3_nV%~pK7>cgrz z%eOY?d|#R{=bgT(69|6HS5z^>)ZSc0-Ghq#v*NdhJ{+36p4+y6;>aWK`kDIM%8$JD zbH?x3@7RCenfF!Ra!tEt-k4oK_uNA5&aAL&zV3Uk+}nNMo_p?OcF)V%OgVObI#Af@C-aquY5x*Z}(fg+N%^b}I>OVK50DDupp4{2l zO)a@#>x1B)`MQUJ{R@Hp_qOK(ou6B&4cziI@MydJPdwcAJ@XykKmOh0_s-?^bpBat zG1N8HLH%dWEe0F1XUNONIhL!~`k>;}oasBZ zJGS}iT+@F1y4Ux^^FKI$e<1hbsfi=`z`9%er}xjC&jq$U2=vZ1d?$1#G=Ds|{YCt` zAN-4Tf3oi5rd)e3oC(KH-v6Z!_GlY;clW(lJ_#JjzV=mAxcqGM_Rsds9-XVb+d04W zo-Nnhp51ZyerNppzGsUZqNDb?hDyX z{Xg=(n)g>e@Ym}5@YY>vzT;jf*V>V7>CA3A`Xk@5rB`^)yJhJtPqtgZj0B}GtSqJaLeR8k2|T?{a{8iBR_w^|h&<-bB7vLOzP!FUTBqD-a-#?9q@lzWSk3^k(9 ze1_t&OKb+!0uD+SH93t`QxwrhvXz?tdrDyD;A9em@XwW?rj;Cg%r2~2VYW=szgOd= zCN64XB4?XsJtd)sIYL+@k^DdC=XFY&s3ETCkx^X#@(wC`juLWlve0xbA}CBh*~P(b z6vl>UsjO(dRBZ<}Yv~~w|0)uP|2gG6TaeVzxIp>c#`;R&Ik2&v8 zxOG3_YW_DT{Dj-|6R!SmxTe41+7`IBzYT28+XG7mv)2MUVqkrK{kgoSDjyVJ1hsF# z*{^EI`wzk@nD2nEBnXj`s2A N`2WMb$OqYa{~x-H$j|@) diff --git a/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/PKG-INFO b/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/PKG-INFO deleted file mode 100644 index da5fd38f..00000000 --- a/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/PKG-INFO +++ /dev/null @@ -1,11 +0,0 @@ -Metadata-Version: 2.4 -Name: harness2-benchmarks -Version: 0.1.0 -License-Expression: Apache-2.0 -Requires-Python: <3.14,>=3.12 -License-File: LICENSE -Requires-Dist: harness2 -Provides-Extra: render -Requires-Dist: openpyxl<4,>=3.1; extra == "render" -Requires-Dist: python-pptx<2,>=1; extra == "render" -Dynamic: license-file diff --git a/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/SOURCES.txt b/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/SOURCES.txt deleted file mode 100644 index c3da0e76..00000000 --- a/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/SOURCES.txt +++ /dev/null @@ -1,77 +0,0 @@ -LICENSE -pyproject.toml -harness2_benchmarks/__init__.py -harness2_benchmarks/config.py -harness2_benchmarks/install.py -harness2_benchmarks/jb_opencode_runner.py -harness2_benchmarks/jobbench.py -harness2_benchmarks/lab.py -harness2_benchmarks/preflight.py -harness2_benchmarks/setup.py -harness2_benchmarks/tasks.py -harness2_benchmarks/wb_render.py -harness2_benchmarks/workbuddy.py -harness2_benchmarks.egg-info/PKG-INFO -harness2_benchmarks.egg-info/SOURCES.txt -harness2_benchmarks.egg-info/dependency_links.txt -harness2_benchmarks.egg-info/entry_points.txt -harness2_benchmarks.egg-info/requires.txt -harness2_benchmarks.egg-info/top_level.txt -harness2_benchmarks/dependencies_meta/ASSETS.json -harness2_benchmarks/dependencies_meta/PINNED.md -harness2_benchmarks/dependencies_meta/README.md -harness2_benchmarks/suites/__init__.py -harness2_benchmarks/suites/jb.py -harness2_benchmarks/suites/lab.py -harness2_benchmarks/suites/wb_code.py -harness2_benchmarks/suites/wb_office.py -harness2_benchmarks/suites/wb_web.py -harness2_benchmarks/third_party/README.md -harness2_benchmarks/third_party/jb_runner/LICENSE -harness2_benchmarks/third_party/jb_runner/PROVENANCE.md -harness2_benchmarks/third_party/jb_runner/run_benchmark_codex_cli_adapt.sh -harness2_benchmarks/third_party/jb_runner/codex_base_harness/AGENTS.md -harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/LongTermMEMORY.md -harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/guardrails.md -harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/harness.yaml -harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/systemprompt.md -harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/plugin/README.md -harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/scripts/README.md -harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/sub_agents/README.md -harness2_benchmarks/third_party/jb_runner/harness_baseline_oc/tool_descriptions/tool_guidance.md -harness2_benchmarks/third_party/lab_runner/LICENSE -harness2_benchmarks/third_party/lab_runner/PROVENANCE.md -harness2_benchmarks/third_party/lab_runner/adapt_runs_cx/run_codex_lab.py -harness2_benchmarks/third_party/lab_runner/opencode/README.md -harness2_benchmarks/third_party/lab_runner/opencode/__init__.py -harness2_benchmarks/third_party/lab_runner/opencode/render.py -harness2_benchmarks/third_party/lab_runner/opencode/render_tree.py -harness2_benchmarks/third_party/lab_runner/opencode/run_opencode.py -harness2_benchmarks/third_party/lab_runner/opencode/session.py -harness2_benchmarks/third_party/lab_runner/opencode/scripts/read_doc.py -harness2_benchmarks/third_party/lab_runner/tts_lab/__init__.py -harness2_benchmarks/third_party/lab_runner/tts_lab/config.py -harness2_benchmarks/third_party/licenses/harvey-labs.LICENSE -harness2_benchmarks/third_party/licenses/job-bench-eval.LICENSE -harness2_benchmarks/third_party/licenses/opencode.LICENSE -harness2_benchmarks/third_party/licenses/workbuddy-bench.LICENSE -harness2_benchmarks/third_party/patches/harvey-labs-judge-vertex-anthropic.diff -harness2_benchmarks/third_party/patches/job-bench-eval-judge.diff -harness2_benchmarks/third_party/patches/opencode-compat.diff -harness2_benchmarks/third_party/patches/workbuddy-bench-runtime.diff -harness2_benchmarks/third_party/wb_runner/LICENSE -harness2_benchmarks/third_party/wb_runner/PROVENANCE.md -harness2_benchmarks/third_party/wb_runner/agents/codex_agent.py -harness2_benchmarks/third_party/wb_runner/agents/codex_harness_agent.py -harness2_benchmarks/third_party/wb_runner/agents/oc_agent.py -harness2_benchmarks/third_party/wb_runner/agents/oc_harness_agent.py -harness2_benchmarks/third_party/wb_runner/configs/harnesses/codex-harness/_defaults.yaml -harness2_benchmarks/third_party/wb_runner/configs/harnesses/codex-harness/versions/0.144.6.yaml -harness2_benchmarks/third_party/wb_runner/configs/harnesses/oc-harness/_defaults.yaml -harness2_benchmarks/third_party/wb_runner/configs/harnesses/oc-harness/versions/1.14.18.yaml -harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/CONFIG.md -harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/_defaults.yaml -harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/docker/Dockerfile -harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/presets/settings/v1_14.json -harness2_benchmarks/third_party/wb_runner/configs/harnesses/opencode/versions/1.14.18.yaml -tests/test_plugin.py \ No newline at end of file diff --git a/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/dependency_links.txt b/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/dependency_links.txt deleted file mode 100644 index 8b137891..00000000 --- a/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/entry_points.txt b/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/entry_points.txt deleted file mode 100644 index f9e6a524..00000000 --- a/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/entry_points.txt +++ /dev/null @@ -1,6 +0,0 @@ -[harness2.suites] -jb = harness2_benchmarks.suites.jb -lab = harness2_benchmarks.suites.lab -wb-code = harness2_benchmarks.suites.wb_code -wb-office = harness2_benchmarks.suites.wb_office -wb-web = harness2_benchmarks.suites.wb_web diff --git a/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/requires.txt b/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/requires.txt deleted file mode 100644 index 22f09d90..00000000 --- a/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/requires.txt +++ /dev/null @@ -1,5 +0,0 @@ -harness2 - -[render] -openpyxl<4,>=3.1 -python-pptx<2,>=1 diff --git a/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/top_level.txt b/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/top_level.txt deleted file mode 100644 index bd82b800..00000000 --- a/plugins/harness2-benchmarks/harness2_benchmarks.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -harness2_benchmarks diff --git a/tests/test_bridge.py b/tests/test_bridge.py index a33ff1a9..d48b3d57 100644 --- a/tests/test_bridge.py +++ b/tests/test_bridge.py @@ -111,11 +111,8 @@ def test_provider_config_and_state_without_launching_server(self): self.assertEqual(env["HARNESS2_BRIDGE_KEY"], endpoint["key"]) self.assertIn("[providers.bridge]", stdout.getvalue()) self.assertIn('api_key_env = "HARNESS2_BRIDGE_KEY"', stdout.getvalue()) - snippet = ( - stdout.getvalue() - .split("Provider configuration:\n", 1)[1] - .split("\nEnvironment:", 1)[0] - ) + text = stdout.getvalue() + snippet = text[text.index("[providers.bridge]") : text.index("\n\nand put its key")] self.assertEqual(tomllib.loads(snippet)["providers"]["bridge"]["type"], "openai") self.assertNotIn("secret", (root / "bridge.yaml").read_text()) diff --git a/tests/test_cli.py b/tests/test_cli.py index f4ee54af..1b34f751 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -184,9 +184,10 @@ def test_lazy_project_and_errors(self): ) self.assertEqual(self.invoke("--quiet", "runs", "show", "missing")[2], "") self.assertIn("command: config", self.invoke("--verbose", "config", "path")[2]) - for command in ("init", "status", "chat", "apply"): + for command in ("status", "apply"): code, _, error = self.invoke(command, project=False) - self.assertEqual((code, error), (1, "not implemented yet\n")) + self.assertEqual(code, 1) + self.assertIn("not inside a harness2 project", error) with self.assertRaises(settings.ConfigError): C.run_path(self.context("runs", "ls"), "../escape") for command in ("--help", "--version"): @@ -256,10 +257,9 @@ def test_dry_run_estimate_and_stream_filters(self): self.assertEqual(list((self.project / ".harness2/runs").rglob("events.jsonl")), []) code, stdout, _ = self.invoke("run", "--dry-run", "--k", "2", "--max-tasks", "1") self.assertEqual(code, 0) - self.assertIn("namespace[parallel]", stdout) - self.assertIn("library root", stdout) + self.assertIn("would refine 1 task(s)", stdout) self.assertIn("accounting", stdout) - self.assertIn("call estimate", stdout) + self.assertIn("Model calls: 8 agent runs + 8", stdout) self.assertEqual( run._estimate(1, 2, ("agg",), {"spent_usd": 4, "calls": 2})["average_cost_per_call"], 2 ) @@ -354,13 +354,13 @@ def test_budget_resume_and_failure_exit_events(self): patch.object(run.schedule, "run_sweep", side_effect=exception), patch.object(run, "Proposer", DemoProposer), ): - code, _, error = self.invoke("run") + code, out, error = self.invoke("run") self.assertEqual(code, expected) self.assertEqual( json.loads((root / "events.jsonl").read_text().splitlines()[-1])["status"], status ) if expected == 130: - self.assertIn("run the same command to resume", error) + self.assertIn("Run the same command to resume", out) for result, expected in ( ({"domains": [{"failed": 1}]}, 3), ({"domains": [{"error": "dead"}]}, 1), @@ -443,7 +443,7 @@ def test_config_task_setup_and_json(self): ) code, stdout, _ = self.invoke("--json", "task", "ls") self.assertEqual((code, json.loads(stdout)), (0, rows)) - self.assertIn("x\td\ttrain", self.invoke("task", "ls")[1]) + self.assertRegex(self.invoke("task", "ls")[1], r"x +d +train +files +yes") self.assertEqual(self.invoke("--json", "task", "import", "tasks.csv", "--force")[0], 0) fake.import_tasks.assert_called_once_with( self.project / ".harness2/tasks", Path("tasks.csv"), force=True @@ -470,14 +470,14 @@ def test_doctor_fake_binary_and_live_deduplication(self): self.assertEqual(code, 0) self.assertEqual(call.call_count, 1) rows = json.loads(stdout) - self.assertEqual(sum(row["check"].startswith("binary") for row in rows), 1) + self.assertEqual(sum(row["check"].endswith(" CLI") for row in rows), 1) self.assertTrue(any("fixture 1.0" in row["detail"] for row in rows)) os.environ.pop("OPENAI_API_KEY") binary.unlink() code, stdout, _ = self.invoke("--json", "doctor") self.assertEqual(code, 1) self.assertTrue( - any(not row["ok"] and row["check"].startswith("binary") for row in json.loads(stdout)) + any(not row["ok"] and row["check"].endswith(" CLI") for row in json.loads(stdout)) ) self.config_path.write_text('[improver]\nmodel="invalid"\n') self.assertEqual(self.invoke("doctor")[0], 1) diff --git a/tests/test_ui.py b/tests/test_ui.py new file mode 100644 index 00000000..d0d9133d --- /dev/null +++ b/tests/test_ui.py @@ -0,0 +1,154 @@ +"""The interactive commands: init, status, apply and the human selector.""" + +import argparse +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from harness2.cli import apply, init, status +from harness2.core.spec import ChannelSpec + + +class _Ctx: + def __init__(self, project: Path): + self.project = project + self.settings = type("S", (), {"path": lambda _self, key: project})() + self.logged: list[str] = [] + + def log(self, message): + self.logged.append(message) + + def out(self, data, text): + print(text(data)) + + +class UiTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name) + + def test_human_selector_returns_the_typed_choice(self): + from harness2.cli.human import HumanSelector + + context = self.root / "context" + (context / "candidates").mkdir(parents=True) + (context / "task.md").write_text("Fix the bug.\n") + for anon in ("A", "B"): + (context / "candidates" / f"{anon}.md").write_text(f"candidate {anon}\n") + selector = HumanSelector(ChannelSpec("human", "")) + with ( + patch.object(sys.stdin, "isatty", return_value=True), + patch("builtins.input", side_effect=["C", "B"]), + patch("sys.stderr"), + ): + reply = selector.call("mission", context, "select.agg2") + self.assertIn("```choice\nB\n```", reply.text) + + def test_apply_keeps_project_edits_made_since_the_run_started(self): + base = {"CLAUDE.md": "base\n", "a.md": "a\n", "b.md": "b\n"} + adopted = {"CLAUDE.md": "base\nrun rule\n", "a.md": "a run\n", "b.md": "b\n"} + current = {"CLAUDE.md": "base\nmy rule\n", "a.md": "a\n", "b.md": "b mine\n"} + written = {} + + def perform(settings, adapter, tree, *, base, branch): + written.update(tree) + return {"written": sorted(tree), "deleted": []} + + args = argparse.Namespace( + name=None, + task=None, + mode="parallel", + interactive=False, + yes=True, + branch=None, + dry_run=False, + force=False, + ) + with ( + patch.object(apply.A, "open_run", return_value=(None, self.root / "run")), + patch.object(apply.A, "adopted_tree", return_value=("t", adopted)), + patch.object(apply.A, "base_tree", return_value=base), + patch.object(apply.A, "project_tree", return_value=current), + patch.object(apply.A, "dirty_paths", return_value=[]), + patch.object(apply.A, "perform", side_effect=perform), + patch("sys.stdout"), + ): + self.assertEqual(apply.run(args, _Ctx(self.root)), 0) + # CLAUDE.md changed on both sides: skipped. a.md changed by the run only: applied. + # b.md changed by the project only: kept. + self.assertEqual( + written, {"CLAUDE.md": "base\nmy rule\n", "a.md": "a run\n", "b.md": "b mine\n"} + ) + + def test_status_reads_tasks_and_spend_from_events(self): + run = self.root / "run" + run.mkdir() + (run / "run.json").write_text( + json.dumps( + { + "name": "r", + "suite": "tasks", + "executor": "claude", + "settings": {}, + "schedule": {"k": 2, "modes": "parallel"}, + "created": "2026-01-01T00:00:00Z", + "stream": [["d", "t1"], ["d", "t2"], ["d", "t3"]], + } + ) + ) + events = [ + {"kind": "run_start", "t": "2026-01-01T00:00:01Z"}, + {"kind": "task_start", "task": "t1", "t": "2026-01-01T00:00:02Z"}, + {"kind": "rollout", "task": "t1", "cost_usd": 0.5, "t": "2026-01-01T00:00:03Z"}, + {"kind": "session", "cost_usd": 0.25, "cached": False, "t": "2026-01-01T00:00:04Z"}, + {"kind": "selection", "task": "t1", "mode": "agg", "chosen": "agg2"}, + {"kind": "task_done", "task": "t1", "status": "ok"}, + {"kind": "task_start", "task": "t2"}, + {"kind": "run_end", "status": "stopped", "reason": "max_calls"}, + ] + (run / "events.jsonl").write_text("\n".join(map(json.dumps, events)) + "\n{partial") + with patch.object( + status.C, "load_run", side_effect=lambda d: json.loads((d / "run.json").read_text()) + ): + snap = status.snapshot(run) + states = {row["task"]: row["state"] for row in snap["tasks"]} + self.assertEqual(states, {"t1": "done", "t2": "stopped", "t3": "pending"}) + self.assertEqual((snap["spent_usd"], snap["calls"]), (0.75, 2)) + self.assertEqual(snap["tasks"][0]["picked"], {"parallel": "agg2"}) + text = status.render(snap) + self.assertIn("1 done · 1 stopped · 1 pending", text) + self.assertIn("run ended: stopped (max_calls) · run the same command to resume", text) + + def test_init_writes_a_loadable_config_once(self): + bin_dir = self.root / "bin" + bin_dir.mkdir() + fake = bin_dir / "claude" + fake.write_text("#!/bin/sh\necho 'fake claude 1.0'\n") + fake.chmod(0o755) + project = self.root / "project" + project.mkdir() + args = argparse.Namespace(project=str(project), yes=True, force=False, executor=None) + env = { + "PATH": f"{bin_dir}{os.pathsep}{os.defpath}", + "XDG_CONFIG_HOME": str(self.root / "config"), + } + with patch.dict(os.environ, env, clear=True), patch("sys.stdout"): + self.assertEqual(init.run(args, None), 0) + from harness2 import settings + + loaded = settings.load(project) + self.assertEqual(loaded.get("executor.name"), "claude") + self.assertEqual(loaded.get("improver.model"), "anthropic/claude-opus-5-5") + self.assertTrue((project / ".harness2/tasks").is_dir()) + self.assertIn("runs/", (project / ".harness2/.gitignore").read_text()) + with patch("sys.stderr"): + self.assertEqual(init.run(args, None), 1) + + +if __name__ == "__main__": + unittest.main() From 0810260bce359bbae60cc21d00609cfb958b7519 Mon Sep 17 00:00:00 2001 From: Bae Seongmin <83870598+3ae3ae@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:21:39 +0900 Subject: [PATCH 10/21] docs: document harness2 as a general-purpose CLI (#10) Rewrites the README around tuning your own agent's harness: what it does, install with uv/pipx, a quick start from `harness2 init` to `harness2 apply`, the command list, configuration, other agents through the command executor, and where the paper's benchmarks went. Adds docs/how-it-works.md (the method, what a run costs and stores), docs/tasks.md (task layout, workspaces, deliverables, grading, import), docs/configuration.md (settings layers, every key, providers per CLI, profiles, secrets) and docs/cli.md (every command and flag, exit codes, the run directory). CONTRIBUTING.md now describes this fork's setup, layout and conventions; the benchmarks plugin gets a README, and its setup guide no longer points at a missing .env.example or the old Vertex-only bridge. Also orders `harness2 --help` the way the commands are used, makes `task new` say what to do next, and fills in the package metadata. Co-authored-by: Claude Opus 5.5 --- CONTRIBUTING.md | 81 +++--- README.md | 271 ++++++++---------- docs/cli.md | 240 ++++++++++++++++ docs/configuration.md | 214 ++++++++++++++ docs/how-it-works.md | 108 +++++++ docs/tasks.md | 120 ++++++++ harness2/cli/main.py | 14 +- harness2/cli/task.py | 8 +- plugins/harness2-benchmarks/README.md | 53 ++++ .../harness2-benchmarks/docs/BENCHMARKS.md | 5 +- plugins/harness2-benchmarks/docs/SETUP.md | 41 ++- .../dependencies_meta/ASSETS.json | 2 +- .../third_party/jb_runner/PROVENANCE.md | 2 +- plugins/harness2-benchmarks/pyproject.toml | 2 + pyproject.toml | 7 +- 15 files changed, 962 insertions(+), 206 deletions(-) create mode 100644 docs/cli.md create mode 100644 docs/configuration.md create mode 100644 docs/how-it-works.md create mode 100644 docs/tasks.md create mode 100644 plugins/harness2-benchmarks/README.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b16bd944..e4fe97d5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,33 +1,48 @@ -# How to contribute - -We'd love to accept your patches and contributions to this project. - -## Before you begin - -### Sign our Contributor License Agreement - -Contributions to this project must be accompanied by a -[Contributor License Agreement](https://cla.developers.google.com/about) (CLA). -You (or your employer) retain the copyright to your contribution; this simply -gives us permission to use and redistribute your contributions as part of the -project. - -If you or your current employer have already signed the Google CLA (even if it -was for a different project), you probably don't need to do it again. - -Visit to see your current agreements or to -sign a new one. - -### Review our community guidelines - -This project follows -[Google's Open Source Community Guidelines](https://opensource.google/conduct/). - -## Contribution process - -### Code reviews - -All submissions, including submissions by project members, require review. We -use GitHub pull requests for this purpose. Consult -[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more -information on using pull requests. +# Contributing + +Issues and pull requests are welcome. This is an independent fork of +[google-research/harness2](https://github.com/google-research/harness2); changes here are +not sent upstream. + +## Development setup + +```bash +uv venv --python 3.12 .venv +source .venv/bin/activate +uv pip install -e . -e plugins/harness2-benchmarks +python -B -m unittest discover -s tests +python -B -m unittest discover -s plugins/harness2-benchmarks/tests +ruff check harness2 tests plugins && ruff format --check harness2 tests plugins +``` + +The tests run offline: they use the deterministic demo adapter and small fake `claude` / +`codex` executables, never a real model. `harness2 demo` exercises the whole method locally +in a few seconds. + +## Layout + +| Path | Contents | +|---|---| +| `harness2/core/` | The method: recursion modes, prompts, evidence, selection, library, scheduler, events | +| `harness2/substrate/` | Executors (Claude Code, Codex, OpenCode, command): editable surfaces, native harness import/export, trajectory renderers | +| `harness2/suite.py`, `harness2/workspace.py` | The built-in `tasks` suite: tasks from `.harness2/tasks/`, rollouts in git worktrees or scratch directories | +| `harness2/settings.py`, `harness2/providers.py`, `harness2/registry.py` | Layered settings, provider profiles, suite discovery | +| `harness2/cli/` | The `harness2` command, one module per subcommand | +| `plugins/harness2-benchmarks/` | The paper's benchmarks as a separate distribution | +| `docs/` | User documentation | + +A new suite is a module exposing `make_adapter(ctx)` (returning a +`harness2.core.spec.BenchmarkAdapter`) and, optionally, `check(ctx)`, `setup(argv)` and +`DESCRIPTION`, registered under the `harness2.suites` entry-point group. The benchmarks plugin +is a complete example. + +## Conventions + +- Keep changes small, and reuse what is already there before adding code. Prefer the + standard library; the core package has no runtime dependencies. +- New behaviour comes with an offline test. +- Text that a model reads (prompts in `harness2/core/prompts.py`, suite prompt profiles) is + part of the method: change it deliberately, and say why in the pull request. +- Commit messages follow [Conventional Commits](https://www.conventionalcommits.org/) + (`feat:`, `fix:`, `refactor:`, `docs:`, `test:`). +- Run the tests and `ruff` before opening a pull request, and describe how you tested. diff --git a/README.md b/README.md index a5e91206..ff77c9e2 100644 --- a/README.md +++ b/README.md @@ -1,184 +1,167 @@

VZXazi< zE*n7xTTo6kh72P3{ZhmiAs{HszoQ?4*CD+#5RkbY=Nn!T63G>VW?>~`1CzWrm?ww> z0Qm%~Qa092!Y2{rf;H7Pyx`X}(jV|$Www-xY1K47OGFt;Udo{Flc8~?3Sm;oxFVw> z`}}~i{1ydwDW6I72Yfxfiw!*-(C2ognwF)XA*T~-E=Ld{6Dy1HnK_GlQa{-du6|%C z{~YSj+oe%^)u)`v_{?V}&Qlt$y#4JNckO3-4wAr@Ya_Gn@|e3k>Z-tj>iE&Iw~j@2 zPaT`?_-W_+ozwg7@iE8tkMoPed9UoA&99E-S4Z<|BGOdlR6#U{M2uE9I4rA^0T+4F zA6?Qgb!A$dmZw5f{n7kwvzDi0mZ$%2-ig+~WHuGwb$Yl-_x5G_H=8$XFVX*-5+mXZ z+%w3~k#P`;NGh!yg@#`f9bv%@jLP}U$owsGU?{8>5KWB_60gDF57VvEzE>;OUNwc zhs;@NSF1`%hK!U~4*a7=Wn&H>_Dm|W{{ULu{|#Ctih1)ts#V9=)s9(rvYxqE=zc>J zUh%X}Sp(_w>THvUON}#;)CGv7 z3=m1>1&=f{6N*fsR6X3lG;dUL`x_#Hjr{K5BC3e3KxDx;t#5&JN=acdss<)Rl5U~d zb}5YT%iP7Hqf;Hh)0&JaO$25Nn9h(mjQmbB`co&s{)Gr*r8Q8&B79Uyiq)}uA3SIF zoeA0dst?vb z3&e)VLw!3FY&rv--~hZeS(or_3+ofBQW*^q*bp>g7W;gcm?J=Hlu74weBIJ+4 zu)pX0g+PM<(+oN*Ub`X^)TDvZZG4s9pn$NMM9OnTEXX8@B&CqvMBue#2>(dr$8a5F ziR4%+D;_Ry<8#u~r`RxB@PjxudK9gnvr8eAQ$_v=8A=KHz0N4*kDbNgqDWQTSu?hC z&Ql1Xl{07Z*la;{te`rYUjrI1f66q~6V2Teb8d#L4Z^x-AgpuekWEmuL=)D{xHoI0 zZ|Q7aRV=S6n!6(MY$Oml2jZ}P7m|%lx(7C*=*6uSkcO8p%*mh`l)JEzGhis zSw;9LVpS_)4^rU$rQVQ}`_RU@^24S#9j`kg&;GzYwf=r_<6}D-t=pZ*E4j7x=GF-R z$}`d2N>*87WnJV7Vht-2E4-1NQ|0q!eNO#ndLsyShaxDQu~$#6pAN)px2SdJmVity zL@D7I8~Jhmqui3&-0E0v^=xi^EVq7oeKfZ|p1bLxQJ1s%Z}kRL*2v`)&2Qoyxnuhk z5urwBV$C!omMe@}eLRAPu0RJ9@T`SufAwPv<@ z{bmZq*n(RYqL`tx^$M)WibS+jf;nRnikSCn zT5yBx%L*l~z`Xmq7quwjDr!acxjkB82!N^PHrxh=%ke(ScFA3q) zGXpXKDvfMx1;I4biqM+LLR{YjVeK%)w6>Jbh7mp+_TbUf31>Hk5Md}lkg1ZCOkoIF zq(WQZdzyeCgQFCM0M|rV7;7xAe4%HsLGd8M06E3AXfwsuQ$Q-mPyQF4piL`(Cb;4Z ztAwdSm8uo~IYqdnJQvAb@dQeOq$R2^ouxuq*^35MRsDrXQy?D&r&uI8CsFe zR9DKMTVbLPQkvm{50nyd*5uWAPPQV(2#PdOk*gk6HY4zEJSR&@WZOrFl}L;+?xBan zy#@BOudknMiL8kne{0KZ<)&EWrn`AFmD}#++d`Rm@Wr>6r0&r#hzFX4mhC;a~j@#hiHZ)XisR z^H#<3R!yz@d0yi~djT77PUFG!`3aerEkQl>__Sb) z9^9OuoxXG#oQaez2+nU-Tair06nQk;k976Yn(WM^3Q@rBXYCPc%fdz~kOKWOE}&Af zvWgNcZ2Fx6BC6R#a|ETK@!b@?0$B2-aztdO%YoX^%8@+{Nj?@m>WBcTpxb9^O53RY zF>Ta`Y}xEj0L4-}0g$mOLvh8yJ|rcKLCTs;yOEF`eWRU)2P(s7CY&ih(sJcnW|+1z zk=LVV6`5&D9JQZ84T>k}lWX9w4}F7Cy!3>=z6Q!E>l(1$ReiO0FRVoVEZ<=y3^VgU zjaq`Ob=I7;aGCSY8wdU;*=P3Qt4GVMJ@SY7WtQ5tw=LgZ!zrRNvpt zddf0Rp602L@l8|vW_;6xOMynU)o~|$s%`K-fIp+U6V-OvEg@uQwpz`de!oz(s18T= zIzo=r?H^fZ^@V~btcy-`!Cb(A8rW4)AqT`Uo?!bS@(dQzUh+&IM$Em!_AjQHeL}at zU&h^F(65nLq=ZbjC8tz4RN)TW*rnX(KSyH=+Jo@8bCl9 z6+IYn?mg_r0BVBW0+z0pAzGCV*Yt!;ETAdU=yEC4=7V~TUHgNs(j{uDe4Cl}2K96U z&?Yp@T!@2uT+Xz>9FqXlIQ_GNeKQT=BW#J_RiKd z$LgBzRX(WO`Re7jTEBQi#O?P=s`xj|RXpx!0Vj~UHVXs;Rw-%jXUGJ7a@7K<*ZHujK`%wR2 z?Y`U3-l_UR`p>8z=H~2bE}1pV^q$@l<`$e`IToSbFeWRXVg$dLr5Gdb6c-yJuNY^< zs6O&Y6WP5K%;H@78Kp9skYKywlTO-EMxH6Ul~?f-`q-g9cAJgYl|dUWBx@9!@GUCM zWKXnaQjCIw6cB}|c%>0ZDU-dZMkZ3^TZ#b6qV6r}FOj2*rmY<(^K)okayi(JwQ64I zPqs7r7@qCxzc(_u?z_*YxW0s|@|~6`!`r(fTcQom+|%D3y0?>AfYrv_wNY0cRIAPg zXjZK@XjaK@e#L$Gz>7b=c<17Ky=om~F($;^LL>)fV^LQXWHu`fkj%z;khHQWYvb~l z%;tIT=Xoa%y*GOI*gf&X*65+5@y26gI};IaXXEwV zW33-8B4kOpRes-IPPA!gA{5^7z+Lf9aJuI1x(7AS%z5BmN8) z&gX%EC4V}Y@=>#q^Rv~PV%3|X)vX^E&F(%K+kG zJeOBE8F*#$TyDYSxmRlE3Ly!5r3WA5g#a7~;(2Wk?It^2H{@cglDeE=u4HMP4iSQQ z$$pUjyC+gMIU&B)U>;mEjyC`BY;n{gbGv-r(FYQm0t ziGN9#>G_qmqnprtz%ssRP&SBKR3r;EFKSYrY~3rpK%r1xDIyxVDn$JWR%ic z&G?qC32lEB@q?Y$6XBB9hh-2E9L7c{5RW>B$K=yAglazOx2U^KdKLXbFLMob5eR)v zr}>#fl>^iTS876C&OwywXJe(vKbPS6$)ZDP!Ez5O*|w zQ29asy<;Dqi$8TFzV>L$arB|x>@dx9=9F9I0>o@bx7f)d7AW&nrCD2C@}ix^v{kCe zUqT6~I9pn_Zz9A`x8XZ3T13N(2D-pNxyqvl4EJfE8+h*Pb#So&0nbF%>h66-cijVF zmtj=TSZ_`EmhK5yIH0hU2OvBUarK>%G>c^LfV6KhL6jlHuIs`1V$4zy3^ZH~jM**) z30a}aOOJSQNi3$?w~SmGkxyQIWN)kH0h6~K!l^@s40~UY-&xptl@Y+aK#aw%I05sN zX)~IJSzzVJN?_5W)}Xenz9BHIW|y02nqOFXrbnp3s*pW>7eR*4sm2;*l+nJ?c;N(+ zf=$XAr!P)P#cUAAS%U`-!a-n|_z2SV)$j3jcl#t5CXy(dhC5mK2(gc)jMX%|20lwwi%8#>%QV_*xjHnrjCCXTGm#_4jWOO?%??VcJn z7vO4sh=-(vCsWZk^uMORz7(5gJr)fce^fWhr*5N3{lawW+LL?}gjd(lW6<6t9;A`V zjKT!);{%MaOY+@GU2lI1`l>L>$g&2DMfiP4a!()w__CB~f>^syGkmJ*>VMpJr|rGn z58S8k?0%~nWXHH`q1Qhl1O%}A1&zUQ7z+d+*qzQ!R5R5gA?CM!%7L2%pr(cx*3(~y~q|io68kxdI`kz$t5tTHcW*K1* zTr~d`eSMz3@@JF$kkS#0x`7^JtZ3C{O|oI0{sqm|3IwOcaH7) z*y))pxYn5{TsBu$0i+dpdozZ({@oFh3YW*rwmvi)2!e=-`Hnp5!@6ol{$-+(a)t600gE|{}*Ck7Le`gNjTcjErJaP+12Fz zr?H0^k|wrAqI{Aa3xcnVw_!E1uM|@eI?KB#9Y=Lxx{vDmdkR?VRx*vH*0Z#@oXVzD z{H$)mL0{U?#S0J_pbWeDBk0kpNZsI@@iG(A4&wlG7mRQ8egIp8$fG zrjT+G9oR@s|{PNC81#cHK9#ut*6g zwkp?)KP?p@75_N-4-i6n!1E86nRvr5Jse;87o7c%IqRQr<$uC0Xa9;nGUEcz$-MDh zFMMar0JxlAcE79@a4^bw=1rVc7&GFd&OTW+!x#MzT_ImL&mnlE=XAyM^z-3bN?kL< z7tHHCyc?H&&+F-3u|j#T+orttwzKy~#?nuXNT&)4S&2v0Jb!|JWHIx_kMi^Rryn)w z_z+KC-#y z(cw-WvtcQlH?ydPD_T8oWlb8`L)5tTMdO o-NMp&E5#s}SvGH{n1d>%+!!A!awGOA$lH0-XG`;W None: ) -INSTALL_MODES = ("workspace_agents_md", "home_dir_jinja", "workspace_config", "codex_root") +INSTALL_MODES = ( + "workspace_agents_md", + "home_dir_jinja", + "workspace_config", + "codex_root", + "claude_root", + "command_files", +) @dataclass(frozen=True) diff --git a/harness2/core/tree.py b/harness2/core/tree.py index c9bf1341..45cc7268 100644 --- a/harness2/core/tree.py +++ b/harness2/core/tree.py @@ -25,6 +25,7 @@ import json import re from dataclasses import dataclass +from fnmatch import fnmatchcase from functools import lru_cache from pathlib import Path @@ -56,19 +57,27 @@ class SurfaceSpec: hook_events: frozenset[str] = frozenset() compile_skill_scripts: bool = False reject_empty_always_on: bool = False + script_dir: str = "scripts" + skills_need_trigger: bool = True + editable_globs: tuple[str, ...] = () stubs: tuple[tuple[str, str], ...] = () @lru_cache(maxsize=8) -def _res(surface_name: str, skill_dir: str, plugin_dir: str | None) -> dict[str, re.Pattern]: +def _res( + surface_name: str, skill_dir: str, plugin_dir: str | None, script_dir: str +) -> dict[str, re.Pattern]: sd = re.escape(skill_dir) + scripts = re.escape(script_dir) out = { "skill": re.compile(rf"{sd}/({_NAME})/SKILL\.md"), "skill_script": re.compile(rf"{sd}/{_NAME}/scripts/[A-Za-z0-9_.\-]+"), "subagent": re.compile(rf"sub_agents/({_NAME})/AGENT\.md"), - "script": re.compile(r"scripts/[a-z0-9_][a-z0-9_]*\.py"), - "script_ref": re.compile(rf"(?:python3?\s+)((?:{sd}/[\w\-]+/)?scripts/[\w.\-]+\.py)"), + "script": re.compile(rf"{scripts}/[a-z0-9_][a-z0-9_]*\.py"), + "script_ref": re.compile( + rf"(?:python3?\s+)((?:{sd}/[\w\-]+/scripts|{scripts})/[\w.\-]+\.py)" + ), } if plugin_dir is not None: out["plugin"] = re.compile(rf"{re.escape(plugin_dir)}/({_NAME})\.ts") @@ -76,7 +85,7 @@ def _res(surface_name: str, skill_dir: str, plugin_dir: str | None) -> dict[str, def _rx(surface: SurfaceSpec) -> dict[str, re.Pattern]: - return _res(surface.name, surface.skill_dir, surface.plugin_dir) + return _res(surface.name, surface.skill_dir, surface.plugin_dir, surface.script_dir) def normalize_rel(rel: str) -> str: @@ -95,6 +104,8 @@ def is_allowed(surface: SurfaceSpec, rel: str) -> bool: rel = normalize_rel(rel) except ValueError: return False + if surface.editable_globs: + return any(fnmatchcase(rel, g) for g in surface.editable_globs) if rel in surface.always_on: return True if surface.hooks_file is not None and rel == surface.hooks_file: @@ -150,7 +161,9 @@ def read_tree(surface: SurfaceSpec, d: Path | str, *, rejected: list[str] | None def is_tree(surface: SurfaceSpec, d: Path | str) -> bool: d = Path(d) - return d.is_dir() and any((d / a).is_file() for a in surface.always_on) + return d.is_dir() and ( + bool(surface.editable_globs) or any((d / a).is_file() for a in surface.always_on) + ) def write_tree(surface: SurfaceSpec, tree: Tree, dest: Path | str, *, stubs: bool = True) -> Path: @@ -194,6 +207,8 @@ def autotrigger(surface: SurfaceSpec, tree: Tree, *, always_on_max: int | None = A line that would push the body over the always-on cap is skipped instead. """ + if not surface.skills_need_trigger: + return [] cap = surface.always_on_max if always_on_max is None else always_on_max body = always_on_text(surface, tree) fixed: list[str] = [] @@ -289,7 +304,7 @@ def _check_skills(surface: SurfaceSpec, tree: Tree, body: str, errs: list[str]) f"trigger line naming `{name}` stops matching; drop the `name:` line " f"or make it `name: {name}`" ) - if name not in body: + if surface.skills_need_trigger and name not in body: errs.append( f"skill '{name}' is never named in the always-on text, so the agent " f"will not know to invoke it — an untriggered skill is dead weight. " @@ -358,21 +373,25 @@ def validate( f"this framework, so an empty one is a harness that does not exist" ) - skills = _check_skills(surface, tree, body, errs) - subs = _check_subagents(surface, tree, allow_subagents, errs) + skills = [] if surface.editable_globs else _check_skills(surface, tree, body, errs) + subs = [] if surface.editable_globs else _check_subagents(surface, tree, allow_subagents, errs) for rel, src in tree.items(): - if r["script"].fullmatch(rel) or ( + if surface.editable_globs: + if rel.endswith(".py") and is_allowed(surface, rel): + _py_compiles(rel, src, errs) + elif r["script"].fullmatch(rel) or ( surface.compile_skill_scripts and r["skill_script"].fullmatch(rel) and rel.endswith(".py") ): _py_compiles(rel, src, errs) - if surface.hooks_file is not None and surface.hooks_file in tree: + if not surface.editable_globs and surface.hooks_file is not None and surface.hooks_file in tree: check_hooks(surface, tree[surface.hooks_file], errs) - _check_script_refs(surface, tree, r, reserved_scripts, errs) + if not surface.editable_globs: + _check_script_refs(surface, tree, r, reserved_scripts, errs) if errs: raise ValueError(" | ".join(errs)) diff --git a/harness2/substrate/__init__.py b/harness2/substrate/__init__.py index 3fa7f651..e048142f 100644 --- a/harness2/substrate/__init__.py +++ b/harness2/substrate/__init__.py @@ -12,14 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""The two solver substrates behind one interface (substrate/base.py). - -`OpencodeSubstrate` carries OPENCODE_SURFACE and three install modes; `CodexSubstrate` -carries CODEX_SURFACE and one. Benchmark adapters supply the launcher body. -""" +"""Agent-framework and command executors behind the substrate interface.""" from .base import Install, Launcher, RunRequest, Substrate +from .claude import CLAUDE_SURFACE, ClaudeSubstrate from .codex import CODEX_SURFACE, CodexSubstrate +from .command import CommandSubstrate from .opencode import OPENCODE_SURFACE, OpencodeSubstrate __all__ = [ @@ -31,4 +29,7 @@ "OPENCODE_SURFACE", "CodexSubstrate", "CODEX_SURFACE", + "ClaudeSubstrate", + "CLAUDE_SURFACE", + "CommandSubstrate", ] diff --git a/harness2/substrate/__pycache__/__init__.cpython-312.pyc b/harness2/substrate/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..455110aca4a1b999a2961ecce0bdcc3d25a276b3 GIT binary patch literal 718 zcmZuuJ&)5s5Z!g0*!gghqd-*11r#YRCR_t2gcSLLgcMy)IYmO-tQ>pplEZ6g$Yl9+!)+O| z?)ST+qaHmS4WD)Wo^1GWkVo)ui{lTDdM~yebw7<@zH7j;>8~c;)ePAjob--0am(8q z%peIBch^?s1vN2O7eKtIp`^jlP}87~#LF6N4VH$gg5%aLX;B)bA*IqB2Lk>lBunA) zUsgP=AdA|6zLM19(poQhDJ$)CJ>tq<52Ah4Dk=SdZd8i?oOmN;C%l6Iy!K34r=Gr8 z#sP%axyg;dN=1l^gfDjKov01!F=`@*Ul(uKr(tg6yD39SOzoQKF4aGeL nKMNS+Uw0Ai5)p%?1+p6&|G!Vw- literal 0 HcmV?d00001 diff --git a/harness2/substrate/__pycache__/_native.cpython-312.pyc b/harness2/substrate/__pycache__/_native.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b860f0bb77385a23719770c83c9bc8cae1174994 GIT binary patch literal 5433 zcma)ATW}NC89sZHR$ARGncy2(fC&-7Hg`zyz<>>eOKcJoC?Zr9+O;iNmNaKqz9_4X zCesHiu{^LOCD^f8MQ!Y?Tzl6edBqBc&DUps+Beb8U zNSTQ;enx+?epY{Seh!{&)HGuDo56+?O;LWt;D zmuQ8jQ>+sOc)Bb|bX(BMKhY!Fe#ZIh#Cp*VR_+PLw zQkg*tMqotLgg&79k&o@gWNE2pV=KIu!?7Tq@WL*@7Qqg_>U~0%=i@X^9*Roi@JK|% zie{3BV6USZH#QUw4Pi5}VKq~^yJi-pfnYMKXlxJ%bhz$lX!G|$NtW-9qq=-<-r^IFmD%$wIPIvVpW`|g?U?>{lOCBN@vzUfrKaXQbRF52qzf_FyA9nKw@ z8HZw?^)A5-=ot)4wZSACRq9@Ge(V}apnRVqkN{` zl%>6W6b=fU=Ij$f7d}SgLPJt$SaXDe2_=c8Ks>1=l8Ru06F44%L)93ZjA`b+e%-4yXMeRte^J2~29`qDNir8?8Y7K|s*u8N z^2(x56#y{cR;ahZPacMh;J`C;EO)#hv}JlqG{gC+MczH_$U5?CU(VH&>>bGN?*5$3 zpZn1x{=&Z&ZJulXg*Bb?YdZ77_HPkIx)5|6xOeP+*XwhG`2)TAroMvXO#a-3Jb&Te zC5D(hmDj+0zgV{o-Dy0$j`@u99ByDfYv7>#3{t9WaUaZ=BV|>#sIdg4QkYW#8^bTb zJ_t3w5SM#8O4AB)NB6)ywXTQnMgGoMtXZ#_6EK3#7Hz8 z8wRkLHTt!)nmrcBBf)4mB?Yh))vV`&(WF$y25}6HO9ZhZYu2zFsB|?riN*1_0{DVn zEE%8VU`l!sCMv_mro%=O1H=Nq3~k%sXCTLFTd+3GTbpvb3f9(4_ZNa=W>d~{b?c&| zA>Vkg;5d}$4;2Mxrf1PrKhyud<+}5lbCNAOJu_nV9ViIS=|kB=4~3RR_qx3Iz`gE! zd-CfK7u-kkwj(7os`Hk3WOHRs;d+?#8IX`8)ahg+u<&)Hq6oZLnR?JPJ(&-YB@=%-h~9dYWLY?)UhT9W`z)IgrhnIr;sP2iZGhLP|5XBZ4VJ z{KH7qpk@VRT}OZ<_6L#3O`BFAkxRpwRgh>-lNt?3TL6smDqo>7TBTw_EhMUx${R>Y zTU1M=DvFFH)#wN}Eh9Qhq$-1ImNjBDa1^6e%OI7us#aLZ46H;@h38Qob^y%@t6@~f z{YGdbQE5e5M!R(@nxJZU9e5?Dn)PvxDMY*IDMF}VMFRifj9tMSp4qMH`^efAouDeR zB6r7R^i)}u)BPt6vXPp~s7!o6bO19rKaJX| z34uUjA`}D~4FsNYNoAmYzh=f#A_^||5v}o3EEE^Ru|drw5#22N?7H{pRI8bQwZ$+7 zQ5naIBx)QUjK(i(W_V4!219s4ng9&0Yx#f@*Z80uNW|svc-eV7iGc-6K{236`7jL)Ud*v^O{(NYD>&c?4q3CX2aBrD+Z@JAD-0chQz4Pw91^52R?xNE@BV|=E zSwS4lV0t@LXx_Qd+%@0a1tRBMAm7|oX#U9zGt+&QFV;2A*KLM=*PU0Kv&=(R+czH6 z)CN(^-B@a>P{}%jO0L?v4*?y(9vF#)1wEjR>r?^(sRA>@fIJP?03Gt2))a6pgg#JJ zVG{Vj0m)5hnOh*0F+l%5JyoX%LS_)cApDkvH&_E zC~P&#F?!>*&@QYP8me$lrtef&rs12bYUBb=&EBcHJ=`Pt$SNeGnnkMS z`e4OHsuWXO73@>Z*%cPm!uaz~;Q6PQg6o1YF6`bb`uY5y`$jY-d=XL7E0VFZ@GYkQ z`7H;CL{Lql`PZOGb)f)42#AaC#jqTNUnSWA$j zV>qlpfG0QGs07^M6Et>Mnvit@!-vQ?9V9zUvL47ZQ+YUz8yP0tW+fv?Iz3^+44Nq! zBMR4NDbo|D>jRCw40qwOzyL|B49U2UjCh7*yGcffNhU$A%0LwbD~%_BJu5Qf0vKE( zmUVEjAT##7`l-pP>&rbaH)crku1>2eVJ$<)Bll|AW<&@d}g4;K5_vQEW zWsc@;XNnG2=D40MHmuJjZ@qKlotx=L4eezeQgnjO&8EuIwkz8;bucI1irt9a9h?j2 zVukf*@@FsJUw;vFNs+fr3t3@-Z=B~FXWQ=9=lRA5{9f|fnRVvOx3^3>AMiViypvqT zX5@!_Gm#}7-#L)$g&FaB_)7Tw;n|~qIQhGizdt?y!uChb9UyyFO#kzPMVo8-RQA-= z>qVPqx;NXK+x5`aTCyP5&XN^1wBFt~FYL;*yT1Rn71`gQbeZ#Eea}7?uA<~lsWI>m zh4RJHPpT#?!Ob?7vP1z07oqJ^#SoL`njyYOtl8dpOacQEW-!rB<(Q(I5(E7mw3Ar$ z)bZNcj{dR~?N3M{T~IzX!^^WIZs_LZ{}w)~*Jmw9FLI8{#y$qslv7D2q7qrzHg88q z$0dB6ymeKV>s7sEQOm~Tmkn`_j7Vm|=c^?$bE;kphxG5==B*BO$>-6`fq)nf1p=Bi z5Ew}6m;L}Iw2sNY1Tdj2+yoi%A!R5UzN~+;#)jfpDobXfu{BmU!sjmCXU*$Vsj6W~KiR%Eg-2~hfOyNlYbuj#+}L&<9Z literal 0 HcmV?d00001 diff --git a/harness2/substrate/__pycache__/base.cpython-312.pyc b/harness2/substrate/__pycache__/base.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4605aa7db2d54b533d5ddedb931d95ccb2bbeb1a GIT binary patch literal 3794 zcma)9-H#L36`!%k-`)?0rI3cCHz7^qhP8o^mI4V~NJvp(p=?mAiO}rbnQQN4$1~2o z<7MqAQqs_;zVL=B^`Yuh5qZd8(U-nZK_xm;)vcTA11Flq(YxyGZ zFvYsexb>6cv(hq1pRbb>P9&?2QZQ3Yz?QhjlUXH4dwK)(y+fc~f2v?|jL4^v|32@)$$Pc*S3v zj~SowN;mR$(VGS9S$3&ksn?Rym~#d{Aqr~sf`E-9WQT{4LvAERLm}N8<24ujHL=Y6 zG?i$Th#5^|p%9)LdEcTScSocPE7K=y80OGI`zTn1>2%@wF_8>&k2D(Yah;Y- zA%{Dhy~Ng$>mV`&vrh1M6$g)1Z7gbFZiCpA2wIBHW0DX`3M!?lL2ayfW7PEu#)rgc zAEuDHq|xNFTg@+*zz#xF$5>g)I>sk-nGeCd>u{zY2yk>)NVcpU=cnvNG$586&|FVsX-gX0m>up0?+g1z@%9Ic7){r6K?x>2ABo~a-Ln;|WT=K^u!loY{h z%j3;)ZRl-@6zN`RN-tp z#)kMe5rVib%3QXU6XQ(RrAVU(HtA=VZTvKrToK?m38N;X`#TEJ(T#gZEx-up5t31g z6Nnu+0o4xbfXSeWRz~M8r@_(T8@R{0RQSe5X}Edtj^4)=L4ae`FDPtlx%cWvKNmXKCF{A>SBLH{~>nt@TiMEH|o%V z$K&nCS}-p{Mxd6_>h`rEVT6w$fNl{Kq}WxDL}&^sUFN6=I!+h0PEr~>Bq@!3A&CNZ zgFHiJ1Tf?82z|v5p4^GtT|=*9`3~s+ljeT;Bf$e?n69TN)d8n~k3-QX9z`03@ku5* ziW2x8zAth3GoS0H{Ks`E#$Nq97fDZzo{}eU=BsDAaSr;u9wKU4aJONJUemIJM zStkcf&}FHMn=tI`p(3P@irQo;yqR zLG_GQAh8zVXKe6aulAQ{?GX~kNSq)+waM-~{{`wJ(Zs*{IRxGLZ*P{TJ~}_N%-kh& zn9t>^ThrOx&&&t=pW4dm#tZYK#_@$wd&pL9t{*9UYX=s=iI!6Nje zW=B#lLA8tY#lc^#q>|#Plxtp-Z(9k;S7YIx zvGA#J@L}G_RBk?h-`w}_OwK&DWk3wGMrQvOeLkF>9Ier4Z9H1rnl-9(H|^i-yE|9^ y-LX%s+b8bMz4FJ(_dY0YVb?vMXPCKQ-YgCcdegE!Te|CC=S-vW>W2mG@&5qg?)1w5 literal 0 HcmV?d00001 diff --git a/harness2/substrate/__pycache__/claude.cpython-312.pyc b/harness2/substrate/__pycache__/claude.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b632409b037821dfedd3f7582581873add745b03 GIT binary patch literal 17276 zcma)jYj7LKncxiGZ-4;!0AGUmkO+w&C0TD%QYey=C|MLOQjQ5(f*}k@fdT<~W+;gW znB``DwNy@fB4bw(uE=XR>s08q>cZ^0YL(jTMRu|kX1-- zms9@S_w`@~AWbnV+x;wMpzk9`jjMB`BWrV`_v@Y_%x8KBHA&XPe*Xo5&f9KX8>5kXd}ik zlg~t8UBosW)Yfuy+$*3#sR65 zDTlOzsf4tOaY0(m)IeIx)InO$Y=P8G;H^vpz>SO&(k7-E(rwUV<5h~;{#W#SuuAwq zX$#|lbO*uS$+QCOC2$+l{u=$CXkQbv%h$~8_HAR1FneBA`L;7hnP(txVU96-A@?Y# zlN7V>Rf^gFo~pCMY%qBSM>+@tB`BY2FAz3(UH$^uJEph zey@k2Er*rtM06~`UPFB7JOsIMK%B?GcqoXkU9sqJc%+AAW9(bL&Z9rL~oSi1CfYFDH!~N z=P&ei`A?i36x2AtxS;d%p_{xvHY}*3fw7RF36kX#^nu8Yz$E97MLqNreb;lH7rJ}e z#u&jYm;1*8H^X1LOHqQeSQZS7`(YJf3F8;#M(!}@j>G~?h;iSz8iTzENBNjLbUnmQ zy15X?!J4*`9Vpp4_l*D>g?+gagT30jt993|UGCu302>JMu(pDpyB3Z_{7jgAVrdhn z+)j!(!75ha*ZA3q2&@Qn!LqO?9O4Tmo(+$TgjoMon1(rO}$HKf&9)uV8O24>FT#yZq^W30N!G@3?{2@S{ z^vgRc7(@)R)K=-E3atc(xaR(72+n9>4FxsFPh8>Nr95h~l59PEX>pgxDst>TEG(D% z3-LS<5l7aEJq&fh0!IOmqBQWQq$qj~*cfnQUam_}e1S($l*BEj=mb5gChgWG?-kxS zMVK0(^c1g^sz-GaRLr3dWAROb82X+<;&oC!rQjV>eL}%IC0LpdXW`4FvIPB2ic!5O z_whA4p%|bVDc&Su@D>RgRy0v;#guYGYh^q60BjDf{{;zT+GCg#2#h=dvU>49^7hu!fS_t*r-yMwVP9|%X? z0aEUcUBScPF|s%vLGK^v8652GJ2@a|LN~)4&j~7WqSZ*&K#f51u>ZhF1j7q~$V7G2nN-~kXN@h>-8rN6&a=0ky?rESEt}Ej z6-xC3pmHjrzsPWPR3kC~G-9qkOEPOck@iyL*?7?p2E`IhZ_=Za9*z zKAN>0OY4s1jF#!szk95w%$0XdX=C$mDVp?FK-|B`Er!z79a+nf2hXH+Cw`w-V*L}Y z5@z*IbEk*;z+*jbQGRf+?6_X}pY*H`v69@Du)gRG^O!mdG*7c!I&%P(aiQ z>^VUv)C^TkAq=xcj6X12sXe29Q(5TY540%u&uDODMvIK;P1OM9(e=gaU+VOFy)UuH zUW&G~?KtMS3~w#R4wHgJlP6ww8SmD^7u_%Mmv#V#r-?=O71xkrw?G@>254dsU*l@> z;?WBVHYBLW0(|hQKo1D?SwX{v0xZ0#N7&fJINlp@7h#6?m=Yu~kA_7W5L8!U%%or( zW`RjT2Ot3mdUC(x-OPR&P}mnC;qbU3&v3ih=dhj8FW%YMoP7yO@Q(u-8;RZSoNt+H zN!N7TXMTMAC*2=(|7_1M_WpG52WPVO{^>IxYwfGfiuvBT-gNz*#WO2=dop`_9t5&` zPyMXn7fnBH$~w=@=sz|(ldiY6ys>4WEbH32;@X{Y?Ot|0lXFzgoPoNWbIXdeIpb_j z1+vbTg+t#x_U^Il&W?<;W2Pr(uUoMKG zN9x4F-i2+c_KaiqV)uQ=edprojN@q9bTnsmq@6ovudP&U&s1z*ptBV_($<}enniwb z`2O*9+p&!G*n_4AUs*Z%V&>$F=^xFi$AmqhpDLttKiG>nHeZC?aT4(g6vaJ&paG!36hB z?BW_Aj3OpY_(#rUneNG%Y|}lfRn6~27ny9;vny4dnX1l@s=98OW|T8uhI{|@wiWx% zjD6>laVN+S=01A0qI!OGZZuVyt?}Yx4aQ~Sf9{B!&2M4mxp846} z@~-o%6%4MmM#Kb^0kT-|iOmNJ#i zHqUgVRh7ScTn@7V;e<@HF!Yi8!TT4|4PDua?uU(O+gTJ&pk5SC{-M0frTCFc-*r&^ zqlU7sXVgD>Mh$QfR!M#eq5{3P2u1T$i1Gm&@hKP@tS1VeDuh-OutG4(SJhxQsbEx~ z$!nQPMh&?R?Iob^S1~%!_w|g6(L<>LY!n8_jZ6(=gxmzy3KQgJJxKMX|Cl<)4CNNa z!dUOpr8Oc9_6-~KYh|`DWq@Y`ONSluGG;5|fZRT*@iYilQ7HxONt7hx2i9q+aO^;= zjzvJmWZdH+)(a}Xdn6LO5{RHo4Z8gV$e*C$wy{;P0PEy7q-ernkq+jOaw$Bg;xkFT zOQWGdc_ke6C4ww!{o!bs_xs~j8|K+2)x3&rkQSyXI4Co!ztFLl z;BB3Pg*=_i1jT)jAR!f>0g8fGIvivXov*@^30ic{4Aof?urg zG0J-~kCr&`luAz}sLdo!UY0XQWreLc1>PpX30mgcsxWm=ElIO9RiZSic=q=1v{tGq<_Q&}y{D6ANmE~=WHnOXCm4go z9OOq8Exv!l78h;gj^w{8`yH#uLZz+Y|zG|}HIeY7Da@(Tn zk!fGfRF+pm=@Sl*{|CCx%HwY8Lw7@$LW!(OCz_B#jLhZm+{)DbEnr-w+G$b#1DOV- zmiG!g0Zbk;%|eYrI-66fA_XT@qq1BGV-%jE1}9OBD05D!6Y9;VSklHg`nD4=WSW-x zLv2jz>5DYNDGg|F>QR~Mi|s*91FdXI3$?ODTC7c|h?IL8YMsSW31T#)_E}z*Cr4$< zmT;&N16V>UtrWYpbOjR{seDxC1jY7@meJkQOBe}dLQAB|y9vcKefuroH+4l^34$IA z9EqlGgUrcqk!$4jn$j_bgzh@UUf{RNwNQq#E#DxONaJ#LzF8{cw@DDJo^c&7_eeFP zJ0z%><9K5KVVW`BGfRCl7NQlG>Km=aRuY6?olt@XZ0++Xi-t>LsALa8r9kgsUxqxc zZ|^+W(>DkrEPDmsS!@ZW{?K>?{0r|Y*dzF^5&0GoJmZzJ+`L{|W|2041!}_JZ2*z< z9K3oeS+pC742xLbG#r@vLyVwiQLYfQVbR1a7(tvRz6?Jf6LbO2KOW=4H$hSWW(fQ> z06q*JIMl>2@L6^m`b6s+Sau_^8zF}6!U7FAFF?;GhOnYwEy~%)a2B12rDi6^#zl$T zqbJ;uJ%yFGF(Fd6reM|W#gZ_#(FX(37?}S85kV)}+lUlG*klQ-437~{26ETWv&xdn zdFYip3dauRi0V~Sb+S9%FqnP;*`m!bqe7lopJdWad$VQxW;Cl#U~+GDywS0s%+|Di z(VVR+XRlbXH)QM$sl8cy%Zhz>#=bjie+HEA>gE*lo$%Y?)Wt=8 zy6W&u@258Bym8K$yuM^>&N-{*&&-`k?OSrTpn2fz+}U^97Cp<(j$b#lE%1v2*@iv3}K2m%N&~k*RNg-?{AA56ZwhL#Z?Ab}%iT%vPLA+fMx&W@eZ( zBu}PZTo}nT?@!kpct9`Pjz1X8*m_}XbJ^?lD`h(}@ZYo(hPF9pjH2`<$xuke$ZO%9 zUISm{I{sYbeMDzhfrgFv zS{gPy;6Rma3N#f3{+-Y>G@(&_LPsRQt$)BWpl1~Klrj%c5gwo_8rre15{*~^TC}Dp zMS%&OkS?K6_8WjJ5~q$uHBh7%Q3oB5jf-mPUt$Fz8oUQQ29Z#ZJVa{239wf&K_OpI z#v+WMhr~}5X!dO^)Qp8#lqU@EA`^-zX|sOz5|QV9K;UK|0Sc*XN)2W!cYq$M zo|#y6ZAfx=4climvp43fIcs^cYRS5FwQ>7*4!?bPVfg;@%Z*@X$~2ysRlgq2*xaku znq+sXdBKsjdeb_uc-Bx|ksjP>a@I}*0Fpj6DxW9`g`&bl6B?DyXtAQ?HohlIyfg)x zlA@2D1U!#|tP=Yz5k_4_(4(_}1K(O`o#JF7#>Rq%AFtggqm|Zv2;FcEkbsb;Gp*>V zGP6xZ*vM@gBJ!&U#O!bybgaz3Zt9niQ$)GQ2$Pi(wr${o2i}^AhzWz>T$j z71(!S2)MiOppYdHj3}Ohq>ExJRFL)176fhfuK|t2Vd0>O=2BS(NB`{S*F=)gvGyOa zu?LdUHF2)EJQfus1_7U|vuJDe>tzK_zf*+sYy~8mU zgz-2mlkEAA>%CrZ7hj2iRViR0s(hI)MaWAO&V`1X5=Hg{uZ{ce5ak%?g=Bm6AArocx=FJFMD6>0nMaV{(Yx89Y*(q~H-hq&lGTZaz2vvZ8 zD_@CF6=inhT?kQTy+Pc06--KcWH#_Wp#+FRxfvUPD*(N%1@=J`wy+tQNJ4x!bw_na z9aM(F$Z`n;UFp(FsCaqo%?!=gX%PQi!2Z>*DQ?%PsGxF6J*E0KUA(e3b88hZw{OEa zu-XSu-lygx9M7^xrmshB4hMc?Q zJC?UC%kEvvmgh6>U2mw8baMA%=MRQfj`U}a^e^r0pLu0=>n%R*-nC?TKCgr}zqVAa zSQ;{xhSb5Vr8TdHB0P`ocN{6-Uv2q~7T+nA-thl^X?dQ*#a*cGY}b9DR73h%8dd|O zW&<2}4x$bwqCr{VM1CeKoNjn2RXEW!4hoJ6*k<8G0B2BuxDWU6l|oN~9>QAu7-_<> z$mGxxAqE2#0yku$6`2N!%wc~5m@LvBX|s`Dt)hFJ4WLgw#!kk!Y+ULM2>${-an+Da zQy|vb>T;&a)S>(Ohog^C)&&!QL{tUh08DbPigzn2iR(XGx-(>Jo?r$8NB~SW;3Wsq z9h4S4J?wvlr1X|#{~Hv<>o(4IUE9yF4QkFPm)9mQX04vI&hz=JA9n`+=(Rs)db%J) zyZ9$KhW$BqkDpn%q}VjTpEq_&`2P#hrP)*_jah3`TGu4brc~6x{U@{81;Dz+qQ3a% zL2(1l47p8!m*Y@`m$PUyEj5fnAXsQP5+1qA$6Xs2P_Fvl*a*c%SUg?XY!iAUlcyf( zn#qj8>e>KxFw8h)><#tRhoH2}Okd1|j@@_HSi~jIEbP zgt`6$)-7Z5Um%IwB@W27^%r;{`$McHos;I4;?!{wV00t_5dmphP=c@{67~vKAV**c zgw4erQI)iLda0luhFBTA{s|8GRtH4ifdmS1YJLdKL;ER&s{w5C`(K#|L<%)pzaN5P z{C?3DZtCkC?0uof-`_cSYJg>N7897@olk^5QT)S8kwwEIi&_EuAtwKX2~sqnA$AF& zgP5Q@fJlJs7($qQ75=ydNM5BrR&{=AbU;LmW6Sh&k5#m~a!o_4>(?|!wLM==outpu z|4!AYook0p>bA97T78;Egh5CxYD3)YX=KTXC|5R^UQv=?bg;wvtftnvfvQ7L5occP)hAd+a0`<>> zcW85!17wX7WpOhpgHeHlw!m(u=n-&XLxh+fhTHJq!s7#m$HN#V0uF4WSOVTdlLQs> zKi+{2QpygHlw?;yNguN77|d}U-~t9S7Q~d{F((hY0j3m;ro?%tb;OwgViAQ5APpunSxUR(_A;5IsdPhgXin7j!IFq2+vu4Dr@h$j~W zW)fT}sIax57T-`F038ROf|>*J7Ya!PmBssO4gSb!2QddCy^iJ}&?|Z?ni^RQ&qTA9 zFRfTkWh|#2R-~UF$XW(xlpv~DY||%lHpjeW&hnAXJ)?ql`tnsU51pPno#eC5rkNhN zx*;koba(KPu_0Hmqc=Y?x$|bKY%56BCj0bR^tqZG7!Ys*B`G^esIik+@bu? zOauI(Lyc*T{`hwFhudgK*&jg@_Q#OO1OgWzgX!`j=-9QFxp8Uslq; znlYoL|7oMl(ziWN8Ao3ADIKi29n>>hk%cC7MGKz9jyVPN)}PRAy627&*^W4+FX=~L zWNK1BtcG!dm9ZQd;s#t4(a0m=k}07hLXMwo6M2EU7j#(I>RF67~y*> z$Qe7CFfx_kJL!VlNlFwM*(DZz6BUbVEJPdy7Y2@_C3ZOM+UKN0dSz}aWz zd0mEa>d1VCivH`(d{>*#lBo~NyTNR^=ayD;%KVfQwe>0eG%$@)Tlk8mlCLm>0$#YL zF``jg2l$Fd@D~k zV>s3A(wxb8+b``hc{}oy8F@-pWAhm}Hyycz2f)Y?6>$99zgJPrj?LoPiek%Vv22nM zG-XLxIC^qtF#;hWuWUkbts5?+oe2vA5g!o4x&ze;zOF!M)M&i~74yyeiGwC|uT}ac z>;>);3#j(qKK57e#(oM(ytZXH0MP;rgs#D_djgQl`uJft`!9eZ&>nD?O%nGRgtc5D z@`?(>YvPp`Pj#N}>lqlh1ixJg^Pnsd1Kkl86_dEy9lXIDfrx+`3~FcGM1&V~;A$h1 z4?Bk_x{E#M&!0Pg$sIRc>>L>A9T@EF8@%KOu}B{PYa-Zb!B?l6fIxS46sMxXV0us% z$owBe3Hw_}pa5ZrMS$O~Tm!?dpqD!n^yhmzA*3Io^x1z$B>G}pamfw1@M{uKr9m7D zN5SZM0xTM)P!ytbLjC|342R?Ti=ADAz32KaxeL*@*dnfgRYAE(Fp>_$&x)6wJ3i2J z{squM&h@#;Cke)qqPX_r1*nIf&Oy_7V@ovVCejx936>~v!3vhFy$7a4M?E$ohp{1O zMMOIm<6u~nb=jiKhYBEycA%FoC76T(gifUg4wR{<1mJ^LJvf+EGD8J^-~-b z6Ofj{P7cBh*vTq}N|&Q?Benlq)h>|dPtyUFiOKB&wd z=uPiG{jfRRe_^G6DAPZb_W85@FQ?B1(sfssZNZ$$xv{tEWouKevUYxQZZcH`exs$z z;JvGb{;IMEPG{TBK+(gAUrhdVGCg!9doDPuDh!;wc~8B-EcRwwj;6OCOV@qrk*zaV zRx$6Ib0rThmo-UfiF-YZ&G&WRYkzPk+j=&=voBqDZrRokB5lrIIq#YCBzqwcdPWOV zOINmLjIHmVzJKFqs^z_B35sie=iJU@c)^~vx6Wu+jpef=ce!O_W6r&8QJbken65Z9 zb8^+{N^XZ>N^1+WSuL-f=|LxI=$*l3#}0J*y6WF@z2Qpj&$@QZoQ5Fk*{>{F8&dJ~ zzTV9C-duD0;{K)PLvz>ee`Qu#SdE3l_bZlm9(%jufiGLxU;OA-QjU~5b$+u?FHhDi zl{GC?FUFTVovW2K^YOWOs`)#fw>=9d7td!K4nCMzt~^`pYI33cyRLU#i~H~I&w7rh zTe=?X`ElrBcl!KG*%O!2J(tsU{ztZ#b7kepif`1XPA__wnvUhXd%ipL?$Ba<*?T-w z;dy9=35(|-DbGyaKv(v_nQYs055w7(7t`B^(sjO%Y!^v?uVa7PmYWVrjc>d&v2cO( zd_C(0?@f05>GZZU>AL4WvYq|y7OJ8y-%i<^@_jV;lF8{U&sw*vSerA}=B#!5^r?@H zj+}Mxs@5=b@Xw}ZCzrJDoV7A(NV?K+TA7S>$35SQ_h`m@^v8||FaD@5{k$*J>09<* z%vdj`br*ABW;pkGTOI0eL`7kNnBcd#^u|DR2tS7!d_e0?fiMO?71>nOlCL;MNv{9q zqK*G4Eo3)0kFy|;BwnzZgpyIBkPBho9ZzXr9Fc_jfDZrs9~@KG=6PLVNLgrt_OFx( zno35E<8S({a)^$!HV|^J+APFLx~!q^w-Tx;`u4ZLf}r?Q_}LEr`~g2&W|E&`_v;EG{t)bcLS#y1AXV&a}f5-g`KOFl0(1XeBk>{5OzWivr>BWKYVI zscl`U-Ib}`wdl&$KAUxPKO-1ok@gO*f z&#>%gW6dS!OFq4#g3hSKTt!W)FhJs0(3otcV1ZxIf;&O7!jDM(f)@*JOYo<15OMSw zqLUk*91>zi%|RJ>rtxU7-^H9xiSqXop>I;3;>wR%UpL`A*!Le^`kN` z+@|Y}0{nt6=X8lb-f7D^+uPAy@ z!OD)-aq`LX7;ytJWFyWcutFCre}esyrFtks|0qMH!x!6pK`B}9c6JqdK$G+E1h^8w zU{qoEwGycQuHUEP$|BV?kiTk>Xv zEL3?@-inZovODu-2-&HMt$7DRPAn@&sDi59ny*Bt3NGfn3!!QXLd$9ps-?=Rf2%O) z4Ud(kGEKe?%l1=XWnp~IAIYyMe;Iw)6t{s=-xDWf@m44`RCzMIzA6w@Y* z2WXnsuW1alb*o8JwFpW_IU$qG+3`91Xh+O$!ainy z->t6dmT2r`{@B90b-V8S^?kqZyWhS3fzxT@@Z92mIAA)+aX+UY>SI+r{K9v6j=RQ* zTsJ53qAtu2>biA2`SoFax1PNX-3InHb{p|FgiV9yZZnHx3|j`R-B$QbqB(3Ew0GN? z-x78VI=fxWZ-w98?O}dfxMN<+CEX>1rQM~2W!+_i%et2hmUox4FlV@8 zu(G>yu&TRiu)4dN=X6}qD>jEX(IvXytc)Hh8jUpx&Cj*9cXc)liW_VVfxciQhEgM<(7@uLHbbGHD8!;Hr}I(itUN>o zQQ8f{`Tn336iz8ogBOCmg1<%Bbza_cN;ngWh_Ve;9L&}tFBU?>fj;CNL7RJ#4;9E- zLv4po3I;@fC>9(P5MdzL8;eRKf)tF1C{Ajy9Sur;F*p>Cjtru{f*3k;1_6;?^hp5q zITr{I!yAeS2b5Mf1TP>Td4muMX8W=?9FXOp++h2VW`d7T+XInEG!}@3q7k_rzYRwN zv3|VGhXX@Hp-A6{G-dGHIwVEUg-{Qf0;S=|U_d%c@ncdjh_}5{8HvuJVDE=CJ$=Ts zwYNXmd$tEdnzq2{2}phC(vIMTp{Nw=ITH#8<+MG5qR$0;D3`P|JDN(EPoFkt+mW^$ z3Jgb(o0PV73`aVGFQc8Yv>_5045p226w=l}_My=O3RA@uA@ zj!T#1zV!x%deG}~G!hSKiG&dKBK1F}!1?}YnC20tLfi4Sj+cbkuoO{-P8t7zD2Ay2 zFwI6VZm1~cGct$?GTIo4;cFl!Y~HkC`?ejz=@Epbt@7DWINT$Kq(`pot%|ccEqGmUtdt_=|`2~llwv!sHD><1J#K1VdZ2a2;mUsKk}ik zr!G>4qo-&>Yt7ss1RMGqgqlzF2P48Rnul^IHq7P%<|6ulRYmB>EX7#u*?^fCzyeVg zOSZm^O2&c(<&=6+G22RTVcC1f8%z+fnrF2X_|46*d;TrcfKIC`%~nI3Yl6dH=j zUFk9@Nb|HOh}0uJ+PqHNlo)J^xwRovQl*V@3}n@3@af_eOTrpE+L~Jrw_zmX|7zDN zzhCIx-Xw^@7*!||l4GG>p+6WN0v*684M&JppBfs8^+zL{b2CFJrg0a|xjhX-Bc~dK zW>FM~9%G^g(Zm={%B4kX@f!rce~%5dE5VrdWanG1Vt=VrY%5hP9PGnt6G2mng0iTf z(U`87+=1SH0aW{Z1mPfxAyD`upK5317lbiaC$u529HkbGY?aW2J|Bwp3uh$I?7=_` z{UDs8NlM+1;cYtgG$Y{XiU@+T+=eZF#>Gx3m2XfoJ;|!3wa$~BfLhA}7eP6d#_h;5 zM6BX7!QPQxkRUcs(KD>TXar;>%*cAIALIoyQx*=icOGbMW8pE(P$3+R4s8G@WnXA9 zgoR#4?`dTZ(fVZLa-_WtY%3hxkY#>(D`JDt5)KTDL4ONUvl<98==`=|K<|3RS)M+R)HYGkOyLMo~Q> zFKdh(i;w1@#X}MZpp3%BU~h<)T1`{U>2S37Y|Vxmg`(pF%S$*D42lYDPy&S0f!?z! zZ)g3YjY+vSDTzYMx#bY>X=Agk36yYH4xGlI5HdIeVka+}PD?V$PAQE>vMKda^Wj7J z^Qvjd#3Clt(e}cz108Lx!rDkQKd!JlOPe(gBTCVsp`hq{S~x|kuIUsdkFq%$2s5HQ zI4s8mHZE*BfR`xvqCq%9bBR)|&zso{Q3OQro>)lEAt5%Ol{tSpDvnSoj4MPVSd287 z*`CxhG9Nc*dm?U9=DN?ozDrf0tfrR^PkRiCk6&=0!>@5cuA2we*NHr^y{_93 zBurl|8bl+o<#N&Vnz`Eq%xng>G>a=m3*Hvdig4R?-Q(Dlz5rX>fk-J1@>m>4TqQaX zW)t@m=MItW0Z zph=>$;3NvLCgc)iAh{u}>%GvPCN!r%9g6g3g#nJx8u|4c$1e!*T;mdag6n}#c@jGO zqFC%1;coKnK3&?{1IoZanY0rDfNKUh7(SD>Ecz@XS56RwM~54U&B=|M1Hs0Ip@1BW zHqzQ5;3qe3JU1+zyRdObGC2rnbowQU-&G^xqo$c^)oCANX z8ZTlvFbKK<_Ia@*@@KbW*;K@+W7#9>3dej~C+fefS7pUl`Iu2nkgkJCq_$xayAl`A{Jzs-q;FY5}G5_77V4N$)>c|_walbT9>j~c98f*UAW5*0u5 zYUwQyTA-E*Lx{U&{zkT!crHdHhdl{HY`N+mSfToI?_BSQHjxkU3H`vTT&U{%M%L5H zT`^uU_3A@hukK_e$9)>TG-@0*C5!{LYTVpgw5x5E_&3TE#)Roj-5b>gu2(;58Z~~6 zzvWPC+{x8)u@W_99Y!2&An44VfpXQCdnXK{<(5?)C7yegJI`Ivzr>yAd!ZjuN;uE? zobjswrF}smcmbe+L>3aKiLNlgSOE5AQudjsBm@vJgcyaSV8#R>b%tac);D{=q!eLQ zkYhn0kJG{4z_1)-CA1tm08&I)RnAEugvmfN8q&601>(*uiem6xh_@1tfsC4~ClltW zcm!yq0b+`*H3<~3Nm!H1b4`4`nhQ0FpkkJ|Kw1hyF@TzDVv#TI3W$NBn7{XepTvDX zmcMC83Z4mFxCw#xG-y{BQG`%mBq{}aAe@0%r5*XTjvEP2HGwADp?-ky(Z{3@JCp+u z-57F7n-t;P=amj&4AZvbB)zv$W)cyFv~GV_+8mOZXe_nR7Yh^vLzHpaJa|^5A8C75 znr8B|l{j%$lm_Ywwq58AG6aw|$^FBzP&jRp1}UaQD=e++I4rf3$4D9n86pG8Dl&l+ zpPA7&l!(}@>INBteOlieyMWAS`iN=%Y??nO6Zyyvzd{J@z!*U(6bJ)0#4B4CF|3BX zu!~42uK)$T#AQsJ!*kzWJ#DX^bj;W{%zLZnyelUS_l1qq!p7vvEg6pA`V^d^jSG5x zu`9#rovw!#uB3dz^5&i~%e=ejO8j#Cx__#C*1ZX-CoL24q^~Qv{8-9+Jn1+-UsU>q z{`+P1(`EI^hCQ=o&!mc;9W!Ng2CH`tB|EC89o5%6-ud)fpT6b4Q=VGB_wK&r3!OjdQi9K>j&OFaO2>t_wc(1-)fy` zo_P88%im6XD{=R^d)p6j^S;fea&hI@{!bB>~n5nuoJf`=<^=Ve;y@9)~s%5&c@uV}6Je1DS>?qBjXt+ime z{h_{oFk_DbY-yqe&tT=o7*E!?3sG*O{-B2Z!dKwA#`SSA;$G}Y7%%WL;1+0amd2|g z^cFSB{}t$4uks20Ds-)Q<4-i;&zLZP3z)u4zs>lw;LnOb8~*J0b6m~mJsPh-i1SGy zot)1VpXmZ+!n#v*T>0v@Q$VME=dz+#5CB)j9I|kFI22|uC?pBd^AV^Gv$Ip!580md zGl0Rt2!VG+BnU!#!=OA9XPy?G?>y2D;UWmMat={~LlAd@63T$yjKmBm&(s`Spz;Bx z5s1dgF{od5x3(Q`KX&L41Kg(w$@Byx=T7BPNTuk|w1Gn6X1|}w9&w$&H(t)XSyfbS z7lRo5;&fgNZ`u$KX`5`ad)Jw$D+0;twfyjI& zg-n1tlJ+8P)%2ukBWsToRO8Dg_>_KA$b^;k4h>_xd5>Z{`vCTV6 z$9q2?o#=Y!#kXFZJfCtlX7u>{z~Rjp@dg=j7H7=lvvAI`jFo&g&gsqA$>-plMHwgg zT%3DNa?SQ+{f;|)($SQ0(?<{ItjiRU&&xS?W{SyI!Z{lNoSt#(>+PB4^nC^AEYGZ@?}GY$6@BN5tTqMpsjH0MMC{qRtcM4vj~md| zB|c;y0CJtm*|9eG_#b}E+Q5_xL0>^S;@@(KP^P_DAkjgr07c(RLN}HH6d;3wf`RuR zIot-7GB|QT=nF?rV`&^23Pzd_2*)}O397a-H|N1C^{5g=Kf=su3o%ey7An*4M*(b3 z$fiwK#;8W@LHB3%j50o}fUqQ=}) zVb-2q`1Pdrs8eV_69tqE$jt+^hAYM_psmOPTEi%a{D3AZiUR7)Q7IZnYe$mOdQC>ZF^|rJk=AeldGqSm1Xwpg;M1G>jfJE zWjkv7l@)uvruXYs>@7EGD-NG?Gn6{%NrW!MbPogS%%*hNCnuKO;#DDu=b(+ywMz#O z$ERZ*D!rokiPlIG9GTX$N>d!*TA`sHW!EWQwS@8H2mU8YS_22_1B>InrF_~_KH;5s zX{zhKfA_S1_nlD6zdvayzh^nHcrhYL4r_e|Kc3W7Q2K(+9LN$H%1C|GMV8P(s$JB4 z>2rt!l0%b;wOu-_d|<@ZoRvzHo}l71@~R0ew6*nESE$kq1##q8m1<((9Ci8EIow_)?^r&4m)iE0zqgQUC@09C^rl;DB3u ztL}T(PJ7o*mEYcar#|Iv9y5c6T(MoYO>CK}8?((?8t1Ieap$yk`F)FT+JaSd2VC8E z&(iuhEzubJ?;-PR91+Mu2=N;Tr3d;b{S`x&eM9HP^ildzLqeYeHian6IbP=9sGyMs z3t!GOQif6e*D$I%-5Wqqmf45+uVH;)J#FpB zr6E)?yT+vR@JnALN8^}F7M64lp>gk$P6u}UM|_t%;Lx(tdu(&U+ONCbb(zSUiw_J z=M={|-1sW?8VXJ5Ji?wo1vQ0sjLKvWh*Nf2f&kGF;aMau>D%M)F78<2JhX*a@ED9? z#mWO(ujnv|G^#2=p#3dHemV_m(@zF3FtUJR5F5R0Gff^I8iM|mwp-T11_Rm#Y=$Wt z!M**_P%k!Aa|?n@f6hgRp;N^!4JqpUY!eb2mWuI>56VJIEFhnS4&V&4u;~eja@rgV z4F*v$teIVCR@%`q9HCE&qL8G6v@te31f8HBtXf@JH}GBh*W`SSoIilGWM!qzBt#9z zf{*Q2WWPV8G=$W!!bm-LWzXe3N&BjK?~060SF~n9Zy-IK!RZ2vg5>qeoil7?|h6oWL?K8N%&`6(j>i&4)gtebrcYN50f zsl=nykE~Ea$0*B_5Y-sFA(eqddvQwHOX9o4P%n$GQ~}i-)uNIhANOb(s!e3&nD=)z zbU~a)DMXc|?~s#AO3O(3MzP~1xl*+3PJ&@d6pb~FqNNSgg{T5s$J7r?B1nIRZ$6v! zyIQhb1_V}>ejsd<>k;YON}4ZfX}$zs+^gatx#4J5?~~WPOR`|an)13cml@Stxd4rq@atP5dUN3!xe1AyJACvPQIq#CwOwKdp zY*QT)QRE$1iLY@#GHicfcVB9qD_(i&P{yD$F1suJ6NlG=g*Pr=FqIo^nP+)!)tb5D z(vRFLjkX7#;*1e*knxi8jG27c(kRYY$>-pzR%D#yb8)Vs48@03VJluJA|Ey#4)N{$ zM|!idVXoYlVeh6r%DZhpdq1!)TQDLVYSZWVgZw{nym3FjV6qz<7Akq;^YHS-s^zsC>4K9^vu60zm9UkOgdC!?g^B`&w* z5s^X_hY$y%)7eT4xGN6>X?Y}g3X@=y(TUtd#IEFG#5LJ7K=fzHQ#hA62=OfqImM9% z+|$@D?SL=MqX4|Yr}+q&BP~o|5||#t20E>WRV&s?SI4pTmY#iwyV54?IALcGM4p$W z?<4LabZI0D;34U6@!dnbF=%kkT6 z8Eby^+!u<+UwE@@V&mtl=G>k!%beXY)-e_s+xms3`^9zB#dSC2sf}Mfcbk7Vo-D3Q z6>plhZ%P_AvB6%%N;OpRdDMW75kJD`k5EoHQpZ8*;YD$Xnz2WVBa6CL?&Ki0{0YU9 zu(RaJVUK+)NVQUU&|2Jew>hm$3o>|M+EbnR10?GbRhCt8#BloLIT(bPoblskbm+!9nn>FvR`77V`BK-U{Q!>9=x{{wT zujCiCi>5AX*QWe-VbEX!gp8FZH(9goC|cSMi&{az%4sA@^B3~lBBk){lUq@WSV}AE z_I#)I_6s*&oCpi!3Y zG4DV$6v0UAqo)UQ$^>Z!VT(7FSajy#*bXOu6%j6R56rep7w6nX^X8)Q+AqknW)K_i zm`(W^n>APexTtBYdCpUO&3x57!M|x6Yo2#kOsu-zIqP0E8TfkNyL~r8bIZy<4?_g{ zEZ%6H`1C#Zy7`joi5K2HIL?!5e*4w!U)(uYRsH$>88_#B8cf#X&6P0cF1|8yd1RW6 z>UU~q>-MC|52f6Pl9oercIV^t%!rC(s`3lh;kgE*#=Jq)I7LBDgDw19+8zuBU!wxK z!DpsN8m%MKqfwn`D5M600i~`F=0z`I`+rpTimo&JKF_PBM?Q1>iyd@qfUJy2w1fdO zZ7mbThZ;m!J1`s}Q(Ww$lN~qC%)nAR8y1yt-h__wkVwUj#mF!+6^CKxY9rk%Y{?tQ zbfqCAHbHEcao_?cSzwckd9wi)%@PO-cEf`Ki6)US6vgodq9$iy71JP%qvxc*goCMk z6))&^6-ypF?ZS(ST@O~QfvlCKze9|6a6;;qbk@*@QI#W|iYU%lqzC#-Y;H1p8b$30 z6AjYzr1etZe3}m~*$K>xF8ycZ?59cj0z4#i>y5kSUEc96uMgaJ)l9o;rk=ZVAnB?} zx%Q3eu^;59o@hzgg``3F)k7oaDE=jY*RHvW`u8^9e)&(fO?#S>?x*irp8j7MJ-$DZ zff=}0t9{%}YqMSd$F63R{(X*z_kEKQ?jl;DO)e8Kz>{__pzqb#%qZW-jbdj4Y=tj0 z=~R*rRtby*U8JFq23d{8!V9ocQB(cM-Y0BEYH$YBcpDalHn|4?c9uRlwGp-|HTIP- z#yo0FmBAsmld$J2R=?-oqETgIqo%w(OpoUw6(vjsbI`0HBiPdzH7CrUtxA{^`g5FQ zg?%9D-pv&#khDKsZ>KRD+o zflk|63Zu1al~*e#_P)7dY%dnPr*fj^x;X2uMFeoGE9Wks8$a{%O2u{R=jTF3NrcGuW5U}fXM%Za(wYsO9E!xNoXKWA}Wab9+gpT1|Q$gJkvD>L<+!+YuQuO3!Fxp&`PJMFFoeX*AQ2zArr{Ydtr z&F`-$ZCR)L{<`8;&iMUjc(|Iy8A8F8vP^0pD9Ud{M?e`M`m9CY-po_?6;uz4+FDQK z*x8fzri=;W+g#2{nOVAF;Fdn3wiaQ(TewuSw%*MNlW3v3!gkJp9L=Kj96xIL#CqG* z9C9k^Q7cOgnM>3~45B?@rrM6$P=bT?h9O~zCt5zFo96WZogYc^`nWA^r#Mk`c$C^aPP1NNo8mop)eRy?7e#d*DB&$WN? z*sz+Gu$ngI2`?I~8MXflOE3M6OD|)kcRit>mZ|-Ok%vGA-si~cFGq|_L)eqoUk+)5 zSYF7oU95nBVdbafMpz0;Q}3S4uo_ zObYNvm8TDAd`s;c*z}jFezk0ANf!nB`XuEAVsZXA6g*;m27qQm3hB^>Abq$M5q}(P3zCU;)@b!TG+pL(#SLz5UJhpVDTBULiqC#!p)>ifKbezz^QuJ+JQy|rSGo3%m0n>edFCT-*rp~119KU31`1W| z(eA&zfBg7FN6H~wYI|UIBt5k==Gu8z@p$XC{a5!-yp$?gGgX;#Z2>m4IIlP^JCe)R zOz9_wlcoMyOT!!hXZh9giM6Ss+A-5yNjXb&B30rWv(6PQo7gd3R6A4DGG)C}kzBiX z%%o+UT>ez5WaI6fDc5e~yiAxpFulxw)qDHo-JN&WCpWhvGkeka$r*di&CPS(PmQ%? zOCL{s%k=L4JA0G0&)kvj*4{0>d+u(3vh>KTk)>E zuk5_M^Npu}vvj1HA+ut0$}P;e>nHaDZ=(w0_10PYs&~b?;__=vSDRoO$(F=|5$^@g zP~sYEWv1s;vt*TjN}M{Ltk^P(;~h%F%roX?YA?iZte!G_({j^tyCSvr={sGin!U+Y zEy>c>SxcJ|J?U9LZC<~q2~C%qUf*@!UNdd4nXN^4Y{@n!S=IQS@pkW>ovEkVl8yV4 z70=B&_G9bE>c$d<9P-lTm)SYBil){guY4ZOK)eZr8kb?9S%9*3{PLlUoiZOAp<%9M0%DOR=J>%2F}R zVxDFu{lPVGVIxUb@2)o4n%ZV6tSUWVPC)u6veBRJ^oaski}A)rWKj zAT|5~?O34VxT&Q&Dn>Nee30S9V%;V-GO5`;p)(R4tGbFmxe@t*G^pb^Z*h(V$6uZ(o-j^y z+%Qh|PHj!m0jo{P(#`iQTk-}F?baT`|EwQ=4<7Wx$Lhfg>3?!t6B^R1%wvk^t(>MN z|D*#$pxLQ}e$vM@q=gBjV==SiMWH@JyrY@U;z%&%#L@ih4#5TiH&+ZRXMM1tFqC)m z4^9zDxP1n<=isb7T}}Z5Ae<@Gjs`-(*C5%bzYWAe+OZi=hte4zkeuYBDaM9~bvXu( z#)%CPGb357?z1W5Lp79EA>NnpC(}MJQ$hbu{Il`TCGMM7Oq*BCu56q&KgC8XxorKd zw%fJ0M^YP_lk4{;OIu(i_zR0?ymZ1esh@o5o~7|)W-g7!Zz*|?7ha4xc^ld?ZFWFg zmK^~|f7y|e%Ak4E^mhZA^r+1zE~k@Pl9#57(()>T-VNQf^vUVnB7A~Q+ZL8;G`9nDi(>+m=Mz_}4RW-dNn1OfP^v?n<)fV#0qrq#q|o}j#EtWBoYu6Y1QP;fmq2I~nDVh_Q0`L+mW-Xt2beaBjORQ9LQmTkJ=hNnANQceGvYGKu+FBM-iDTJZw|3b0;D>-Duswic1w3ESYqKFC= zwUZq<+uc>jX&$1+B}YKiWNx3|emz)cT&0MONb>uX z?CWr_?*iNVk}C%<9|U>Tr5v@;-l{m%U3TT-<%<(fr`&5)5ZE$n7oeY)QUwBIWyZa_z}v>1Xa)dgiRIq<8I%b?uz1^vc=GXMxgPwSdDc@hdli zxAx!e{Kny#4bR@`yz5KVA4;x1oGfj>XE}m|X%A_iozOlL!Tz(-eO26FR~7Hur2p%U zJY0#em~;z{Cg&4LBhY|1zt9BIKu1qT4Y}RYJXWR=NtGbCVsW4RD)a$R9~YMP^T@3U z%34Kry-dV8xPtLDIUh_1$E6kfiYLU)Q#9w8X~JAU8SBo2MP{7A!@+FT9u`|Zr=O6& z#&*D@G-jQ!6wKd=9%K)g)6)Duk9J!bdH&oJO41bGqt?8VtOZIsg>5~JVX9@qj>KL_ zahk9e>|>4jJ|Wzxj_wn}U7rx{R>N<3)LvG<@Wc!%(H}+UIC)K8TMN$yP2HaBi5Tsw zuqV%kM~7V7@>u341$O0kSp9UPW+9vrWe>2GE0^(hSs3kw5KySvh~1saS=67x{glWIC!+kh3sS`3nZR(E?oojT zR?AcnqY!}TGkGyY zs7k|=>}pRM1Vz6Jg>fu;Hi%OlxY^-i2*#qgQ7|a(zSuiTD(~GFqoY)=qJ7?dQ5vO7 zU@$g4N~(0R5+4^;<$ZK2=u3^WOv@k(&$0n9X5vF>zBkQBB@-%-%Mk6X`bv`}+-A%% zr#-ZR+GaVyl;(hOWUnv%N9vzi(#QuxPykQPZZm8rWwuY zk4eiY5k^uJrGG>57^GvMjc{GsC=L$}$%+_E`cg%ELEG)pU&HY^rC*Sr!MNMxBkiaH z?H1s}W_v6*O41gTmDU3f)4_dq^R1XRi^}zO?DSI}v{Sd^zeO36^F9*G{{RO-&cl^g zUTT|nRa~!|h`qh;zH7Cz`+ofL@$r2VdsEJp*n+n^$CuC8SIm`eyj}C{`ft_WsYq>V zNp5Vt+ng-kH{;m%V9`dqrudt8luwjQ3{Qq`SI##BSj$XFX0j;)3w-CaND zDZS>n>d5H$(z>74t$TOhT;zGwQ5>CX8-cmG}8?9t8-s($D{Ht(&R*!0e} zx3*2LdHb1^cm0gF^VWqsx^E=l`EKlc@$bYxsQL3(J`g{Ulh1TQf16dg?zqu5RXY_+ z)on@EZcSEfn{{j_3xO7%bJu6~@pLc$TEp5}vJcUPIJ3hXi$p3q+rhoB?rI@Mb5dVIDIR$%L!?)VF zk=fbD^p;FG3&&qadA!Xql%fOV*7ysD$gnUHjriFWyUG=_>aG(rRBmFt9+#^^l_)Vv zo3$;*{bw+Y$Mt86*p6bBtPs2h7&JwK{sH-jOQoIc5TMG0(q1yBQtrT2ONH^$6Uj9r z%}L6P=ra_SM0#N=i%cXUwo17qCbQC(9(GnvrhFL@{|BPTlo-lMyYo`(5;|I#EKZ7F z?|-*{>RhUR+nvUl`U7*`GIm^STgqEA)~qU1#|>o8RET=s zv)5@dE2QPg=!|>oeA$YL^Qp2m<9b}?cx~s^onL%ZaVO zqU|i=I~U)&m|Xu%YUQ)3vSuyGmng~STQjSONY&tIxY$UtVPHwx(2qEzL|c!F+Jk9`zfC_GLu3UphL|hz z&!|X~!Xa5q-2In-h>L~W^1|`|8_0&Uk!7VXr*BL0wB)79#mv=%)i|3wO~&7@#n*Qnp&uptYepUrqbPw()W;x&SNRK5eKjIlnqlOy-2=J3Wu5~)*lX?R;+SO&L$lZ&I@p8CtV6Qzz!KemwubG_kt(~CW%zATV9lVemwN^ zd2&eiAd#RZkrFp;#nni0fJA_p7W(Z~&ir;9Insp_?>PA0)~OsAV~4p25Gh*+e@ni5 zwOe(QqG$Ul|w y2z(0<6LtXrfI32f%FL^k?m+&4&bN(L{Yd~iJ literal 0 HcmV?d00001 diff --git a/harness2/substrate/__pycache__/command.cpython-312.pyc b/harness2/substrate/__pycache__/command.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2c13e0ee5319d4ab5bf8650ea725089d75396f32 GIT binary patch literal 6789 zcmb_hTWlQHc|J2cv-b;cB9{~ytuCRoinxqqyOtu6F)7)WEQ?Z1TCP{6$D^GgIpplc zXJ$omyGm78ZD}AuEF=hN*n!nlKxs%V^i(KMb<;i==*w=_E;nNeHIN3V-!`EOr+P{M z|IE%VNzp|M^icbsf6o2?OTuh2AcPHJD3%V!mO?p{dQTL_&Nk8yz#iIw(!DNW>Uf{#Y z2;+TvOS(1L3cOzl=xyotWP7?J*^%Cr+{Ma*dS|*T*~R#f9!+;AyE(#>=ZO-&NtDQ4 z-d=ICXVDdJDZDTeGiFp>A4p}>X*r|B)CDz_H?uT0ubDG3IRgbbT}x@Ez8F&#&6Ka| zYRpj0SavFwquE(CWyYp7T{RM)xn=!`VTeu{fGmv{O$hn-Bnf@GC zfZsHqNy~H&o0wEpArFk_>6DyO$8&1xa}JxitRO5POHGMQVg)iV@HJJ!4pzvpo7;6b z zla0C9e3s4`IS^&S^3)rr>icI>K)Z!+ZexbYqR1No69vwa!riU&Z*oPhNTfzhk%zCk z1rcI;8a?=%q{yEkck2aCe`sCz5JDC~n`4DRXXy-XGu@t;mjmDOQ?}h5wdhgycZlJPvw9-4eBz)rS-hf_Yi!NKB zfLY*KV3x^d>r;5er3i`$o+q$Bj{?YAB(n|6!_n-dcBFAezu!K(uE< zE@F{FIAP3bx^4{QGkHT*tWfrvN-4ODAt@StE*`dojGR_2pRCWzi-wfVSfK_hrR4>y zqxqSVa}rf$0~-0*usDVlu~{5OqIs0PNww)|l}a;O#@fu723e2vR9kzhozZH~-s;}{)z1EE?>DO5yQ_VB zknTTJ?b?n1cJFzA=$)Zz?CW3jMOyqd67l)zUKk_M~ug zPs8G)CK@SA3cqv9cln?24&1!SnWEFa2~M+Kr>vPPu6u&h%gL_`J~XmGvz5a^X~^N5 zd9eYLML{F?#Jlyia>R@{t%`!#>hQB@ncGrx48^_k{xr@Rcbx1eW`{Gt0Yc$mHxV*e znngF>mWl#%kGk7%Gjd_bFJ*ObfS%G>mz>fcbmS&Lweo%w}i+|~Y6_|n;L!~*2ju#q*nV(%qsW}Ws zEVnU}H#OaI(=>L9i_9%xBKTlVCKwQTSgw@00BuchNyR{e0+e)cGLVMjmTI!D8ET<- zyT}roYM+6Q#@_+CL8^hsTA;Ta=)E2JMPLwsv)Xq4qcdwSPL^MsES-P7)FN-V`1Wv( z@S*S*ZqnBIqnWkN{&Hu3DKYYK=V+z%^pd;Ai@vsMuw^aSTMqWFOsw?{l=}u&XDWS9 zKj{4MYAM+Jaq!%Fd(X-r+&T6)!+$k=|4?QB$p@K_l(jLbJSLTfUaz#vrJ($~^+@k; z3GZuL54Qc_!dn+s`tJ*$1fQw~TWcbeJ~GhK|Is@V9_=DO@7gzd!i7Pcm#P@*sY=Sh zV8oJ!D}M_NVzIi*dHj&ozMQ`|5`+@UD^x>-_O41E#F zaU|Gin>A8|=N+W5$xAL_uWtYWD{SjrK62~G4--qC^~mm(Gj|5=dmeOD_6`4Y=%eY< zD_1Jd{81$`S@KTWa&Z(5eF+A7OfD|~^Hp-uAOzc4@lBMw*FZB40>TgYF$ypb2Eca0 zlnOkaRhF+2!q#IoJ5C8gAgtvi`x;Q_d&~hhS5Lq=#y%jhcW-db8!daID{U*Uu1>5C z43!6l9%z+;vn6lz6YsgLrT|?C`aRq7YruTfmcMGdvjPsWH95yjk%z-#ix}udXbEP3 z+GH}Y0th~UV+cx)CwS`kNtsH~Y8#-`fL6e~><&V4kQb|8fsRHT$Y!B*taZoB-SO3f zYlnx*hleWN!yo!emtJDB8MkGFzV0OSQYM>!w~uhL(-FQC0yN!WcJp}kaZL4-_T||o!Vxe;xVdL zh_e=sV1RKHqQrT4xpMOC9*dNICtk2Ja-kC ziRBO(SRPYLt61PtGuPrlc7L(lQ&|c}Eq2t^1xtJ*pEXs>1yflf#0n6R3)!5SvHTEN z!#Y!NU$b2B-e9@MC(c}adBPGDEoEAMs+xI<>rmrf20Qc*VJG$m@qGgQkEXClL4uOU zh2e$_glF(TmKZis9ZCuvk3kZQlY|=MAAsB-wHDIewLEZZV6Cmc+}6L|xqEr;*4&-_ zx3#}cR638YNBiCnyc4){dNoiv@MIP(57Fe(*|5U@|jd4LO_T@9ArrGU0?u3 zxt$Kw=6l0Vn1Q#!rgvQzi@YMvodIpQi@*oKf_q4YD~i|MO^kqG+KsOP#m12fQHE!X zUV&B6Ng(k0q{7QdO8tPR2=JEMjzTFqW+8m#;^_FZFI|TF^Tn~r7~GwG_#6hO4qg*M z`*6!r7fgy_e%xc9R{+^)7E%sgyDT5PT-egEd~ku3QV<{;l;T7J_5>k54a{S~0uWFYMh0{d$d;HO1R=i+7ZmVv z2-yo?drW3q>#_ScfxrP6>G_}J1u^l`b2S%_S6SY<%GQw^`rC_ z1o(B^c*ZXNaW_4KZEkqJY7w9)hZ(=2$EYKR&YPZH{yU+qR921*P#)`4A zY(`~`A9>DBBqs3gU&#!!gUmJ4DW8?65vE7%5?u(zMtjXh}rtFV;*af3tWw98l@ zlrt<8ZLPUbx?`y&PU%_MjJMz;8hZkgEWd>JWq1jaBr7OMZ{%gYUgMD@C7Y5Y`gI&S zhJ=OxXOY8OjU5Jl2f3G#=tvMg>7OF`GbDJ}GqbP{9(-b;NcO?c_igv4j-7Fi^x<`fVg$EacBlSU#*s0A#DoTP#ogN?~8ag4h=C zV)oUGJuZkCCM+<_XR_X=1j~EM*7z{}F?7W6)ffR%b8#H^o39b>JKV3yzF(2xrzG?# sY5kN$9+K!o((`K)dr0~olKuZihRS5EbcQYMuD1{mo;+5i9m literal 0 HcmV?d00001 diff --git a/harness2/substrate/__pycache__/opencode.cpython-312.pyc b/harness2/substrate/__pycache__/opencode.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..83cda9bebec246a1f905b013e4d353a7f57eff6a GIT binary patch literal 19071 zcmbV!dvqJudFKq??*{>r67?FAqDV-5L=Vf7EX$!p>Sc;lL|PBp4g_I_Bq$J|XFxq* z&{bWZExC4&$k?seM%mIyPeZNMhG}yeWw*Ogx82s~w7UfbI)W$4R&BcHG^hX2rn=j@ z|LpI(g8@K_N_uu|-nqEX?|%2Y-~C>rzjV7D9G<)UkB0wzAIJTae&~-^%kawwd5*iq ziCjM?@}eQmj~Mz5JmrmXW51ETP5mbJHusy^+tP2r+Z4Bs*!pd(tvPNVar8Tow}{rb zbHvr}W_cU(o_;UO+vC0wf4?7jhvYr)uK&ob!Qa1Ltm&^2H}r21H}-EtFVpY=eTk*F zxalLTlAiv}Vo2Qlj;Vi(xLw?Ww;=8i1-xs;X0aCUIskR6uA@!u2?>n7^IF4 zf-oXn0z!vIBEZ#{0)&htuQ0SkQl*+kMYt4E06;I6K5TV)j33NwT&;Smv~Cl4cO zKi1jZ*V~GM$S9UC3Oj^W9S?19DwkvNxYEpejiXA5%CXUu!Ui2mj!0oKCWnV(iQx!M zL(e%HG#pKij3yOHY#Wu4OSKN~Lg$FUiW7vNk;rHfBeWS204rcp-jQxWQ| zbmO<_bR;!|xAhdz0F3{PsGNSwV+q1GH32zDcsqMF%Ih7KqCcYp6f&sx=#Uh>9FE3g zstrjvA`e~(8CB~Ejr>$wS7Z$PCdsPf%vj=#^v0N^q*R*@WyqBFD%X{iG*ZCVi1>wj za4aH=awHa4usdmgS28i!C&?qHI#2bSxj;pAw5%}bnr#ZLq>Ysfj>W`CA}XOx?EdQ< zmp%x31}UbJm!*UNAS8x7DFw>88cPjnWJxCxl#{%g5Rzl5(J>Izm^dhz)=Y{~~LE1#@?ikx5Kp0a528V#-A;u-;P_qzEMzphAoQ1V$t&{!Om2^{! zz=nF0Ff{(`=8itqG8!Knj3q**kRk0a(0JRK&i0N|orLJ*!P=GH498$d zipfIHxo+$~Sk1Lm2`WHeHlqel8{-W?fHvTOj852oU_iJ?{PaL^?u(3#T`VKT16Xlk zDLxIkWy|<2Z5jIxlr93TjRJt8DMdKZ-Fu>=lhGxEJ)TUCVmMZiOvJAXaiVDa1lx;7 z5`vTfcg0&w-2Jl{j+x zqQGzqml4es5!OVyTQyN%)vjGO;bcN}7qd9R*VJI?XEZVz#zhA{soEK6#MD)pC_U4`=<1}G`j7a2<>^^$>Dc|DzfwMr*T4E@F!V8)#Tl2IKR+SwG2p=9#15@x;4 zN($F~jQaxY)Tjp5xUf{u$_q#Jl^SVxMB4x6o9t4S$D-_77#~l&UVHPkH#4uj*?_#eAI8GOp#`z2{@|U0IIN6&r7n*URA;W*2 z6OG?6>h0dJy&{X4Eng^#*9(kTsE^gV;nAs zeWmssL{7cO*A1d|6p`W+o2bP;}Q817OaUZ3f-9O!tKS$iKbqGFgbw3f@6&K zI54Sk8^cb0n|^q~(u%@`ju4X2wM{LsoEyd;duQVZ}5EHx2+8{e#NB)~=>=O|cM99Ov}f-7KO z3D8=oRd_`q8CwCf1&?RzYbH4boJLey)2|oTrPHZKDxKXOr8}3j0-XVs5vb6#dee(? ztfq>Dx7BT&m8aR>gYV9I?hJoV^XIqCuK;b5*u2zgb$xV z6Ub;%1e$|t9+Z<~qpC#)46#wwhJ&CGu0tlsr3IN>)!g7R8_PpOM_)mMomzf73JRS*#A_Y};?N=dHGh_P4J* z@~m4j@ZJsi4I5|bmpH@btxHCuyJ6XA_Bxh0v)!@e;=I)lJ)0Lin-{n2oZmJ7M$WVU z#<52>&rSEFdulLe+kB(_Q>(2okT37^-?C0yXZZIW6K^eX#?6&WMw7d0*=V5wEf~%entUne-1yMhu;6T1+_rDgxnCQ={iS82ky;wj(#W|EECYOQ?F(fFHMh7_@j4kU zULw>-dqi9hBCd@ZE~OW7!?Y8tp5g_8Ye(La;n%uS#NwqhY+uzKKXu05%Za95E@bXb zSAMnZSRZ&z+1}K;BeXSk9$;!L(bojNS=J^LGOGrtz0DX-uT#JcHVq`^7*Nq?;d#Q+_2!>kaKRznhs=}4j`5HVu)xtV(Bh6(tRFBrX2ma zhf%)z$?*EECGziZcMZB(W85Gbhl}@msh4Pahszi`is;mP6H3#^dcd6OT1e}GR9uF+ zI!zmbmZ0&(g6><8=Pr&iX*zL{F`2r$))n+b^b?>3wIXwi!!kxkN?3q+X9ShyewT;dB+Oai4H*#{4(`?nd2G!ukVMT!;2iK7^SP9 zwN?U=iZ2No&!B$8-6H+#oAPz;8>ZjjZ}B|$-U}w~y6N|fS9xrcu|>F;HeQ4nza6vR z;Z=)7>=Xt7ll*0rFigorrK%kgu^1*D>5f4)w0=o$N9hXswjYGXEL|Iw52fpW4hHo` zze@dIMDh>ZKZEUYhDyt)uSK%w&QH`&L~agE4&5A?9GM8u99a~0&ewi_$9H$!zc9Ze zx2q>7oX&3f%A)Pe!fTO#QHV0%eyN@3AthUsqznyEH4Vm-msHbm5*jR(rl1dssTm@V3wEV4a}Q%RK(Z`Kw&Wo&MLuV=+joBsyxM zWl_E=sQ4x8Z%Jq^$R3byVO)xk;Spw-g2A(w*)4UsTCbg?rxuXUnfUC$@qr3aFqjprzu@e!l`)Zhq*dsIyj2nfe}T@@$_8Y-NInW2 z!t}(zZrk3b8wJuQ>xK5S?I&RP?CKO)X|JP;>|RWQCWMun83ZUcnzURpoiXcyq#SUh zDtXYH|9R&+h*-T~Dq@Op@ikpD0Xm%5jP*A)HeC&X)c@Gr$ zb_Mu0M|@j;2^~Yc#+y$dU(SW)mr;t~Vu7bXP1UA2x+yL6k}b_lTPJ@vxz&aTs)XN2A(XHJ~%>vgnWfkl>?fIt)% z-_j-Eu-*-~OWM#Zq}#~QtKqI-78IUtT(tXLY)xAoj+T}dha==veXCs>auq;Vtuf75 zA|J;unH)NdYpWK`axZsL$tg;@DQQ9yvdBGnsr-P-r&Kcpf|Q~fW8yW{S^!uf9!-R; zUk#p>$dq9~=HA1fQVULcgL~rRD(W8kcRcX#ST>v7mcRAym}t-Aw)o2A$%oD@3(hTz z&f2-N^N~CK51S7!G#~yrlx;qoYwmm4+`rJ=pKE?C+xXh-8%RDkb9P7GzhlYFnITwX ztS1i6xnX8+wtjEU@%)V=pIRMxoBO6~(v|o0XJ7qlwm+P8U3_9R*d0$Sg_Z|$ju*;X zLTPZ<=j*r4HY^!=cj$=`^7<1S=dD}_w$GmZ_NyPhI=^|b;mf&Td(P8-F5{@%1k*84d0ipBPP$`W*oCSG#_>Ytdi7Xxf^0u6yX*wBXz{ zBjuc1v!<>8^7&3IfuZyLcD}=EiUMiHhp>!?Uw(k}7AJB25QO>-A`fSx0gl8VtU+ct z&?-gCJ61RptHAF~a41%bHoUE(9S%Md`*Ykj+qqx#pSVtR!Vzba?4nC_-!?qmUR*DF z(8BR_jhz!~L@#D_l9TZ^|8xs+gIIySE^(t6Ksz_weVbH&&uU*^`V}f-e!fc*tcE2= zTd%Gy67!arA5SwJ#0zc`fpiK1Hvgi#t3??b9gW8%u~juBquug0;E{ySOR+>z4WO%@ z_zgrIepx^UQZvMeFml?HG2mBGK)KFoyan!?us*RIK}$$cCiu?=RNIQ;l_Hca3~4!* z3Ww7*Pp!OFuloaxrhwVu;N;iWPMF^F6FD>;Gxtfxt z+LkeAY$QRC+cWmz&3fz7d$_ig*K?u`t8(g?#;WYY^?F_Dow16pyY5n}_mUW}DDqHoWcIX8er;atuz{W1Y zRW?Sm-oErMLS(z^FYD8$x5yOt19&it6%6_SgWi>K{eNTBwI)U#D>0ga(4;*XbBc^+ z>=`EQY}HdbOJXo%it%?Vb(-wu>bcaH^yXVRc|Ej}5>>B_krlXjo4d+gGhX1JK38?8 z-36JKxpIW`E@IH+TZ4r*sfSLGmxp*c`D=?@o_IHORYgcQ3QZIbaxE+;(eaWLhgTcI zb5fvd;#%=u2&E6MHMlPFlSxt_BUMQoATJV`d=;26LTP(MjEtsQlF=5GZ^1QTLCrwy zS|74*c(;_WbPX=taPo56OdSuvS^|BUtR)JJriHpM%wws6mVM|0J1U@tRamC8p!%P3 zyr+GuJ?}JAY%r0OrLbmXz$K$wVAJ+uScK{-$0;rzC#mn4iFwGJa1X$xJ}Q&T%>rYD z98p3W_qS&>e+DN5ht~Tq)Ws!cs zQZO`@ip9w?497KEs%E$qMikWn7_eKUVyJE`VI&fdrQrsY;;Ip*a{0?Ng&D)ZPw2QR zqZ6}~Fi}xHOO+;yOHqw55yBQd!k{1;R^5kTR_?qOl}5=yCzIbwHJmvmUqvS5W;Yuk z1t@I!CIulDsxg|nMgzjjL$a)VgpwDirx6e_!;DNst{PHc!h{LtkHzYy1!vQvijDb-EwiSXYuUyN zj{;k>6%E;jE=ZjIZXR~xfCqMBw+C`(aNUgU{X-MBM_&KU^kjOmX8YXnMelReKd@nH zdv@dbtnUKMDAdD%9u}^m>Vu(&RofP-w$0n`ZM|1@?~QD0`(oAMod3v#Wyw%&uXyCD zxY;w=Gjn`);9kckuJ*^9gx|jKn-~7WtJC%=<5V96j9Z7M56w1y64?HzX5+(}?F*Q7 z_g!WF`HwPlZ)JBJ%GJCybtJ!T^K9TZT~qD(VCAjj)5mXhPj}DlxwmzydolPDm32*b z%?9VKQ(d0~_boZO%^?`beZg#X;{$KwqYa_CZMh9Qr)-ZaH)J>OT&Ucc-E(xI^5~Qy zAE>-_VEVuZ2OkA%ZoNGH^6waD4}Z%#SC!q>u@LH540clasp(U*+W^Tz-Jy@gPlCOV ze6_Ppi@qKCs*N*mE>vxwdv&2|-;^=$3*2%{J0ALk1)ne%yl2Y#ghk(rRP34d%v_sq zoboLC4nS@2RbxGO4}YioPWQcix#o_K6WOm`%ASfsS@1!(fX498Pwbp;)91)C1fH5a zHM4Er^oi@ayvx62M(Gy{amN2yBHDs+lnp*uc&3&%x7eE%-7~ zK|wQ6rQiY~awgd^s>@VlsiQw8Q~f?Nx46MFr$EMbjaOW_SGcuoVekwWWSws!{BRn^ z|7nCCPL~@)i(5di)_uV1l8;caEZqE;1~= zSXYNo2sR*#p%b5qJY&}YwE%Ox62oby3FQHDVjAq`Z)=1<#OS_=C{*}|5Ej55iWZ0F=(MK0vfm=G=unp?FOgABm% zXeRjsYX9q${01erD4~s4O@ttYL|7eOU8S}GVipp%ma3rF{S+OQKScuTu;B7J9=R*0 zqF+nS^!@hv-#kB?&bfCj8BzS0$Q5rm&fI||E9GpQJGf*=4hO;Qo3g*#z2u~F7w4{8 za#PO3xhs~ul=D%UpK=vc7NA^^b5}1_Qm%@|tVV9a^jQt(^5e|NsQunu!qQ< zlw@{X9wUS3TabRqi&hGO5RDlF1a}DiqA_NKiPHq(dL0SYs4VaPG%r3-8_cm{YY4&?8m%;INo>y)&7Jz#Xl zdr$Oqv!hoadVykhT4agB0u&tZ2yB+g0R$&#(SJnom;~{BlGLj5+f|b&C9Z?n(T=5! zZQHl271%>bPDBuWsD~q|fi-4Tw-pOJ&jXkfrx0w~-WBe7rSJ4Bec`^Impi+|r`pel z%Zio_Lkq5r`MS_V=TzU+(V643+L#usxUeR!lfF z9i8qF{UJ@69grzxp5+p};Vh9g?h!1D9u|*O$ZI89S|avLxqe0`mGQaCs` zl!bc=1a0h2-+oG}60Rb&5b;?M1re16TY4e#KtK>nxv@vPS&HT!80sOw;|^f4IL!9L zVu6Gv#L(c90o{%vt{QIDp(Mgt5IdTpus3mqPor?0s}=X(n2N+0`C2>(YPevMINn0yks)$K*H9- zd*wf*qVFML{y9c^1-jEhC8V|Ju28js=xm0n!(|`PSrHk6cx=w(m9Ev;JZ0U)6s+_?OL#u5;N77a>N1g_Eqq2~C)uf3~Dqt*_t1XcF9^um3kaKOCTYoQ>?fYs;DAZ0^`Kh4w0XzIMX%yvH`9*RX z4|fG{!bFsiL57{9AXN+@S8lC??k5ezV}nDf^oFNbQmkyFHITeRaT5Nit+$${n`Taa zVryhDYlcgcETKW#+od0*jL?lRsWNE>K>)Ip8j}+;u?~0x-o=aUg#2sBec>;JyT6M4 zK>SroeqH`EYC#%Qs6zfa2$PWnakJsD>I#S77>mRUHP&!gOh&_D&7n0wgT^TN8YRDt1n!?z*P}d7 zb@wRw6H5M+l37Z~9K<+{Oh#^&5Gs}3pq_WQPfg*+PVbG5C6mEieP8}3IO(gGZM<1n zwlu-za*^k%Hsv?hn4PB|Cl+?4Zh6&shl zl=D%UpK=vc7NA^^^Z1u&G>DXrz;ZR^`gpE(XFjy!Gq>LycwAAxWX8KN3*{W#hRsV( z%DK38jZ4%DvpCi*dnvcgX?En-ZCEnWds~b4-q)_ZyL#CBu|2SCM*Y%8?i}CGe`YkJ z?S?H&?A-?WnSQ@?SbLxDW$(v!*RmPqPdvQ2Ny8_ha;yQ#-`)7#rX6o4#_2m8SrPv3 z#s|1pNPB^71a``j|64VNOu&YrbG9?Gu!vQg5|TJ9FF_cVHg!J9QFi^^jH zl)&+@6R(1(rg^P@1qC4WzB8|M9|<2lg%2q--3!d0S6K6MXRT_c4`fE=MfBPW0#&|> z3=AwD??i0UaJzTb|Mjy70JPaDDr{=oOkK{lsTdIWL15}D?^n(2{Mv@R*EeCyJ6#iJ zCL$BheQ@AmVCzC)>+dLYJHK^hp8qhN4Q$N?b}cw}Wlg)Z4JB)i{GX5%J=27D($w(g zm(Rm7qG=nV@wQ>yRJL11F^D+?in~?X#kGP{$dz~3Qrtt!AU6&Le68KkjolsGl=1c6 zf}x(Sj#ZD|jv4Zaqpf;M4gfd~w!wAra~&TUyDrF$J5rVUP#H(6O3&-CW3*El$C|E+ zqS1ZaSvH3A=`rL{82Z<8cDQsXU_+zLP9lKhDSMr{pkz zTh&v;jk_|o;i5C8)LQhWw&=eyF43c7Wo4Mpd;hC!{cGdOr>|iR?D(I7-(sHxpK*-4 zGwwC9^Q_)}eMTMTaZkpRF!(0gDv{J`_t^9A%)s))s%3HngOl+(x8w)b$GiF?0*6P`GUbC{5SUv75!|>X>u|e;%CSCo4KANSDcQbi5RED_};Ee@P+~Jz0a7Xv22#$&l%^ei4&XFbYT_) z5r5vPAFs+(Dg5RT*k_F#TYeQ;6(el^(|QFwD(nnC~7WHI8!M$gFV9) zdam@wSSFskK)C(~y-#RI`olAt%t*=}c{%Zs?M2;4Es%}+qHwaer@Kr0>YS7?`s$oT zHIg-4f$Ryh$soRw$A`!TV5ujD+wccK4~azT+plUp~=&3^JUF;-6V4G@!(E@+cO| zBtK}lZ&3ailJv%=0mQ@sM!0cF6NhxHE9(aYl@E2R{F|zoc|l>}i=-l|bp#$0nB`;5 z5ChY;S3A#~={fVdpqeN^4bxtSDKw9vRV({Y6+RaJ)wJPt0iUs$$k7Tr4t@HJ#j19e zL%4bz8X^i?=p$H7G?#x$Ad-NidZ_9|u}16Pjo425i~(Lj7B!t3Q&?CL+@J}>bTWg! zqFOWyEGg2;SUhzUc(DTK5#&`DFh2$lZ$yd4Vrl!U?MM1xYsEtNrgZStp2NMJXU?`$ zkx*D++EZTox*&txxpY0bvaxi0#6#bfYW5OPKp7%KauzK^m6`&|bkyUtFl$dQGgQH6 zi|D8neS4{Bp@F}Q>VheiK9p8eFHu>CMDF)Oml$XW%ZGDE{*N@65uba@bUVxEC?Ppr z)1o580V;tACtU4gMy zmQ~}3GAREIMnrfbZMPyl(r}aOH1B@o zshHaH?(joT-GZlX?%2H(2no!2j!qb%vaYK^EWF2mGclRKCrA^9yxV*8{N(v}`;mpG zcYJburgza>n|B21Xz3^fBL*(Use){>D{kcHH z=SCF#eW05Bt~1g1Uz*)Bdn6lZTC&0$g!Q>AZgx#}&FuS~&bj(I$4 zV)jS1JLV35=jffI^Lsz){^O3t+V%h4bzg?Z+ZViLo3_nV%~pK7>cgrz z%eOY?d|#R{=bgT(69|6HS5z^>)ZSc0-Ghq#v*NdhJ{+36p4+y6;>aWK`kDIM%8$JD zbH?x3@7RCenfF!Ra!tEt-k4oK_uNA5&aAL&zV3Uk+}nNMo_p?OcF)V%OgVObI#Af@C-aquY5x*Z}(fg+N%^b}I>OVK50DDupp4{2l zO)a@#>x1B)`MQUJ{R@Hp_qOK(ou6B&4cziI@MydJPdwcAJ@XykKmOh0_s-?^bpBat zG1N8HLH%dWEe0F1XUNONIhL!~`k>;}oasBZ zJGS}iT+@F1y4Ux^^FKI$e<1hbsfi=`z`9%er}xjC&jq$U2=vZ1d?$1#G=Ds|{YCt` zAN-4Tf3oi5rd)e3oC(KH-v6Z!_GlY;clW(lJ_#JjzV=mAxcqGM_Rsds9-XVb+d04W zo-Nnhp51ZyerNppzGsUZqNDb?hDyX z{Xg=(n)g>e@Ym}5@YY>vzT;jf*V>V7>CA3A`Xk@5rB`^)yJhJtPqtgZj0B}GtSqJaLeR8k2|T?{a{8iBR_w^|h&<-bB7vLOzP!FUTBqD-a-#?9q@lzWSk3^k(9 ze1_t&OKb+!0uD+SH93t`QxwrhvXz?tdrDyD;A9em@XwW?rj;Cg%r2~2VYW=szgOd= zCN64XB4?XsJtd)sIYL+@k^DdC=XFY&s3ETCkx^X#@(wC`juLWlve0xbA}CBh*~P(b z6vl>UsjO(dRBZ<}Yv~~w|0)uP|2gG6TaeVzxIp>c#`;R&Ik2&v8 zxOG3_YW_DT{Dj-|6R!SmxTe41+7`IBzYT28+XG7mv)2MUVqkrK{kgoSDjyVJ1hsF# z*{^EI`wzk@nD2nEBnXj`s2A N`2WMb$OqYa{~x-H$j|@) literal 0 HcmV?d00001 diff --git a/harness2/substrate/_native.py b/harness2/substrate/_native.py new file mode 100644 index 00000000..19966640 --- /dev/null +++ b/harness2/substrate/_native.py @@ -0,0 +1,109 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared CLI resolution and bounded native-file operations for executors.""" + +from __future__ import annotations + +import shlex +import shutil +import subprocess +from pathlib import Path + +from ..core import tree as HT +from ..settings import ConfigError + + +def cli_argv(binary: str, default: str) -> list[str]: + argv = shlex.split(binary or default) + resolved = shutil.which(argv[0]) if argv else None + if resolved is None: + raise ConfigError(f"executor binary not found: {binary or default}") + return [resolved, *argv[1:]] + + +def check_cli(binary: str, default: str) -> None: + argv = cli_argv(binary, default) + try: + subprocess.run([*argv, "--version"], check=True, capture_output=True, timeout=10) + except (OSError, subprocess.SubprocessError) as exc: + raise ConfigError(f"executor version check failed: {binary or default}") from exc + + +def checked_path(root: Path, rel: str) -> Path: + rel = HT.normalize_rel(rel) + if not rel or rel == ".": + raise ValueError("managed file path must not be empty") + path = root + for part in Path(rel).parts: + path = path / part + if path.is_symlink(): + raise ConfigError(f"managed path is a symlink: {rel}") + return path + + +def surface_files(surface: HT.SurfaceSpec, tree: HT.Tree) -> HT.Tree: + stubs = {rel for rel, _ in surface.stubs} + out = {} + for rel, content in tree.items(): + rel = HT.normalize_rel(rel) + if rel in stubs: + continue + if not HT.is_allowed(surface, rel): + raise ValueError(f"file is outside the {surface.name} surface: {rel}") + out[rel] = content + return out + + +def native_tree( + surface: HT.SurfaceSpec, + root: Path, + entries: tuple[str, ...], + rejected: list[str] | None = None, +) -> HT.Tree: + out = {} + stubs = {rel for rel, _ in surface.stubs} + for entry in entries: + path = root if entry == "." else checked_path(root, entry) + candidates = sorted(path.rglob("*")) if path.is_dir() else [path] + for candidate in candidates: + if not candidate.is_file() or "__pycache__" in candidate.parts: + continue + rel = candidate.relative_to(root).as_posix() + if rel in stubs: + continue + if HT.is_allowed(surface, rel): + out[rel] = checked_path(root, rel).read_text(encoding="utf-8", errors="replace") + elif rejected is not None: + rejected.append(rel) + return out + + +def export_files(root: Path, files: HT.Tree, base: HT.Tree) -> dict[str, list[str]]: + paths = {rel: checked_path(root, rel) for rel in files.keys() | base.keys()} + written = [] + deleted = [] + for rel, content in sorted(files.items()): + path = paths[rel] + if path.is_file() and path.read_text(encoding="utf-8", errors="replace") == content: + continue + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + written.append(rel) + for rel in sorted(base.keys() - files.keys()): + path = paths[rel] + if path.is_file(): + path.unlink() + deleted.append(rel) + return {"written": written, "deleted": deleted} diff --git a/harness2/substrate/claude.py b/harness2/substrate/claude.py new file mode 100644 index 00000000..c23b7f43 --- /dev/null +++ b/harness2/substrate/claude.py @@ -0,0 +1,326 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Claude Code's editable surface, native project files and stream-json renderer.""" + +from __future__ import annotations + +import json +import re +import shutil +import tempfile +from pathlib import Path +from typing import Mapping + +from .. import providers +from ..core import runmark as RM +from ..core import tree as HT +from ..core.tree import SurfaceSpec +from ..settings import ConfigError +from ._native import check_cli, checked_path, cli_argv, export_files, native_tree, surface_files +from .base import Install +from .codex import _TRUNC_FMT, _clip, _text_of + +CLAUDE_SURFACE = SurfaceSpec( + name="claude", + always_on=("CLAUDE.md",), + always_on_max=40_000, + always_on_cap_reason="CLAUDE.md is loaded whole into every session, and Claude Code " + "warns above 40,000 characters", + skill_dir=".claude/skills", + skill_name_rule="overrides_dir", + trigger_hint="", + autotrigger_file="CLAUDE.md", + trigger_line="", + subagent_requires_mode=False, + plugin_dir=None, + hooks_file=".claude/hooks.json", + hook_events=frozenset( + { + "PreToolUse", + "PostToolUse", + "Notification", + "UserPromptSubmit", + "Stop", + "SubagentStop", + "PreCompact", + "SessionStart", + "SessionEnd", + } + ), + compile_skill_scripts=True, + reject_empty_always_on=False, + script_dir=".claude/scripts", + skills_need_trigger=False, + stubs=(), +) + +NATIVE_PATHS = ( + "CLAUDE.md", + ".claude/skills", + ".claude/agents", + ".claude/scripts", + ".claude/settings.json", +) +_SETTINGS = ".claude/settings.json" + + +def _settings(root: Path) -> dict: + path = checked_path(root, _SETTINGS) + if not path.exists(): + return {} + try: + doc = json.loads(path.read_text(encoding="utf-8")) + except (ValueError, UnicodeError) as exc: + raise ConfigError(f"{path} is not valid JSON: {exc}") from exc + if not isinstance(doc, dict): + raise ConfigError(f"{path} must contain a JSON object") + return doc + + +def _agent_body(name: str, body: str) -> str: + frontmatter = re.match(r"\A---\r?\n(.*?)^---(?:\r?\n|$)", body, re.S | re.M) + if frontmatter: + name_line = re.search(r"^name:[ \t]*(.*)$", frontmatter.group(1), re.M) + if name_line and name_line.group(1).strip(): + return body + if name_line: + start = frontmatter.start(1) + name_line.start() + end = frontmatter.start(1) + name_line.end() + return body[:start] + f"name: {name}" + body[end:] + return body[: frontmatter.start(1)] + f"name: {name}\n" + body[frontmatter.start(1) :] + return f"---\nname: {name}\n---\n{body}" + + +def _native_files(tree: HT.Tree) -> tuple[HT.Tree, dict | None]: + files = {} + hooks = None + for rel, content in surface_files(CLAUDE_SURFACE, tree).items(): + if rel == CLAUDE_SURFACE.hooks_file: + try: + doc = json.loads(content) + except ValueError as exc: + raise ConfigError(f"{rel} is not valid JSON: {exc}") from exc + if not isinstance(doc, dict) or not isinstance(doc.get("hooks"), dict): + raise ConfigError(f'{rel} needs a top-level "hooks" object') + hooks = doc["hooks"] + elif rel.startswith("sub_agents/"): + name = rel.split("/")[1] + files[f".claude/agents/{name}.md"] = _agent_body(name, content) + else: + files[rel] = content + return files, hooks + + +class ClaudeSubstrate: + """Claude Code with isolated per-run global configuration.""" + + name = "claude" + NATIVE_PATHS = NATIVE_PATHS + + def __init__(self, bin: str = ""): + self.bin = bin + + def surface(self) -> SurfaceSpec: + return CLAUDE_SURFACE + + def materialise(self, tree_dir: Path, spec, ctx: Mapping[str, str]) -> Install: + if spec.install_mode != "claude_root": + raise ValueError( + f"ClaudeSubstrate needs install_mode 'claude_root', got {spec.install_mode!r}" + ) + staged = Path(tempfile.mkdtemp(prefix="harness2-claude-")) + try: + self.export_tree(HT.read_tree(CLAUDE_SURFACE, tree_dir), staged, base={}) + return Install(staged, RM.fingerprint(tree_dir), {"install_mode": spec.install_mode}) + except BaseException: + shutil.rmtree(staged, ignore_errors=True) + raise + + def import_tree(self, root: Path, *, rejected: list[str] | None = None) -> HT.Tree: + tree = native_tree( + CLAUDE_SURFACE, root, ("CLAUDE.md", ".claude/skills", ".claude/scripts"), rejected + ) + agents = checked_path(root, ".claude/agents") + for path in sorted(agents.rglob("*")): + if not path.is_file(): + continue + rel = path.relative_to(root).as_posix() + canonical = f"sub_agents/{path.stem}/AGENT.md" + if ( + path.parent == agents + and path.suffix == ".md" + and HT.is_allowed(CLAUDE_SURFACE, canonical) + ): + tree[canonical] = checked_path(root, rel).read_text( + encoding="utf-8", errors="replace" + ) + elif rejected is not None: + rejected.append(rel) + settings = _settings(root) + if "hooks" in settings: + if not isinstance(settings["hooks"], dict): + raise ConfigError(f'{_SETTINGS} needs a "hooks" object') + tree[CLAUDE_SURFACE.hooks_file] = ( + json.dumps({"hooks": settings["hooks"]}, indent=2) + "\n" + ) + return tree + + def export_tree(self, tree: HT.Tree, root: Path, *, base: HT.Tree) -> dict[str, list[str]]: + files, hooks = _native_files(tree) + old, old_hooks = _native_files(base) + if hooks is not None or old_hooks is not None: + settings = _settings(root) + merged = dict(settings) + if hooks is None: + merged.pop("hooks", None) + else: + merged["hooks"] = hooks + if merged != settings: + files[_SETTINGS] = json.dumps(merged, indent=2) + "\n" + return export_files(root, files, old) + + def strip_native(self, workspace: Path) -> None: + self.export_tree({}, workspace, base=self.import_tree(workspace)) + + def install_into(self, install: Install, workspace: Path) -> None: + self.export_tree(self.import_tree(install.staging_dir), workspace, base={}) + + def argv(self, prompt_file: Path, workspace: Path, model: str, effort: str) -> list[str]: + return [ + *cli_argv(self.bin, "claude"), + "-p", + "--output-format", + "stream-json", + "--verbose", + "--dangerously-skip-permissions", + "--setting-sources", + "project,local", + "--model", + model, + "--effort", + effort, + ] + + def env( + self, base_env: Mapping[str, str], provider: providers.Provider, config_dir: Path + ) -> dict[str, str]: + config_dir.mkdir(parents=True, exist_ok=True) + return { + **{k: v for k, v in base_env.items() if k not in providers.STRIP_ENV}, + **providers.claude_env(provider), + "CLAUDE_CONFIG_DIR": str(config_dir), + } + + def read_trajectory( + self, stream_path: Path, *, obs_trunc: int, max_chars: int, args_trunc: int = 2000 + ) -> str: + return render_events( + stream_path, obs_trunc=obs_trunc, max_chars=max_chars, args_trunc=args_trunc + ) + + def final_message(self, stream_path: Path) -> str: + return final_message(stream_path) + + def cost_usd(self, stream_path: Path) -> float: + return cost_usd(stream_path) + + def preflight(self) -> None: + check_cli(self.bin, "claude") + + +def parse_events(path: Path) -> list[dict]: + try: + raw = path.read_text(encoding="utf-8", errors="replace").replace("\x00", "") + except OSError as exc: + return [{"type": "_read_error", "message": str(exc)}] + events = [] + for line in raw.splitlines(): + try: + event = json.loads(line) + except ValueError: + continue + if isinstance(event, dict): + events.append(event) + return events + + +def render_events(path: Path, *, obs_trunc: int, max_chars: int, args_trunc: int = 2000) -> str: + lines = [] + for event in parse_events(path): + kind = event.get("type") + if kind == "_read_error": + return f"(failed to read trajectory: {event.get('message')})" + if kind == "system": + if event.get("subtype") == "init": + lines.append(f"[HARNESS] init | model={event.get('model')} cwd={event.get('cwd')}") + continue + if kind == "result": + if event.get("is_error"): + text = event.get("result") or _text_of(event.get("errors")) or event.get("subtype") + lines.append(f"[ERROR] {_clip(text, obs_trunc)}") + elif event.get("result"): + lines.append(f"[ASSISTANT] {_clip(event['result'], obs_trunc)}") + continue + message = event.get("message") + if kind not in ("assistant", "user") or not isinstance(message, dict): + continue + content = message.get("content", []) + if isinstance(content, str): + content = [{"type": "text", "text": content}] + if not isinstance(content, list): + continue + for block in content: + if not isinstance(block, dict): + continue + block_type = block.get("type") + if kind == "assistant" and block_type in ("text", "thinking"): + tag = "ASSISTANT" if block_type == "text" else "REASONING" + text = block.get("text" if block_type == "text" else "thinking") + lines.append(f"[{tag}] {_clip(text, obs_trunc)}") + elif kind == "assistant" and block_type == "tool_use": + args = json.dumps(block.get("input", {}), ensure_ascii=False) + lines.append(f"[ACTION] {block.get('name', 'tool')}: {_clip(args, args_trunc)}") + elif kind == "user" and block_type == "tool_result": + tag = "OBSERVATION ERROR" if block.get("is_error") else "OBSERVATION" + lines.append(f"[{tag}] {_clip(_text_of(block.get('content')), obs_trunc)}") + elif kind == "user" and block_type == "text": + lines.append(f"[USER] {_clip(block.get('text'), obs_trunc)}") + rendered = "\n".join(lines).strip() + if not rendered: + return "(no parseable events in trajectory)" + if len(rendered) > max_chars: + return rendered[:max_chars] + "\n" + _TRUNC_FMT.format(n=len(rendered) - max_chars) + return rendered + + +def final_message(path: Path) -> str: + last = "" + result = None + for event in parse_events(path): + if event.get("type") == "result" and isinstance(event.get("result"), str): + result = event["result"] + elif event.get("type") == "assistant" and isinstance(event.get("message"), dict): + text = _text_of(event["message"].get("content")) + if text.strip(): + last = text + return result if result is not None else last + + +def cost_usd(path: Path) -> float: + for event in reversed(parse_events(path)): + if event.get("type") == "result": + value = event.get("total_cost_usd") + return float(value) if isinstance(value, (int, float)) else 0.0 + return 0.0 diff --git a/harness2/substrate/codex.py b/harness2/substrate/codex.py index a434b435..c555b230 100644 --- a/harness2/substrate/codex.py +++ b/harness2/substrate/codex.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""CodexSubstrate: codex-cli (config.CODEX_VERSION) as a solver substrate. +"""CodexSubstrate: codex-cli as a solver substrate. Carries the c1..c8 surface (CODEX_SURFACE), the one install mode (codex_root: AGENTS.md, -.agents, scripts and .codex copied to the workspace root, where `codex exec -C ` finds +.agents and .codex copied to the workspace root, where `codex exec -C ` finds them), the `codex exec` flag contract, and the codex thread-item trajectory renderer. Per-deployment differences are values in Install.extras, never subclasses. """ @@ -23,17 +23,16 @@ from __future__ import annotations import json -import os import shutil -import socket import tempfile from pathlib import Path from typing import Mapping -from .. import config +from .. import providers from ..core import runmark as RM from ..core import tree as HT from ..core.tree import SurfaceSpec +from ._native import check_cli, cli_argv, export_files, native_tree, surface_files from .base import Install, Launcher, RunRequest CODEX_SURFACE = SurfaceSpec( @@ -72,11 +71,12 @@ ), compile_skill_scripts=True, reject_empty_always_on=True, + script_dir=".agents/scripts", stubs=( ( - "scripts/README.md", - "\n", + ".agents/scripts/README.md", + "\n", ), ( "sub_agents/README.md", @@ -103,16 +103,19 @@ ) -CODEX_ROOT_ENTRIES = ("AGENTS.md", ".agents", "scripts", ".codex") +CODEX_ROOT_ENTRIES = ("AGENTS.md", ".agents", ".codex") +NATIVE_PATHS = ("AGENTS.md", ".agents/skills", ".agents/scripts", ".codex/hooks.json") class CodexSubstrate: """codex-cli behind the one Substrate interface.""" name = "cx" + NATIVE_PATHS = NATIVE_PATHS - def __init__(self, launcher: Launcher | None = None): + def __init__(self, launcher: Launcher | None = None, *, bin: str = ""): self._launcher = launcher + self.bin = bin def surface(self) -> SurfaceSpec: return CODEX_SURFACE @@ -129,14 +132,11 @@ def materialise(self, tree_dir: Path, spec, ctx: Mapping[str, str]) -> Install: if not HT.is_tree(CODEX_SURFACE, tree_dir): raise ValueError(f"{tree_dir} is not a codex harness tree (no AGENTS.md)") staged = Path(tempfile.mkdtemp(prefix="adapt-cx-root-")) - for entry in CODEX_ROOT_ENTRIES: - src = tree_dir / entry - if not src.exists(): - continue - if src.is_dir(): - shutil.copytree(src, staged / entry) - else: - shutil.copy2(src, staged / entry) + try: + self.export_tree(HT.read_tree(CODEX_SURFACE, tree_dir), staged, base={}) + except BaseException: + shutil.rmtree(staged, ignore_errors=True) + raise return Install( staging_dir=staged, fingerprint=RM.fingerprint(tree_dir), @@ -147,7 +147,13 @@ def materialise(self, tree_dir: Path, spec, ctx: Mapping[str, str]) -> Install: ) def exec_argv( - self, prompt: str, workspace: Path, model: str, *, reasoning_effort: str = "high" + self, + prompt: str, + workspace: Path, + model: str, + *, + reasoning_effort: str = "high", + bin: str = "", ) -> list[str]: """The reference `codex exec` argv; the real launchers build their own. @@ -155,7 +161,7 @@ def exec_argv( Callers use stdin=DEVNULL and `exec_env`. """ return [ - str(Path(config.CODEX_DIST) / "codex"), + *cli_argv(bin or self.bin, "codex"), "exec", prompt, "--model", @@ -173,16 +179,67 @@ def exec_argv( str(workspace), ] - def exec_env(self, base_env: Mapping[str, str], *, codex_home: Path) -> dict[str, str]: + def exec_env( + self, base_env: Mapping[str, str], *, codex_home: Path, bin: str = "" + ) -> dict[str, str]: """The reference `codex exec` environment; the real launchers build their own. The custom provider comes from CODEX_HOME, not a global OpenAI URL override. """ - env = dict(base_env) - env.pop("OPENAI_BASE_URL", None) - env["CODEX_HOME"] = str(codex_home) - env["PATH"] = str(Path(config.CODEX_DIST) / "codex-path") + os.pathsep + env.get("PATH", "") - return env + return self.env(base_env, codex_home) + + def import_tree(self, root: Path, *, rejected: list[str] | None = None) -> HT.Tree: + return native_tree(CODEX_SURFACE, root, NATIVE_PATHS, rejected) + + def export_tree(self, tree: HT.Tree, root: Path, *, base: HT.Tree) -> dict[str, list[str]]: + files = { + k: v + for k, v in surface_files(CODEX_SURFACE, tree).items() + if not k.startswith("sub_agents/") + } + old = { + k: v + for k, v in surface_files(CODEX_SURFACE, base).items() + if not k.startswith("sub_agents/") + } + return export_files(root, files, old) + + def strip_native(self, workspace: Path) -> None: + self.export_tree({}, workspace, base=self.import_tree(workspace)) + + def install_into(self, install: Install, workspace: Path) -> None: + self.export_tree(self.import_tree(install.staging_dir), workspace, base={}) + + def argv(self, prompt_file: Path, workspace: Path, model: str, effort: str) -> list[str]: + return [ + *cli_argv(self.bin, "codex"), + "exec", + "--json", + "--skip-git-repo-check", + "--dangerously-bypass-approvals-and-sandbox", + "--dangerously-bypass-hook-trust", + "--ephemeral", + "-C", + str(workspace), + "-", + ] + + def write_home(self, home: Path, provider: providers.Provider, model: str, effort: str) -> None: + content = providers.codex_config(provider, model, effort) + home.mkdir(parents=True, exist_ok=True) + (home / "config.toml").write_text(content, encoding="utf-8") + + def env(self, base_env: Mapping[str, str], home: Path) -> dict[str, str]: + return { + **{k: v for k, v in base_env.items() if k not in providers.STRIP_ENV}, + "CODEX_HOME": str(home), + } + + def final_message(self, stream_path: Path) -> str: + return final_message(stream_path) + + def cost_usd(self, stream_path: Path) -> float: + return 0.0 def execute( self, task_ref: str, install: Install, run_ids: list[str], timeout: int @@ -204,28 +261,8 @@ def read_trajectory( Path(stream_path), obs_trunc=obs_trunc, max_chars=max_chars, args_trunc=args_trunc ) - def preflight(self, *, require_dist: bool = False, require_bridge: bool = False) -> None: - """Check the CLI and Responses endpoint, including container configuration.""" - from urllib.parse import urlparse - - config.require("HARNESS2_CODEX_BASE_URL", needed_by="Codex Responses endpoint") - config.require("HARNESS2_CODEX_API_KEY", needed_by="Codex endpoint authentication") - url = urlparse(config.CODEX_BASE_URL) - if url.scheme not in ("http", "https") or not url.hostname: - raise config.ConfigError("HARNESS2_CODEX_BASE_URL must be an HTTP(S) URL") - binary = Path(config.CODEX_DIST) / "codex" - if not binary.is_file(): - raise RuntimeError( - "Codex is not installed; run scripts/setup_benchmark.py with --substrate cx" - ) - if require_bridge and not config.CODEX_CONTAINER_URL: - raise RuntimeError( - "Set HARNESS2_CODEX_CONTAINER_URL to a container-accessible Responses endpoint" - ) - with socket.create_connection( - (url.hostname, url.port or (443 if url.scheme == "https" else 80)), timeout=5 - ): - pass + def preflight(self) -> None: + check_cli(self.bin, "codex") _TRUNC_FMT = "…(+{n} chars — render truncation)" diff --git a/harness2/substrate/command.py b/harness2/substrate/command.py new file mode 100644 index 00000000..80fe7e25 --- /dev/null +++ b/harness2/substrate/command.py @@ -0,0 +1,128 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A shell-command executor with an explicitly editable set of project files.""" + +from __future__ import annotations + +import shlex +import shutil +import subprocess +import tempfile +from pathlib import Path +from typing import Mapping + +from ..core import runmark as RM +from ..core import tree as HT +from ..core.tree import SurfaceSpec +from ._native import export_files, native_tree, surface_files +from .base import Install +from .codex import _clip + + +class CommandSubstrate: + """Run a command template inside the task workspace.""" + + name = "command" + + def __init__(self, template: str, editable: tuple[str, ...]): + if not editable: + raise ValueError("command executor needs non-empty editable globs") + self.template = template + self.editable = editable + self.NATIVE_PATHS = tuple(f":(glob){g}" for g in editable) + + def surface(self) -> SurfaceSpec: + return SurfaceSpec( + name="command", + always_on=(), + always_on_max=200_000, + always_on_cap_reason="", + skill_dir=".skills-unused", + skill_name_rule="overrides_dir", + trigger_hint="", + autotrigger_file="", + trigger_line="", + subagent_requires_mode=False, + plugin_dir=None, + hooks_file=None, + editable_globs=self.editable, + skills_need_trigger=False, + ) + + def materialise(self, tree_dir: Path, spec, ctx: Mapping[str, str]) -> Install: + if spec.install_mode != "command_files": + raise ValueError( + f"CommandSubstrate needs install_mode 'command_files', got {spec.install_mode!r}" + ) + staged = Path(tempfile.mkdtemp(prefix="harness2-command-")) + try: + self.export_tree(self.import_tree(tree_dir), staged, base={}) + return Install(staged, RM.fingerprint(tree_dir), {"install_mode": spec.install_mode}) + except BaseException: + shutil.rmtree(staged, ignore_errors=True) + raise + + def import_tree(self, root: Path, *, rejected: list[str] | None = None) -> HT.Tree: + return native_tree(self.surface(), root, (".",), rejected) + + def export_tree(self, tree: HT.Tree, root: Path, *, base: HT.Tree) -> dict[str, list[str]]: + surface = self.surface() + return export_files(root, surface_files(surface, tree), surface_files(surface, base)) + + def strip_native(self, workspace: Path) -> None: + self.export_tree({}, workspace, base=self.import_tree(workspace)) + + def install_into(self, install: Install, workspace: Path) -> None: + self.export_tree(self.import_tree(install.staging_dir), workspace, base={}) + + def run( + self, + prompt_file: Path, + workspace: Path, + run_dir: Path, + timeout: int, + env: Mapping[str, str], + ) -> int: + command = self.template.format( + workspace=shlex.quote(str(workspace)), + prompt_file=shlex.quote(str(prompt_file)), + output=shlex.quote(str(workspace / "output")), + ) + run_dir.mkdir(parents=True, exist_ok=True) + with (run_dir / "stream.log").open("w", encoding="utf-8") as stream: + return subprocess.run( + command, + shell=True, + cwd=workspace, + stdout=stream, + stderr=subprocess.STDOUT, + timeout=timeout, + env=dict(env), + ).returncode + + def read_trajectory( + self, stream_path: Path, *, obs_trunc: int, max_chars: int, args_trunc: int = 2000 + ) -> str: + text = stream_path.with_name("stream.log").read_text(encoding="utf-8", errors="replace") + tail = text[-obs_trunc:] if obs_trunc > 0 else text + return _clip(f"[OBSERVATION] {tail}", max_chars) + + def final_message(self, stream_path: Path) -> str: + return stream_path.with_name("stream.log").read_text(encoding="utf-8", errors="replace")[ + -4000: + ] + + def cost_usd(self, stream_path: Path) -> float: + return 0.0 diff --git a/harness2/substrate/opencode.py b/harness2/substrate/opencode.py index ad0a2ea4..29b1b0b6 100644 --- a/harness2/substrate/opencode.py +++ b/harness2/substrate/opencode.py @@ -14,7 +14,7 @@ """OpencodeSubstrate: the OpenCode agent framework as a solver substrate. -Carries the c1..c8 surface (OPENCODE_SURFACE), the three install modes, the bun-run CLI +Carries the c1..c8 surface (OPENCODE_SURFACE), the three install modes, the CLI contract, and the one opencode event-stream trajectory renderer. The install modes are different delivery channels for the always-on text and are not interchangeable: changing the one a benchmark uses moves that benchmark's baseline. @@ -33,10 +33,10 @@ from pathlib import Path from typing import Mapping -from .. import config from ..core import runmark as RM from ..core import tree as HT from ..core.tree import SurfaceSpec +from ._native import check_cli, cli_argv from .base import Install, Launcher, RunRequest OPENCODE_SURFACE = SurfaceSpec( @@ -175,8 +175,9 @@ class OpencodeSubstrate: name = "oc" - def __init__(self, launcher: Launcher | None = None): + def __init__(self, launcher: Launcher | None = None, *, bin: str = ""): self._launcher = launcher + self.bin = bin def surface(self) -> SurfaceSpec: return OPENCODE_SURFACE @@ -233,19 +234,14 @@ def run_argv( variant: str | None = None, agent: str = "", session: str | None = None, + bin: str = "", ) -> list[str]: """The reference argv; the real launchers build their own. - The upstream checkout runs from source through bun: there is no `opencode` binary in - the contract. With `agent=""` the rendered AGENTS.md appends to the provider prompt. + With `agent=""` the rendered AGENTS.md appends to the provider prompt. """ argv = [ - str(config.BUN), - "run", - "--cwd", - str(config.OPENCODE_DIR / "packages" / "opencode"), - "--conditions=browser", - "src/index.ts", + *cli_argv(bin or self.bin, "opencode"), "run", prompt, "--dir", @@ -305,19 +301,7 @@ def read_trajectory( ) def preflight(self) -> None: - """Checks the OpenCode checkout and bun exist; starts nothing.""" - oc = config.OPENCODE_DIR - if not (oc / "packages" / "opencode" / "src" / "index.ts").is_file(): - raise RuntimeError( - f"OpenCode checkout missing at {oc} (no packages/opencode/src/index.ts): " - f"run scripts/setup_benchmark.py to fetch the pinned checkout " - f"at {config.DEPENDENCY_PINS['opencode'][:12]} (v1.14.18)." - ) - if not config.BUN.is_file(): - raise RuntimeError( - f"bun not found at {config.BUN}: install bun on PATH or at ~/.bun/bin/bun " - f"(the upstream OpenCode is run from source through bun)." - ) + check_cli(self.bin, "opencode") _TRUNC_FMT = "…(+{n} chars — render truncation)" diff --git a/plugins/harness2-benchmarks/harness2_benchmarks/jobbench.py b/plugins/harness2-benchmarks/harness2_benchmarks/jobbench.py index ce6015bd..a670b84d 100644 --- a/plugins/harness2-benchmarks/harness2_benchmarks/jobbench.py +++ b/plugins/harness2-benchmarks/harness2_benchmarks/jobbench.py @@ -818,6 +818,8 @@ def _launch_codex(self, req: RunRequest, install: Install) -> list[Path]: env = self.substrate.exec_env(os.environ, codex_home=codex_home) env.update( { + # the pinned Codex dist ships its helper binaries beside the CLI + "PATH": str(Path(config.CODEX_DIST) / "codex-path") + os.pathsep + env["PATH"], "HARNESS2_CODEX_BIN": str(Path(config.CODEX_DIST) / "codex"), "TASK_IDS": task_id, "LABEL": label, diff --git a/tests/fixtures/claude-stream.jsonl b/tests/fixtures/claude-stream.jsonl new file mode 100644 index 00000000..d37b3e89 --- /dev/null +++ b/tests/fixtures/claude-stream.jsonl @@ -0,0 +1,7 @@ +{"type":"system","subtype":"init","cwd":"/workspace","session_id":"fixture-session","tools":["Bash","Read"],"model":"claude-fixture","permissionMode":"bypassPermissions","apiKeySource":"ANTHROPIC_API_KEY","claude_code_version":"2.1.0","uuid":"init-1"} +{"type":"assistant","message":{"id":"msg_1","type":"message","role":"assistant","model":"claude-fixture","content":[{"type":"thinking","thinking":"Inspect the input first.","signature":"fixture"},{"type":"text","text":"I will inspect the input."},{"type":"tool_use","id":"tool_1","name":"Bash","input":{"command":"cat input.txt"}}],"stop_reason":"tool_use","usage":{"input_tokens":20,"output_tokens":15}},"parent_tool_use_id":null,"session_id":"fixture-session","uuid":"assistant-1"} +{"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"tool_1","content":[{"type":"text","text":"input contents"}]}]},"parent_tool_use_id":null,"session_id":"fixture-session","uuid":"user-1"} +{"type":"assistant","message":{"id":"msg_2","type":"message","role":"assistant","content":[{"type":"tool_use","id":"tool_2","name":"Read","input":{"file_path":"missing.txt"}}]},"parent_tool_use_id":null,"session_id":"fixture-session","uuid":"assistant-2"} +{"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"tool_2","content":"File does not exist.","is_error":true}]},"parent_tool_use_id":null,"session_id":"fixture-session","uuid":"user-2"} +{"type":"assistant","message":{"id":"msg_3","type":"message","role":"assistant","content":[{"type":"text","text":"Finished the task."}],"stop_reason":"end_turn"},"parent_tool_use_id":null,"session_id":"fixture-session","uuid":"assistant-3"} +{"type":"result","subtype":"success","is_error":false,"duration_ms":1000,"duration_api_ms":900,"num_turns":2,"result":"Task complete.","total_cost_usd":0.0123,"usage":{"input_tokens":30,"output_tokens":25},"modelUsage":{},"permission_denials":[],"session_id":"fixture-session","uuid":"result-1"} diff --git a/tests/test_bridge.py b/tests/test_bridge.py new file mode 100644 index 00000000..a33ff1a9 --- /dev/null +++ b/tests/test_bridge.py @@ -0,0 +1,139 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline configuration and endpoint persistence checks for the provider bridge.""" + +from __future__ import annotations + +import contextlib +import io +import json +import os +import tempfile +import tomllib +import unittest +from pathlib import Path +from unittest.mock import patch + +from harness2 import providers, settings +from harness2.cli import bridge + + +class BridgeTests(unittest.TestCase): + def setUp(self): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + self.root = Path(temp.name) + self.project = self.root / "project" + (self.project / ".harness2").mkdir(parents=True) + self.binary = self.root / "bin/litellm" + self.binary.parent.mkdir() + self.binary.write_text("fixture") + + def test_endpoint_reuses_key_and_port_and_has_private_permissions(self): + path = self.root / "state/bridge.json" + with patch.object(bridge, "_free_port", return_value=12345) as free: + initial = bridge._endpoint(path, "127.0.0.1", 0) + self.assertEqual(initial["port"], 12345) + self.assertEqual(bridge._endpoint(path, "127.0.0.1", 0), initial) + self.assertEqual(free.call_count, 1) + updated = bridge._endpoint(path, "127.0.0.1", 23456) + self.assertEqual(updated, {"port": 23456, "key": initial["key"]}) + self.assertEqual(path.stat().st_mode & 0o777, 0o600) + self.assertEqual(json.loads(path.read_text()), updated) + self.assertGreater(bridge._free_port("127.0.0.1"), 0) + + def test_provider_config_and_state_without_launching_server(self): + for kind, extra in ( + ("anthropic", 'api_key_env="TEST_KEY"\nbase_url="https://example.test"'), + ("openai", 'api_key_env="TEST_KEY"\nbase_url="https://example.test/v1"'), + ("vertex", 'project="test-project"\nregion="test-region"'), + ("bedrock", 'region="test-region"'), + ): + with self.subTest(kind=kind): + (self.project / ".harness2/config.toml").write_text( + f'[providers.custom]\ntype="{kind}"\n{extra}\n' + ) + stdout = io.StringIO() + cache = self.root / kind + with ( + patch.dict( + os.environ, + { + "XDG_CONFIG_HOME": str(self.root / "config"), + "XDG_CACHE_HOME": str(cache), + "TEST_KEY": "secret", + }, + ), + patch.object(bridge.settings, "find_project", return_value=self.project), + patch.object(bridge.sys, "executable", str(self.binary.with_name("python"))), + patch.object(bridge.os, "execve") as execute, + contextlib.redirect_stdout(stdout), + ): + bridge.main(["--model", "custom/test-model", "--port", "12345"]) + expected = providers.litellm_params( + settings.load(self.project).provider("custom"), "test-model" + ) + root = cache / "harness2/bridge" + config = json.loads((root / "bridge.yaml").read_text()) + self.assertEqual( + config["model_list"], [{"model_name": "test-model", "litellm_params": expected}] + ) + self.assertEqual( + config["general_settings"]["master_key"], "os.environ/HARNESS2_BRIDGE_KEY" + ) + binary, argv, env = execute.call_args.args + self.assertEqual(binary, str(self.binary)) + self.assertEqual( + argv, + [ + str(self.binary), + "--config", + str(root / "bridge.yaml"), + "--host", + "127.0.0.1", + "--port", + "12345", + ], + ) + endpoint = json.loads((root / "bridge.json").read_text()) + self.assertEqual(env["HARNESS2_BRIDGE_KEY"], endpoint["key"]) + self.assertIn("[providers.bridge]", stdout.getvalue()) + self.assertIn('api_key_env = "HARNESS2_BRIDGE_KEY"', stdout.getvalue()) + snippet = ( + stdout.getvalue() + .split("Provider configuration:\n", 1)[1] + .split("\nEnvironment:", 1)[0] + ) + self.assertEqual(tomllib.loads(snippet)["providers"]["bridge"]["type"], "openai") + self.assertNotIn("secret", (root / "bridge.yaml").read_text()) + + def test_model_required_and_unknown_provider_fails(self): + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as raised: + bridge.main([]) + self.assertEqual(raised.exception.code, 2) + with ( + patch.object(bridge.sys, "executable", str(self.binary.with_name("python"))), + patch.object(bridge.settings, "find_project", return_value=self.project), + patch.dict(os.environ, {"XDG_CONFIG_HOME": str(self.root / "config")}), + contextlib.redirect_stderr(io.StringIO()), + ): + with self.assertRaises(SystemExit) as raised: + bridge.main(["--model", "unknown/model"]) + self.assertEqual(raised.exception.code, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_executors.py b/tests/test_executors.py new file mode 100644 index 00000000..ffc20f51 --- /dev/null +++ b/tests/test_executors.py @@ -0,0 +1,592 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline executor contracts, native-file safety and documented Claude stream shapes.""" + +from __future__ import annotations + +import json +import os +import shlex +import shutil +import subprocess +import sys +import tempfile +import unittest +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from harness2 import providers +from harness2.core import prompts +from harness2.core import tree as HT +from harness2.core.spec import INSTALL_MODES, BenchPrompts +from harness2.settings import ConfigError +from harness2.substrate import ( + CLAUDE_SURFACE, + CODEX_SURFACE, + OPENCODE_SURFACE, + ClaudeSubstrate, + CodexSubstrate, + CommandSubstrate, + OpencodeSubstrate, +) +from harness2.substrate.claude import parse_events + + +class SurfaceTests(unittest.TestCase): + def test_script_dir_and_references(self): + for surface in (CLAUDE_SURFACE, CODEX_SURFACE, OPENCODE_SURFACE): + with self.subTest(surface=surface.name): + script = f"{surface.script_dir}/helper.py" + self.assertTrue(HT.is_allowed(surface, script)) + if surface.script_dir != "scripts": + self.assertFalse(HT.is_allowed(surface, "scripts/helper.py")) + tree = {surface.always_on[0]: f"python3 {script}", script: "print('ok')"} + HT.validate(surface, tree) + del tree[script] + with self.assertRaisesRegex(ValueError, "but that file is not in the harness"): + HT.validate(surface, tree) + tree = { + surface.always_on[0]: "helper", + f"{surface.skill_dir}/helper/SKILL.md": f"---\nname: helper\ndescription: Help\n---\npython3 {surface.skill_dir}/helper/scripts/tool.py", + } + with self.assertRaisesRegex(ValueError, "tool.py.*not in the harness"): + HT.validate(surface, tree) + tree[f"{surface.skill_dir}/helper/scripts/tool.py"] = "pass" + HT.validate(surface, tree) + + def test_skills_can_be_discovered_without_trigger(self): + tree = {".claude/skills/helper/SKILL.md": "---\ndescription: Help\n---\nDo it."} + before = dict(tree) + HT.validate(CLAUDE_SURFACE, tree) + self.assertEqual(HT.autotrigger(CLAUDE_SURFACE, tree), []) + self.assertEqual(tree, before) + required = replace(CLAUDE_SURFACE, skills_need_trigger=True) + with self.assertRaisesRegex(ValueError, "never named"): + HT.validate(required, tree) + tree[".claude/skills/helper/SKILL.md"] = "missing description" + with self.assertRaisesRegex(ValueError, "description"): + HT.validate(CLAUDE_SURFACE, tree) + + def test_globs_override_component_rules_but_keep_caps_and_compile(self): + substrate = CommandSubstrate("true", ("custom/*",)) + surface = substrate.surface() + self.assertEqual(substrate.NATIVE_PATHS, (":(glob)custom/*",)) + self.assertTrue(HT.is_allowed(surface, "custom/a.txt")) + self.assertFalse(HT.is_allowed(surface, "sub_agents/helper/AGENT.md")) + self.assertFalse(HT.is_allowed(surface, "../custom/a.txt")) + HT.validate(surface, {"custom/hook.json": "bad json", "ignored.py": "bad syntax!"}) + with self.assertRaisesRegex(ValueError, "does not compile"): + HT.validate(surface, {"custom/tool.py": "bad syntax!"}) + special = replace(surface, editable_globs=("**",), hooks_file="hooks.json") + HT.validate( + special, + {"sub_agents/a/AGENT.md": "no description", "hooks.json": "bad"}, + allow_subagents=False, + ) + capped = replace(surface, always_on=("custom/a.txt",), always_on_max=2) + with self.assertRaisesRegex(ValueError, "over the 2 cap"): + HT.validate(capped, {"custom/a.txt": "long"}) + with self.assertRaises(ValueError): + CommandSubstrate("true", ()) + + def test_new_install_modes_and_all_surface_prompts(self): + self.assertIn("claude_root", INSTALL_MODES) + self.assertIn("command_files", INSTALL_MODES) + for surface in ( + CLAUDE_SURFACE, + CODEX_SURFACE, + OPENCODE_SURFACE, + CommandSubstrate("true", ("*.md",)).surface(), + ): + with self.subTest(surface=surface.name): + text = prompts.propose(BenchPrompts(work="x"), "INV", surface) + self.assertNotIn("__", text) + + def test_glob_surface_accepts_empty_directory_as_tree(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + command = CommandSubstrate("true", ("custom/*",)).surface() + self.assertTrue(HT.is_tree(command, root)) + self.assertFalse(HT.is_tree(command, root / "missing")) + file = root / "file.txt" + file.write_text("content") + self.assertFalse(HT.is_tree(command, file)) + for surface in (CLAUDE_SURFACE, CODEX_SURFACE, OPENCODE_SURFACE): + self.assertFalse(HT.is_tree(surface, root)) + always_on = root / surface.always_on[0] + always_on.write_text("") + self.assertTrue(HT.is_tree(surface, root)) + always_on.unlink() + + +class NativeTests(unittest.TestCase): + def setUp(self): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + self.root = Path(temp.name) + + def write(self, rel, text): + path = self.root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + def native_project(self, substrate): + surface = substrate.surface() + self.write(surface.always_on[0], "Use helper.") + self.write(f"{surface.skill_dir}/helper/SKILL.md", "---\ndescription: Help\n---\nBody") + self.write(f"{surface.skill_dir}/helper/scripts/tool.py", "pass\n") + self.write(f"{surface.skill_dir}/helper/reference.md", "keep reference") + self.write(f"{surface.script_dir}/tool.py", "pass\n") + self.write(f"{surface.script_dir}/extra.txt", "keep extra") + hooks = {"hooks": {"Stop": [{"hooks": [{"type": "command", "command": "true"}]}]}} + if surface.name == "claude": + self.write( + ".claude/settings.json", json.dumps({"permissions": {"allow": ["Read"]}, **hooks}) + ) + self.write(".claude/agents/helper.md", "---\ndescription: Help\n---\nDo it.") + self.write(".claude/agents/notes.txt", "keep notes") + else: + self.write(surface.hooks_file, json.dumps(hooks)) + self.write(".codex/config.toml", "keep = true") + return hooks + + def test_claude_round_trip_hooks_merge_and_bounded_deletions(self): + sub = ClaudeSubstrate() + hooks = self.native_project(sub) + rejected = [] + tree = sub.import_tree(self.root, rejected=rejected) + self.assertEqual( + set(rejected), + { + ".claude/skills/helper/reference.md", + ".claude/scripts/extra.txt", + ".claude/agents/notes.txt", + }, + ) + self.assertIn("sub_agents/helper/AGENT.md", tree) + self.assertEqual(json.loads(tree[".claude/hooks.json"]), hooks) + dest = self.root / "destination" + settings = dest / ".claude/settings.json" + settings.parent.mkdir(parents=True) + settings.write_text('{"model": "keep"}') + result = sub.export_tree(tree, dest, base={}) + self.assertIn(".claude/agents/helper.md", result["written"]) + self.assertIn("name: helper", (dest / ".claude/agents/helper.md").read_text()) + self.assertEqual(json.loads(settings.read_text()), {"model": "keep", **hooks}) + imported = sub.import_tree(dest) + self.assertEqual( + sub.export_tree(imported, dest, base=imported), {"written": [], "deleted": []} + ) + self.assertEqual(tree.keys(), imported.keys()) + changed = {k: v for k, v in tree.items() if k not in (".claude/hooks.json", "CLAUDE.md")} + result = sub.export_tree(changed, self.root, base=tree) + self.assertEqual(result["deleted"], ["CLAUDE.md"]) + self.assertIn(".claude/settings.json", result["written"]) + self.assertEqual( + json.loads((self.root / ".claude/settings.json").read_text()), + {"permissions": {"allow": ["Read"]}}, + ) + self.assertEqual( + (self.root / ".claude/skills/helper/reference.md").read_text(), "keep reference" + ) + + def test_claude_agent_name_inserted_only_when_missing(self): + sub = ClaudeSubstrate() + for name, body in ( + ("present", "---\nname: present\ndescription: Help\n---\nBody"), + ("plain", "Body"), + ("empty", "---\nname: \ndescription: Help\n---\nBody"), + ("header", "---\n---\nBody"), + ): + sub.export_tree({f"sub_agents/{name}/AGENT.md": body}, self.root, base={}) + exported = (self.root / f".claude/agents/{name}.md").read_text() + self.assertIn(f"name: {name}\n", exported) + self.assertEqual(sum(line.startswith("name:") for line in exported.splitlines()), 1) + if name == "present": + self.assertEqual(exported, body) + + def test_codex_round_trip_no_stubs_or_subagents_and_bounded_deletions(self): + sub = CodexSubstrate() + self.native_project(sub) + rejected = [] + tree = sub.import_tree(self.root, rejected=rejected) + self.assertIn(".agents/skills/helper/reference.md", rejected) + dest = self.root / "destination" + candidate = { + **tree, + "sub_agents/helper/AGENT.md": "description: Help", + **dict(CODEX_SURFACE.stubs), + } + result = sub.export_tree(candidate, dest, base={}) + self.assertEqual(sub.import_tree(dest), tree) + self.assertFalse((dest / "sub_agents").exists()) + self.assertFalse(any(dest.rglob("README.md"))) + self.assertEqual(set(result["written"]), set(tree)) + self.assertEqual(sub.export_tree(tree, dest, base=tree), {"written": [], "deleted": []}) + result = sub.export_tree({}, self.root, base=tree) + self.assertEqual(set(result["deleted"]), set(tree)) + self.assertTrue((self.root / ".codex/config.toml").is_file()) + self.assertTrue((self.root / ".agents/skills/helper/reference.md").is_file()) + + def test_strip_native_preserves_unmanaged_files_and_settings(self): + for sub in (ClaudeSubstrate(), CodexSubstrate()): + with self.subTest(substrate=sub.name): + self.native_project(sub) + sub.strip_native(self.root) + self.assertEqual(sub.import_tree(self.root), {}) + surface = sub.surface() + self.assertTrue((self.root / f"{surface.skill_dir}/helper/reference.md").is_file()) + self.assertTrue((self.root / f"{surface.script_dir}/extra.txt").is_file()) + if sub.name == "claude": + self.assertIn( + "permissions", json.loads((self.root / ".claude/settings.json").read_text()) + ) + self.assertTrue((self.root / ".claude/agents/notes.txt").is_file()) + + def test_materialise_and_install_native_layout(self): + for sub, mode in ( + (ClaudeSubstrate(), "claude_root"), + (CodexSubstrate(), "codex_root"), + (CommandSubstrate("true", ("custom/*",)), "command_files"), + ): + with self.subTest(mode=mode): + canonical = self.root / mode + surface = sub.surface() + if mode == "command_files": + tree = {"custom/file.txt": "custom"} + else: + tree = { + surface.always_on[0]: "instructions", + f"{surface.script_dir}/tool.py": "pass", + } + if mode == "claude_root": + tree["sub_agents/helper/AGENT.md"] = "---\ndescription: Help\n---\nBody" + tree[".claude/hooks.json"] = '{"hooks":{"Stop":[]}}' + HT.write_tree(surface, tree, canonical) + (canonical / "unmanaged.txt").write_text("do not stage") + install = sub.materialise(canonical, SimpleNamespace(install_mode=mode), {}) + self.addCleanup(shutil.rmtree, install.staging_dir, True) + self.assertTrue(install.fingerprint) + self.assertFalse((install.staging_dir / "unmanaged.txt").exists()) + workspace = self.root / f"{mode}-workspace" + if mode == "claude_root": + (workspace / ".claude").mkdir(parents=True) + (workspace / ".claude/settings.json").write_text('{"model":"keep"}') + sub.install_into(install, workspace) + imported = sub.import_tree(workspace) + self.assertEqual(set(imported), set(tree)) + if mode == "claude_root": + self.assertIn("name: helper", imported["sub_agents/helper/AGENT.md"]) + self.assertEqual( + json.loads((workspace / ".claude/settings.json").read_text())["model"], + "keep", + ) + else: + self.assertEqual(imported, tree) + with self.assertRaises(ValueError): + sub.materialise(canonical, SimpleNamespace(install_mode="wrong"), {}) + + def test_native_exports_reject_unmanaged_paths_and_symlinks_before_writes(self): + external = self.write("external.txt", "keep") + (self.root / "CLAUDE.md").symlink_to(external) + with self.assertRaisesRegex(ConfigError, "symlink"): + ClaudeSubstrate().export_tree({"CLAUDE.md": "overwrite"}, self.root, base={}) + self.assertEqual(external.read_text(), "keep") + with self.assertRaisesRegex(ConfigError, "symlink"): + ClaudeSubstrate().import_tree(self.root) + (self.root / "CLAUDE.md").unlink() + (self.root / ".agents").symlink_to(self.root, target_is_directory=True) + with self.assertRaisesRegex(ConfigError, "symlink"): + CodexSubstrate().export_tree( + {"AGENTS.md": "new", ".agents/scripts/tool.py": "pass"}, self.root, base={} + ) + self.assertFalse((self.root / "AGENTS.md").exists()) + for sub in (ClaudeSubstrate(), CodexSubstrate(), CommandSubstrate("true", ("*.md",))): + with self.subTest(substrate=sub.name): + with self.assertRaises(ValueError): + sub.export_tree({"unmanaged.txt": "bad"}, self.root, base={}) + with self.assertRaises(ValueError): + sub.export_tree({"../escape.md": "bad"}, self.root, base={}) + + def test_malformed_settings_and_hooks_never_overwrite_data(self): + settings = self.write(".claude/settings.json", '{"broken":') + sub = ClaudeSubstrate() + for text in ('{"broken":', "[]"): + settings.write_text(text) + with self.assertRaises(ConfigError): + sub.export_tree( + {"CLAUDE.md": "new", ".claude/hooks.json": '{"hooks":{}}'}, self.root, base={} + ) + self.assertEqual(settings.read_text(), text) + self.assertFalse((self.root / "CLAUDE.md").exists()) + settings.write_text('{"hooks":[]}') + with self.assertRaises(ConfigError): + sub.import_tree(self.root) + for content in ("bad", "[]", '{"hooks":[]}'): + with self.assertRaises(ConfigError): + sub.export_tree({".claude/hooks.json": content}, self.root, base={}) + + def test_command_native_files_and_run_with_quoted_paths(self): + sub = CommandSubstrate( + 'cat {prompt_file}; printf "diagnostic" >&2; ' + "mkdir -p {output}; cp {prompt_file} {output}/answer.txt; " + 'printf "%s" {workspace} > {output}/workspace.txt', + ("custom/*",), + ) + self.write("custom/rules.txt", "rules") + self.write("keep.txt", "keep") + base = sub.import_tree(self.root) + self.assertEqual(base, {"custom/rules.txt": "rules"}) + sub.strip_native(self.root) + self.assertTrue((self.root / "keep.txt").exists()) + self.assertFalse((self.root / "custom/rules.txt").exists()) + sub.export_tree(base, self.root, base={}) + prompt = self.write("prompt ' $file.md", "hello") + workspace = self.root / "work ' $dir" + workspace.mkdir() + run_dir = self.root / "run" + self.assertEqual(sub.run(prompt, workspace, run_dir, 5, os.environ), 0) + self.assertEqual((workspace / "output/answer.txt").read_text(), "hello") + self.assertEqual((workspace / "output/workspace.txt").read_text(), str(workspace)) + stream = run_dir / "stream.jsonl" + self.assertEqual(sub.final_message(stream), "hellodiagnostic") + self.assertEqual( + sub.read_trajectory(stream, obs_trunc=10, max_chars=100), "[OBSERVATION] diagnostic" + ) + self.assertEqual(sub.cost_usd(stream), 0.0) + self.assertEqual( + CommandSubstrate("exit 7", ("*.md",)).run(prompt, workspace, run_dir, 5, os.environ), 7 + ) + with self.assertRaises(subprocess.TimeoutExpired): + CommandSubstrate("sleep 2", ("*.md",)).run(prompt, workspace, run_dir, 0.01, os.environ) + + +class ClaudeStreamTests(unittest.TestCase): + fixture = Path(__file__).parent / "fixtures/claude-stream.jsonl" + + def test_documented_message_and_result_fields(self): + sub = ClaudeSubstrate() + rendered = sub.read_trajectory(self.fixture, obs_trunc=1000, max_chars=10000) + for text in ( + "[HARNESS] init | model=claude-fixture cwd=/workspace", + "[REASONING] Inspect the input first.", + "[ASSISTANT] I will inspect the input.", + '[ACTION] Bash: {"command": "cat input.txt"}', + "[OBSERVATION] input contents", + "[OBSERVATION ERROR] File does not exist.", + ): + self.assertIn(text, rendered) + self.assertEqual(sub.final_message(self.fixture), "Task complete.") + self.assertAlmostEqual(sub.cost_usd(self.fixture), 0.0123) + clipped = sub.read_trajectory(self.fixture, obs_trunc=5, max_chars=10000, args_trunc=5) + self.assertIn("…(+", clipped) + self.assertIn("render truncation)", clipped) + capped = sub.read_trajectory(self.fixture, obs_trunc=1000, max_chars=25) + self.assertTrue(capped.startswith(rendered[:25])) + self.assertIn("render truncation)", capped) + + def test_errors_fallback_junk_and_missing_capture(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "stream.jsonl" + path.write_text( + 'junk\n[]\n{"type":\n' + + json.dumps( + { + "type": "assistant", + "message": {"content": [{"type": "text", "text": "fallback"}]}, + } + ) + + "\n" + + json.dumps( + { + "type": "result", + "subtype": "error_during_execution", + "is_error": True, + "errors": ["failed to execute"], + "total_cost_usd": 0.1, + } + ) + + "\n" + ) + sub = ClaudeSubstrate() + self.assertEqual(len(parse_events(path)), 2) + self.assertEqual(sub.final_message(path), "fallback") + self.assertIn( + "[ERROR] failed to execute", + sub.read_trajectory(path, obs_trunc=100, max_chars=1000), + ) + path.write_text("junk\n{}\n") + self.assertEqual(sub.final_message(path), "") + self.assertEqual(sub.cost_usd(path), 0.0) + self.assertEqual( + sub.read_trajectory(path, obs_trunc=100, max_chars=1000), + "(no parseable events in trajectory)", + ) + path.unlink() + self.assertIn( + "failed to read trajectory", + sub.read_trajectory(path, obs_trunc=100, max_chars=1000), + ) + self.assertEqual(sub.final_message(path), "") + self.assertEqual(sub.cost_usd(path), 0.0) + + +class CliTests(unittest.TestCase): + def setUp(self): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + self.root = Path(temp.name) + self.capture = self.root / "capture.json" + script = ( + "#!" + + sys.executable + + "\n" + + """import json, os, sys +from pathlib import Path +if '--version' in sys.argv: + print('fixture 1.0') + raise SystemExit(int(os.environ.get('VERSION_EXIT', '0'))) +Path(os.environ['CAPTURE']).write_text(json.dumps({'argv': sys.argv[1:], 'env': dict(os.environ), 'stdin': sys.stdin.read(), 'cwd': os.getcwd()})) +print('{"type":"result","result":"ok","total_cost_usd":0.01}') +""" + ) + for name in ("claude", "codex", "opencode", "wrapper cli"): + binary = self.root / name + binary.write_text(script) + binary.chmod(0o755) + self.patch = patch.dict( + os.environ, + { + "PATH": str(self.root) + os.pathsep + os.environ["PATH"], + "CAPTURE": str(self.capture), + }, + ) + self.patch.start() + self.addCleanup(self.patch.stop) + + def test_argv_environment_and_stdin_on_fake_clis(self): + base = {**os.environ, **dict.fromkeys(providers.STRIP_ENV, "stale")} + provider = providers.Provider("test", "anthropic", base_url="https://example.test") + prompt = self.root / "prompt.md" + prompt.write_text("task input") + for sub in (ClaudeSubstrate(), CodexSubstrate()): + with self.subTest(substrate=sub.name): + if sub.name == "claude": + home = self.root / "claude-home" + env = sub.env(base, provider, home) + self.assertTrue(home.is_dir()) + self.assertEqual(env["CLAUDE_CONFIG_DIR"], str(home)) + self.assertEqual(env["ANTHROPIC_BASE_URL"], provider.base_url) + self.assertTrue( + all(k not in env for k in providers.STRIP_ENV if k != "ANTHROPIC_BASE_URL") + ) + else: + home = self.root / "codex-home" + codex_provider = providers.Provider( + "test", "openai", base_url="https://example.test/v1", api_key_env="TEST_KEY" + ) + sub.write_home(home, codex_provider, "test-model", "high") + self.assertEqual( + (home / "config.toml").read_text(), + providers.codex_config(codex_provider, "test-model", "high"), + ) + env = sub.env(base, home) + self.assertEqual(env["CODEX_HOME"], str(home)) + self.assertTrue(all(k not in env for k in providers.STRIP_ENV)) + self.assertEqual(sub.exec_env(base, codex_home=home), env) + self.assertEqual(sub.cost_usd(prompt), 0.0) + argv = sub.argv(prompt, self.root, "test-model", "high") + with prompt.open() as stdin: + subprocess.run( + argv, cwd=self.root, env=env, stdin=stdin, check=True, capture_output=True + ) + capture = json.loads(self.capture.read_text()) + self.assertEqual(capture["stdin"], "task input") + self.assertEqual(Path(capture["cwd"]), self.root.resolve()) + self.assertEqual(capture["env"]["PATH"], base["PATH"]) + if sub.name == "claude": + self.assertEqual( + capture["argv"], + [ + "-p", + "--output-format", + "stream-json", + "--verbose", + "--dangerously-skip-permissions", + "--setting-sources", + "project,local", + "--model", + "test-model", + "--effort", + "high", + ], + ) + else: + self.assertEqual( + capture["argv"], + [ + "exec", + "--json", + "--skip-git-repo-check", + "--dangerously-bypass-approvals-and-sandbox", + "--dangerously-bypass-hook-trust", + "--ephemeral", + "-C", + str(self.root), + "-", + ], + ) + + def test_wrapped_binary_overrides_legacy_argv_and_preflight(self): + wrapper = shlex.quote(str(self.root / "wrapper cli")) + " --wrapper" + for cls in (ClaudeSubstrate, CodexSubstrate, OpencodeSubstrate): + sub = cls(bin=wrapper) + sub.preflight() + with patch.dict(os.environ, {"VERSION_EXIT": "2"}): + with self.assertRaisesRegex(ConfigError, "version check failed"): + sub.preflight() + with self.assertRaisesRegex(ConfigError, "not found"): + cls(bin="nonexistent-harness2-binary").preflight() + cx = CodexSubstrate(bin="codex") + argv = cx.exec_argv("task", self.root, "model", reasoning_effort="low", bin=wrapper) + self.assertEqual(argv[:3], [str(self.root / "wrapper cli"), "--wrapper", "exec"]) + self.assertIn('model_reasoning_effort="low"', argv) + oc = OpencodeSubstrate(bin=wrapper) + argv = oc.run_argv( + "task", + self.root, + "test/model", + title="title", + agent="helper", + variant="high", + session="session", + ) + self.assertEqual( + argv[:5], [str(self.root / "wrapper cli"), "--wrapper", "run", "task", "--dir"] + ) + self.assertEqual( + argv[-6:], ["--agent", "helper", "--variant", "high", "--session", "session"] + ) + self.assertEqual( + oc.run_argv("task", self.root, "test/model", bin="opencode")[0], + str(self.root / "opencode"), + ) + + +if __name__ == "__main__": + unittest.main() From 41011be5c5610d97c84740f25c34e556718098b1 Mon Sep 17 00:00:00 2001 From: Bae Seongmin <83870598+3ae3ae@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:56:16 +0900 Subject: [PATCH 06/21] feat: single harness2 command with subcommands (#6) Replaces the seven harness2-* scripts with one `harness2` command (`python -m harness2` works too) whose global options -C, --profile, -c KEY=VALUE, -q, -v and --json apply to every subcommand: run, doctor, setup, judge, report, runs, diff, task, config, demo, bridge, completion, plus init, status, chat and apply (stubs for now). The old script names stay as aliases. A run lives in /.harness2/runs// and records run.json: the identity settings, the task stream and the schedule. Resuming with different identity settings is refused with the differing keys, and judge, report, status, diff and apply find everything from the run name (default: the most recent run) instead of repeating flags. `run --dry-run` prints the task stream and the number of rollouts and model sessions the run will make; --budget-usd and --max-calls stop a run cleanly (exit 4) and Ctrl-C stops it for a later resume (exit 130). judged.json records the judge model it was graded with. harness2/config.py is gone: its generic parts were replaced by settings.py, its benchmark parts moved to the plugin. Co-authored-by: Claude Opus 5.5 --- MANIFEST.in | 8 +- harness2/__main__.py | 22 ++ harness2/cli/_adopt.py | 145 +++++++++ harness2/cli/_common.py | 332 +++++++++++++------- harness2/cli/apply.py | 28 ++ harness2/cli/bridge.py | 55 +++- harness2/cli/chat.py | 28 ++ harness2/cli/check.py | 50 --- harness2/cli/completion.py | 92 ++++++ harness2/cli/config.py | 74 +++++ harness2/cli/demo.py | 48 ++- harness2/cli/diff.py | 50 +++ harness2/cli/doctor.py | 121 ++++++++ harness2/cli/human.py | 23 ++ harness2/cli/init.py | 28 ++ harness2/cli/judge.py | 199 ++++++------ harness2/cli/main.py | 102 ++++++ harness2/cli/report.py | 102 +++--- harness2/cli/run.py | 257 +++++++++++++++ harness2/cli/run_sweep.py | 239 -------------- harness2/cli/runs.py | 93 ++++++ harness2/cli/setup.py | 37 +++ harness2/cli/status.py | 28 ++ harness2/cli/task.py | 59 ++++ harness2/config.py | 189 ----------- pyproject.toml | 17 +- tests/test_cli.py | 619 +++++++++++++++++++++++++++++++++++++ 27 files changed, 2240 insertions(+), 805 deletions(-) create mode 100644 harness2/__main__.py create mode 100644 harness2/cli/_adopt.py create mode 100644 harness2/cli/apply.py create mode 100644 harness2/cli/chat.py delete mode 100644 harness2/cli/check.py create mode 100644 harness2/cli/completion.py create mode 100644 harness2/cli/config.py create mode 100644 harness2/cli/diff.py create mode 100644 harness2/cli/doctor.py create mode 100644 harness2/cli/human.py create mode 100644 harness2/cli/init.py create mode 100644 harness2/cli/main.py create mode 100644 harness2/cli/run.py delete mode 100644 harness2/cli/run_sweep.py create mode 100644 harness2/cli/runs.py create mode 100644 harness2/cli/setup.py create mode 100644 harness2/cli/status.py create mode 100644 harness2/cli/task.py delete mode 100644 harness2/config.py create mode 100644 tests/test_cli.py diff --git a/MANIFEST.in b/MANIFEST.in index e7cc8b27..6fe169b9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,8 +1,2 @@ -include README.md .env.example -recursive-include docs *.md -recursive-include scripts *.py *.sh -recursive-include harness2/third_party * -include dependencies/README.md dependencies/PINNED.md +include README.md LICENSE global-exclude __pycache__ *.py[cod] *.log - -recursive-include assets *.png diff --git a/harness2/__main__.py b/harness2/__main__.py new file mode 100644 index 00000000..f900011d --- /dev/null +++ b/harness2/__main__.py @@ -0,0 +1,22 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run the harness2 CLI as a module.""" + +from __future__ import annotations + +from .cli.main import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/harness2/cli/_adopt.py b/harness2/cli/_adopt.py new file mode 100644 index 00000000..1fc5eb75 --- /dev/null +++ b/harness2/cli/_adopt.py @@ -0,0 +1,145 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Open stored runs and export selected harness trees.""" + +from __future__ import annotations + +import difflib +import subprocess +from pathlib import Path + +from .. import registry +from ..core import schedule +from ..core.spec import BenchmarkAdapter +from ..core.tree import Tree, read_tree +from ..settings import ConfigError +from . import _common as C + + +def open_run(ctx, name: str | None) -> tuple[BenchmarkAdapter, Path]: + root = C.resolve_run(ctx, name) + record = C.load_run(root) + config = C.run_settings(ctx, record) + split_file = record["schedule"].get("split_file") + suite_ctx = registry.SuiteContext( + config, ctx.project, root, Path(split_file) if split_file else None + ) + return registry.load(record["suite"]).make_adapter(suite_ctx), root + + +def adopted_tree(adapter, run_dir, task: str | None, mode: str) -> tuple[str, Tree]: + record = C.load_run(run_dir) + mode = schedule.mode_token(mode) + plan = record["schedule"] + ns = schedule.ns( + adapter.spec.name, adapter.substrate.name, mode, plan["k"], seed=plan.get("seed") + ) + summary_ns = C.summary_namespace(adapter, plan["k"], seed=plan.get("seed")) + tasks = [task] if task else [pair[1] for pair in reversed(record["stream"])] + rows = { + row["task_id"]: row + for path, row in C.summaries(run_dir) + if path.parent.parent.name == summary_ns + } + for task_id in tasks: + selection = (rows.get(task_id, {}).get("selections") or {}).get(mode) or {} + if not selection.get("ok") or not selection.get("chosen_label"): + continue + if selection.get("chosen_tree_is_base"): + return task_id, base_tree(adapter) + path = ( + C.runs_root_for(adapter) + / ns + / task_id.replace("/", "__") + / "harnesses" + / selection["chosen_label"] + ) + if not path.is_dir(): + raise ConfigError(f"selected harness is missing: {path}") + return task_id, read_tree(adapter.substrate.surface(), path) + raise ConfigError(f"no successful selection for {task or 'any task'} in {C.mode_label(mode)}") + + +def base_tree(adapter) -> Tree: + return read_tree(adapter.substrate.surface(), adapter.base_harness()) + + +def project_tree(adapter, settings) -> Tree: + return adapter.substrate.import_tree(settings.path("paths.harness")) + + +def changes(before: Tree, after: Tree) -> list[tuple[str, str]]: + return [ + (rel, "A" if rel not in before else "D" if rel not in after else "M") + for rel in sorted(before.keys() | after.keys()) + if before.get(rel) != after.get(rel) + ] + + +def unified(rel: str, before: Tree, after: Tree) -> str: + lines = [] + for line in difflib.unified_diff( + before.get(rel, "").splitlines(keepends=True), + after.get(rel, "").splitlines(keepends=True), + fromfile=f"a/{rel}" if rel in before else "/dev/null", + tofile=f"b/{rel}" if rel in after else "/dev/null", + ): + lines.append(line if line.endswith("\n") else line + "\n\\ No newline at end of file\n") + return "".join(lines) + + +def dirty_paths(settings, adapter) -> list[str]: + root = settings.path("paths.harness") + probe = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--show-toplevel"], capture_output=True, text=True + ) + if probe.returncode: + return [] + result = subprocess.run( + [ + "git", + "-C", + str(root), + "status", + "--porcelain", + "-z", + "--untracked-files=all", + "--", + *adapter.substrate.NATIVE_PATHS, + ], + capture_output=True, + check=True, + ) + fields = result.stdout.decode("utf-8", errors="replace").split("\0") + paths, index = set(), 0 + while index < len(fields) and fields[index]: + entry = fields[index] + paths.add(entry[3:]) + index += 1 + if "R" in entry[:2] or "C" in entry[:2]: + paths.add(fields[index]) + index += 1 + return sorted(paths) + + +def perform(settings, adapter, tree: Tree, *, base: Tree, branch: str | None) -> dict: + root = settings.path("paths.harness") + if branch: + result = subprocess.run( + ["git", "-C", str(root), "switch", "-c", branch], capture_output=True, text=True + ) + if result.returncode: + raise ConfigError(result.stderr.strip() or f"cannot create branch {branch!r}") + return adapter.substrate.export_tree(tree, root, base=base) diff --git a/harness2/cli/_common.py b/harness2/cli/_common.py index ea88dbc9..087a28ef 100644 --- a/harness2/cli/_common.py +++ b/harness2/cli/_common.py @@ -12,83 +12,214 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Shared plumbing for the CLIs: adapter construction and the on-disk conventions. - -The conventions are defined here once, because run_sweep writes them and -judge/report read them back. -""" +"""CLI settings, output, and persisted run conventions.""" from __future__ import annotations -import argparse +import json +import os +import sys +from contextlib import contextmanager, redirect_stdout +from dataclasses import dataclass +from datetime import datetime, timezone +from functools import cached_property +from importlib.metadata import version from pathlib import Path -from .. import config +from .. import settings from ..core import library as LIB -from ..core import schedule as SCHED -from ..core.spec import BenchmarkAdapter - -BENCHES = ("jb", "wb", "lab") -WB_SUBSETS = ("office", "web", "code") -SUBSTRATES = ("oc", "cx") - -MODES_DEFAULT = "parallel" -MODES_HELP = ( - "parallel | sequential (agg / seq accepted: the on-disk tokens); " - "comma list, e.g. 'parallel' or 'parallel,sequential'" -) - - -class HelpFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter): - """Defaults in the help text, module docstring kept verbatim as the description.""" - - -def make_adapter( - bench: str, subset: str | None, substrate: str, split_file: Path | None = None -) -> BenchmarkAdapter: - """The one place a CLI turns (--bench, --subset, --substrate) into an adapter.""" - if bench not in BENCHES: - raise ValueError(f"unknown bench {bench!r}; expected one of {BENCHES}") - if substrate not in SUBSTRATES: - raise ValueError(f"unknown substrate {substrate!r}; expected one of {SUBSTRATES}") - if bench == "wb": - if subset not in WB_SUBSETS: - raise ValueError(f"--bench wb needs --subset from {WB_SUBSETS}, got {subset!r} ") - from harness2_benchmarks.workbuddy import WbAdapter - - return WbAdapter( - subset, - substrate=substrate, - split_file=split_file, - runs_root=config.RUNS / "wb" / subset, - ) - if subset: - raise ValueError(f"--subset applies to --bench wb only, got {subset!r} for {bench}") - if bench == "jb": - from harness2_benchmarks.jobbench import JobBenchAdapter +from ..core import schedule +from ..settings import ConfigError + + +@dataclass +class Context: + args: object + flags: dict[str, str] + + @cached_property + def project_or_none(self) -> Path | None: + return settings.find_project(Path(self.args.project or Path.cwd())) - return JobBenchAdapter( - substrate=substrate, split_file=split_file, runs_root=config.RUNS / "jb" + @property + def project(self) -> Path: + if self.project_or_none is None: + raise ConfigError( + "not inside a harness2 project (no .harness2/ found); run `harness2 init`" + ) + return self.project_or_none + + @cached_property + def overrides(self) -> dict: + values = {} + for raw in self.args.config: + key, sep, value = raw.partition("=") + if not sep or not key.strip(): + raise ConfigError("--config requires KEY=VALUE") + values[key.strip()] = (settings.parse_value(value), "-c") + for name, key in self.flags.items(): + value = getattr(self.args, name, None) + if value is not None: + values[key] = (value, f"flag --{name.replace('_', '-')}") + return values + + @cached_property + def settings(self) -> settings.Settings: + return settings.load( + self.project_or_none, profile=self.args.profile, overrides=self.overrides ) - from harness2_benchmarks.lab import LabAdapter - return LabAdapter(substrate=substrate, split_file=split_file, runs_root=config.RUNS / "lab") + def out(self, data, text_fn) -> None: + if self.args.json: + print(json.dumps(data, ensure_ascii=False, default=str)) + else: + text = text_fn(data) + if text is not None: + print(text) + def log(self, message: str) -> None: + if not self.args.quiet: + print(message, file=sys.stderr) -def runs_root_for(adapter: BenchmarkAdapter) -> Path: - """Scheduler state: summaries, harnesses and selections, not the rollout store.""" - return adapter.spec.runs_root / "adapt" + def debug(self, message: str) -> None: + if self.args.verbose: + self.log(message) + + @contextmanager + def progress(self): + if self.args.quiet: + with open(os.devnull, "w") as stream, redirect_stdout(stream): + yield + else: + with redirect_stdout(sys.stderr): + yield + + +def run_path(ctx: Context, name: str) -> Path: + if not name or name in (".", "..") or Path(name).name != name: + raise ConfigError("run name must be a single directory name") + return ctx.settings.path("paths.runs") / name + + +def resolve_run(ctx: Context, name: str | None) -> Path: + ctx.project + if name is not None: + path = run_path(ctx, name) + if not (path / "run.json").is_file(): + raise ConfigError(f"run {name!r} has no run.json") + return path + root = ctx.settings.path("paths.runs") + runs = ( + [p for p in root.iterdir() if p.is_dir() and (p / "run.json").is_file()] + if root.is_dir() + else [] + ) + if not runs: + raise ConfigError(f"no runs under {root}") + return max(runs, key=lambda p: p.stat().st_mtime_ns) + + +def load_run(run_dir: Path) -> dict: + try: + record = json.loads((run_dir / "run.json").read_text(encoding="utf-8")) + if not isinstance(record, dict) or not all( + key in record for key in ("name", "settings", "suite", "executor", "stream", "schedule") + ): + raise ValueError("missing run metadata") + if not isinstance(record["settings"], dict) or not isinstance(record["schedule"], dict): + raise ValueError("settings and schedule must be objects") + if not isinstance(record["stream"], list) or any( + not isinstance(pair, list) + or len(pair) != 2 + or not all(isinstance(item, str) for item in pair) + for pair in record["stream"] + ): + raise ValueError("stream must contain [domain, task] pairs") + return record + except (OSError, ValueError) as exc: + raise ConfigError(f"cannot read {run_dir / 'run.json'}: {exc}") from exc + + +def identity(config: settings.Settings) -> dict: + return { + key: value + for key, value in config.flat().items() + if key.startswith(("suite.", "executor.", "improver.", "selector.", "reflector.")) + or key in ("paths.harness", "paths.tasks", "run.seed", "run.obs_trunc") + } + + +def default_name(config: settings.Settings) -> str: + name = f"{config.get('executor.name')}-k{config.get('run.k')}" + seed = config.get("run.seed") + return name + (f"-s{seed}" if seed is not None else "") -def library_root_for(adapter: BenchmarkAdapter, k: int, *, seed: int | None, tag: str = "") -> Path: - """Per-(benchmark, substrate, k, seed, tag) experience-library root (the improver's R). +def save_run(run_dir: Path, config: settings.Settings, stream: list, plan: dict) -> dict: + subset = identity(config) + path = run_dir / "run.json" + old = load_run(run_dir) if path.is_file() else None + if old: + differences = [ + f"{key}: {old['settings'].get(key)!r} -> {subset.get(key)!r}" + for key in sorted(old["settings"].keys() | subset.keys()) + if old["settings"].get(key) != subset.get(key) + or (key in old["settings"]) != (key in subset) + ] + if old["stream"] != [list(pair) for pair in stream]: + differences.append(f"stream: {old['stream']!r} -> {stream!r}") + if differences: + raise ConfigError("run identity differs:\n" + "\n".join(differences)) + record = { + "name": run_dir.name, + "created": old["created"] if old else datetime.now(timezone.utc).isoformat(), + "harness2_version": version("harness2"), + "suite": config.get("suite.name"), + "executor": config.get("executor.name"), + "settings": subset, + "stream": [list(pair) for pair in stream], + "schedule": plan, + } + run_dir.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(".json.tmp") + temporary.write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8") + temporary.replace(path) + return record - Separate seeds and tags keep independent task streams from sharing entries. - """ - SCHED.validate_tag(tag) + +def run_settings(ctx: Context, record: dict) -> settings.Settings: + overrides = {key: (value, "run.json") for key, value in record["settings"].items()} + for key in ("k", "modes", "seed"): + if key in record["schedule"]: + overrides[f"run.{key}"] = (record["schedule"][key], "run.json") + overrides.update( + {key: value for key, value in ctx.overrides.items() if key.startswith("judge.")} + ) + current = settings.load(ctx.project, profile=ctx.args.profile, overrides=overrides) + data = {name: current.section(name) for name in settings.DEFAULTS} + origins = {key: current.origin(key) for key in current.flat()} + # Empty snapshot sections must not inherit fields added to the project since the run. + for name in ("suite", "executor", "improver", "selector", "reflector"): + data[name] = {} + for key, value in record["settings"].items(): + parts = key.split(".") + table = data + for part in parts[:-1]: + table = table.setdefault(part, {}) + table[parts[-1]] = value + origins[key] = "run.json" + return settings.Settings(ctx.project, data, origins) + + +def runs_root_for(adapter) -> Path: + return adapter.spec.runs_root / "adapt" + + +def library_root_for(adapter, k: int, *, seed: int | None, tag: str = "") -> Path: + schedule.validate_tag(tag) variant = adapter.substrate.name if seed is not None: - variant += f"-s{int(seed)}" + variant += f"-s{seed}" if tag: variant += f"-{tag}" return LIB.root_for( @@ -96,57 +227,46 @@ def library_root_for(adapter: BenchmarkAdapter, k: int, *, seed: int | None, tag ) -def summary_namespace(adapter: BenchmarkAdapter, k: int, *, seed: int | None, tag: str = "") -> str: - """The namespace whose task dirs hold summary.json for BOTH modes: run_task keeps the - summary in the parallel mode's namespace (on-disk token "agg") whichever modes ran.""" - return SCHED.ns(adapter.spec.name, adapter.substrate.name, "agg", k, seed=seed, tag=tag) - +def summary_namespace(adapter, k: int, *, seed: int | None, tag: str = "") -> str: + return schedule.ns(adapter.spec.name, adapter.substrate.name, "agg", k, seed=seed, tag=tag) -def add_cell_args(p: argparse._ActionsContainer) -> None: - """The (benchmark, substrate) cell selectors; `p` may be a parser or an argument group.""" - p.add_argument("--bench", required=True, choices=BENCHES, help="jb | wb | lab") - p.add_argument( - "--subset", default=None, choices=WB_SUBSETS, help="WB only: office | web | code" - ) - p.add_argument( - "--substrate", - default="oc", - choices=SUBSTRATES, - help="task-executor substrate: oc (opencode) | cx (codex)", - ) - -def add_modes_arg(p: argparse._ActionsContainer) -> None: - p.add_argument("--mode", "--modes", dest="modes", default=MODES_DEFAULT, help=MODES_HELP) +def parse_modes(raw: str) -> tuple[str, ...]: + tokens = {schedule.mode_token(s.strip()) for s in raw.split(",") if s.strip()} + if not tokens: + raise ConfigError("--mode requires parallel or sequential") + return tuple(t for t in schedule.MODES if t in tokens) -def add_run_identity_args(p: argparse._ActionsContainer) -> None: - """The run-identity knobs that enter the namespace.""" - p.add_argument( - "--k", type=int, default=1, help="scale: R=4k rollouts, P=3k improver sessions per task" - ) - p.add_argument( - "--seed", type=int, default=None, help="stream-order seed; omit for sorted task order" - ) - add_modes_arg(p) +def mode_label(token: str) -> str: + return schedule.MODE_NAMES.get(token, token) -def parse_modes(raw: str) -> tuple[str, ...]: - """'parallel,sequential' (or the on-disk tokens 'agg,seq') -> tokens in MODES order.""" - names = [s.strip() for s in raw.split(",") if s.strip()] - if not names: - raise ValueError( - f"--modes takes a comma list of parallel | sequential (agg | seq), got {raw!r}" - ) - tokens = set() - for name in names: +def summaries(run_dir: Path) -> list[tuple[Path, dict]]: + rows = [] + for path in sorted((run_dir / "adapt").glob("*/*/summary.json")): try: - tokens.add(SCHED.mode_token(name)) - except ValueError as exc: - raise ValueError(f"--modes: {exc} (got {raw!r})") from None - return tuple(t for t in SCHED.MODES if t in tokens) + rows.append((path, json.loads(path.read_text(encoding="utf-8")))) + except (OSError, ValueError) as exc: + raise ConfigError(f"cannot read {path}: {exc}") from exc + return rows -def mode_label(token: str) -> str: - """Human name for an on-disk token ('agg' -> 'parallel'); unknown tokens pass through.""" - return SCHED.MODE_NAMES.get(token, token) +def event_totals(run_dir: Path) -> dict: + records = [] + path = run_dir / "events.jsonl" + if path.is_file(): + try: + records = [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + except (OSError, ValueError) as exc: + raise ConfigError(f"cannot read {path}: {exc}") from exc + calls = [ + r + for r in records + if r["kind"] == "rollout" or (r["kind"] == "session" and not r.get("cached")) + ] + return { + "spent_usd": round(sum(r.get("cost_usd", 0) or 0 for r in records), 6), + "calls": len(calls), + "last_activity": records[-1].get("t") if records else None, + } diff --git a/harness2/cli/apply.py b/harness2/cli/apply.py new file mode 100644 index 00000000..e4bd38e1 --- /dev/null +++ b/harness2/cli/apply.py @@ -0,0 +1,28 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Apply command placeholder.""" + +from __future__ import annotations + +import sys + + +def add_parser(subparsers): + return subparsers.add_parser("apply") + + +def run(args, ctx) -> int: + print("not implemented yet", file=sys.stderr) + return 1 diff --git a/harness2/cli/bridge.py b/harness2/cli/bridge.py index 79124870..ce364f59 100644 --- a/harness2/cli/bridge.py +++ b/harness2/cli/bridge.py @@ -50,23 +50,25 @@ def _endpoint(path: Path, host: str, port: int) -> dict: return endpoint -def main(argv=None): - p = argparse.ArgumentParser(description=__doc__) +def _arguments(p): p.add_argument("--model", required=True) p.add_argument("--host", default="127.0.0.1") p.add_argument("--port", type=int, default=0) - args = p.parse_args(argv) + return p + + +def add_parser(subparsers): + return _arguments(subparsers.add_parser("bridge")) + +def run(args, ctx): litellm = Path(sys.executable).with_name("litellm") if not litellm.is_file(): - p.exit(1, "LiteLLM is not installed; run: uv pip install -e '.[bridge]'\n") - try: - config = settings.load(settings.find_project(Path.cwd())) - provider_name, model = providers.split_model(args.model) - provider = config.provider(provider_name) - params = providers.litellm_params(provider, model) - except ConfigError as exc: - p.exit(1, f"{exc}\n") + raise ConfigError("LiteLLM is not installed; run: uv pip install -e '.[bridge]'") + config = ctx.settings + provider_name, model = providers.split_model(args.model) + provider = config.provider(provider_name) + params = providers.litellm_params(provider, model) root = settings.cache_dir() / "bridge" endpoint = _endpoint(root / "bridge.json", args.host, args.port) @@ -86,19 +88,40 @@ def main(argv=None): encoding="utf-8", ) - print("Provider configuration:") - print( + snippet = ( "[providers.bridge]\n" 'type = "openai"\n' f'base_url = "http://127.0.0.1:{port}/v1"\n' - 'api_key_env = "HARNESS2_BRIDGE_KEY"', + 'api_key_env = "HARNESS2_BRIDGE_KEY"' + ) + ctx.out( + {"provider": snippet, "environment": {"HARNESS2_BRIDGE_KEY": endpoint["key"]}}, + lambda data: ( + "Provider configuration:\n" + + data["provider"] + + "\n\nEnvironment:\nHARNESS2_BRIDGE_KEY=" + + data["environment"]["HARNESS2_BRIDGE_KEY"] + ), ) - print("\nEnvironment:") - print(f"HARNESS2_BRIDGE_KEY={endpoint['key']}", flush=True) + sys.stdout.flush() env = {**os.environ, "HARNESS2_BRIDGE_KEY": endpoint["key"]} command = [str(litellm), "--config", str(litellm_config), "--host", args.host] os.execve(str(litellm), [*command, "--port", str(port)], env) + return 0 + + +def main(argv=None): + from ._common import Context + + p = _arguments(argparse.ArgumentParser()) + args = p.parse_args(argv) + args.project, args.profile, args.config = None, None, [] + args.json, args.quiet, args.verbose = False, False, False + try: + return run(args, Context(args, {})) + except ConfigError as exc: + p.exit(1, f"{exc}\n") if __name__ == "__main__": diff --git a/harness2/cli/chat.py b/harness2/cli/chat.py new file mode 100644 index 00000000..239d2d76 --- /dev/null +++ b/harness2/cli/chat.py @@ -0,0 +1,28 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Chat command placeholder.""" + +from __future__ import annotations + +import sys + + +def add_parser(subparsers): + return subparsers.add_parser("chat") + + +def run(args, ctx) -> int: + print("not implemented yet", file=sys.stderr) + return 1 diff --git a/harness2/cli/check.py b/harness2/cli/check.py deleted file mode 100644 index ca79e5d2..00000000 --- a/harness2/cli/check.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Check one installed benchmark before running adaptation and judging.""" - -from __future__ import annotations - -import argparse - -from harness2_benchmarks.preflight import check - -from harness2.cli._common import make_adapter - - -def main(argv=None): - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--bench", choices=("jb", "lab", "wb"), required=True) - p.add_argument("--subset", choices=("office", "web", "code")) - p.add_argument("--substrate", choices=("oc", "cx"), default="oc") - args = p.parse_args(argv) - try: - adapter = make_adapter(args.bench, args.subset, args.substrate) - errors = check(adapter, judge=True) - except (OSError, RuntimeError, ValueError) as exc: - errors = [str(exc)] - for error in errors: - print(f"FAIL: {error}") - if errors: - print(f"Setup incomplete: {len(errors)} issue(s). See plugins/harness2-benchmarks/docs/BENCHMARKS.md.") - return 1 - print( - f"Ready: {adapter.spec.name}/{args.substrate}; {len(adapter.tasks())} tasks. " - "Model access is not exercised; the first real run is the first model call." - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/harness2/cli/completion.py b/harness2/cli/completion.py new file mode 100644 index 00000000..cd942a9d --- /dev/null +++ b/harness2/cli/completion.py @@ -0,0 +1,92 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generate shell completion from the argparse command tree.""" + +from __future__ import annotations + +import argparse +import shlex + + +def add_parser(subparsers): + p = subparsers.add_parser("completion") + p.add_argument("shell", choices=("bash", "zsh")) + return p + + +def generate(parser, shell: str) -> str: + choices, transitions = {}, [] + + def visit(parser, path=""): + words = { + option + for action in parser._actions + for option in action.option_strings + if option.startswith("--") and action.help != argparse.SUPPRESS + } + for action in parser._actions: + if isinstance(action, argparse._SubParsersAction): + words.update(action.choices) + for name, child in action.choices.items(): + next_path = f"{path} {name}".strip() + transitions.append( + f" {shlex.quote(path + ':' + name)}) command_path={shlex.quote(next_path)} ;;" + ) + visit(child, next_path) + choices[path] = " ".join(sorted(words)) + + visit(parser) + start = ["#compdef harness2"] if shell == "zsh" else [] + start += [ + "_harness2_complete() {", + " local command_path='' word candidates", + " local -a previous", + ] + if shell == "zsh": + start += [' previous=("${words[@]:1:$((CURRENT-2))}")'] + else: + start += [' previous=("${COMP_WORDS[@]:1:$((COMP_CWORD-1))}")'] + start += [ + ' for word in "${previous[@]}"; do', + ' case "$command_path:$word" in', + *transitions, + " esac", + " done", + ' case "$command_path" in', + ] + start += [ + f" {shlex.quote(path)}) candidates={shlex.quote(words)} ;;" + for path, words in choices.items() + ] + start += [" esac"] + if shell == "zsh": + start += [" compadd -- ${=candidates}", "}", "compdef _harness2_complete harness2"] + else: + start += [ + ' COMPREPLY=( $(compgen -W "$candidates" -- "${COMP_WORDS[COMP_CWORD]}") )', + "}", + "complete -F _harness2_complete harness2", + ] + return "\n".join(start) + + +def run(args, ctx) -> int: + from .main import build_parser + + ctx.out( + {"shell": args.shell, "script": generate(build_parser(), args.shell)}, + lambda data: data["script"], + ) + return 0 diff --git a/harness2/cli/config.py b/harness2/cli/config.py new file mode 100644 index 00000000..539fc2da --- /dev/null +++ b/harness2/cli/config.py @@ -0,0 +1,74 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Inspect and edit layered settings.""" + +from __future__ import annotations + +from .. import settings + + +def add_parser(subparsers): + p = subparsers.add_parser("config") + commands = p.add_subparsers(dest="action", required=True) + show = commands.add_parser("show") + show.add_argument("--origin", action="store_true", help="Include setting origins.") + get = commands.add_parser("get") + get.add_argument("key") + set_parser = commands.add_parser("set") + set_parser.add_argument("key") + set_parser.add_argument("value") + set_parser.add_argument( + "--global", dest="global_", action="store_true", help="Edit global settings." + ) + commands.add_parser("path") + return p + + +def run(args, ctx) -> int: + config = ctx.settings + if args.action == "show": + values = config.flat() + if args.origin: + values = { + key: {"value": value, "origin": config.origin(key)} for key, value in values.items() + } + ctx.out( + values, + lambda data: "\n".join(f"{key} = {value!r}" for key, value in sorted(data.items())), + ) + elif args.action == "get": + value = config.get(args.key) + ctx.out(value, str) + elif args.action == "set": + path = settings.global_path() if args.global_ else settings.project_path(ctx.project) + value = settings.parse_value(args.value) + settings.load( + ctx.project_or_none, + profile=ctx.args.profile, + overrides={**ctx.overrides, args.key: (value, "config set")}, + ) + settings.set_value(path, args.key, value) + ctx.out({"path": path, "key": args.key, "value": value}, lambda data: str(data["path"])) + else: + ctx.out( + { + "global": settings.global_path(), + "project": settings.project_path(ctx.project_or_none) + if ctx.project_or_none + else None, + }, + lambda data: "\n".join(f"{key}: {value}" for key, value in data.items()), + ) + return 0 diff --git a/harness2/cli/demo.py b/harness2/cli/demo.py index 2d957dba..e1a6fed9 100644 --- a/harness2/cli/demo.py +++ b/harness2/cli/demo.py @@ -16,7 +16,6 @@ from __future__ import annotations -import argparse import json from pathlib import Path @@ -24,35 +23,52 @@ from ..demo import DemoAdapter, DemoProposer -def main(argv=None) -> int: - parser = argparse.ArgumentParser(description=__doc__) +def add_parser(subparsers): + parser = subparsers.add_parser("demo") parser.add_argument("--output", type=Path, default=Path("runs/demo")) parser.add_argument("--k", type=int, default=2) parser.add_argument("--mode", choices=("parallel", "sequential", "both"), default="both") - args = parser.parse_args(argv) + return parser + + +def run(args, ctx) -> int: if args.k < 1: - parser.error("--k must be positive") + raise ValueError("--k must be positive") root = args.output.expanduser().resolve() adapter = DemoAdapter(root) proposer = DemoProposer(adapter.spec.channels["proposer"]) modes = ("agg", "seq") if args.mode == "both" else (mode_token(args.mode),) - result = run_domain( - "accounting", - adapter.tasks(), - args.k, - adapter=adapter, - prop=proposer, - runs_root=adapter.spec.runs_root, - library_root=root / "library" / f"k{args.k}", - modes=modes, - ) + with ctx.progress(): + result = run_domain( + "accounting", + adapter.tasks(), + args.k, + adapter=adapter, + prop=proposer, + runs_root=adapter.spec.runs_root, + library_root=root / "library" / f"k{args.k}", + modes=modes, + ) result["execution_count"] = adapter.execution_count result["editor_calls"] = len(proposer._calls) report = root / "demo.json" report.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") - print(f"Local demo: {result['done']} tasks completed; {result['failed']} failed. {report}") + ctx.out( + result, + lambda data: ( + f"Local demo: {data['done']} tasks completed; {data['failed']} failed. {report}" + ), + ) return int(result["failed"] > 0) +def main(argv=None) -> int: + import sys + + from .main import main as cli_main + + return cli_main(["demo", *(sys.argv[1:] if argv is None else argv)]) + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/harness2/cli/diff.py b/harness2/cli/diff.py new file mode 100644 index 00000000..bc3d4738 --- /dev/null +++ b/harness2/cli/diff.py @@ -0,0 +1,50 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compare the stored base harness with a selection.""" + +from __future__ import annotations + +from ..core.tree import diff_summary +from . import _adopt + + +def add_parser(subparsers): + p = subparsers.add_parser("diff") + p.add_argument("name", nargs="?") + p.add_argument("--task", help="Task ID.") + p.add_argument( + "--mode", choices=("parallel", "sequential"), default="parallel", help="Recursion mode." + ) + p.add_argument("--stat", action="store_true", help="Print change counts.") + return p + + +def run(args, ctx) -> int: + with ctx.progress(): + adapter, root = _adopt.open_run(ctx, args.name) + task, after = _adopt.adopted_tree(adapter, root, args.task, args.mode) + before = _adopt.base_tree(adapter) + data = ( + diff_summary(adapter.substrate.surface(), before, after) + if args.stat + else { + "task": task, + "diff": "".join( + _adopt.unified(rel, before, after) for rel, _ in _adopt.changes(before, after) + ), + } + ) + ctx.out(data, str if args.stat else lambda row: row["diff"].rstrip("\n")) + return 0 diff --git a/harness2/cli/doctor.py b/harness2/cli/doctor.py new file mode 100644 index 00000000..88abea01 --- /dev/null +++ b/harness2/cli/doctor.py @@ -0,0 +1,121 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Check suite configuration and model clients.""" + +from __future__ import annotations + +import subprocess + +from .. import providers, registry +from ..core.proposer import Proposer +from ..core.spec import ChannelSpec +from ..substrate._native import cli_argv +from . import _common as C + + +def add_parser(subparsers): + p = subparsers.add_parser("doctor") + p.add_argument("--live", action="store_true", help="Call each model; costs money.") + return p + + +def run(args, ctx) -> int: + config = ctx.settings + rows = [] + + def check(name, operation): + try: + detail = operation() + rows.append({"check": name, "ok": True, "detail": detail or "ok"}) + except (OSError, RuntimeError, ValueError, subprocess.SubprocessError) as exc: + rows.append({"check": name, "ok": False, "detail": str(exc)}) + + def suite_check(): + suite = registry.load(config.get("suite.name")) + suite_ctx = registry.SuiteContext( + config, ctx.project, C.run_path(ctx, C.default_name(config)) + ) + with ctx.progress(): + problems = suite.check(suite_ctx) if hasattr(suite, "check") else [] + if problems: + raise RuntimeError("; ".join(problems)) + + check("suite", suite_check) + roles = [config.role(name) for name in ("improver", "selector", "reflector", "judge")] + roles.insert(0, {**config.section("executor"), "channel": config.get("executor.name")}) + binaries, models, referenced = set(), {}, set() + for role in roles: + channel = role["channel"] + if channel == "human": + continue + if channel != "command": + binaries.add((role["bin"], channel)) + model = role["model"] + if model: + + def resolve(model=model, channel=channel, role=role): + provider_name, _ = providers.split_model(model) + provider = config.provider(provider_name) + referenced.add(provider_name) + if channel != "command": + models.setdefault( + (channel, model), + ChannelSpec( + channel, + model, + role["effort"], + role.get("steps", 120), + role["timeout"], + role["bin"], + provider, + ), + ) + + check(f"model {channel}/{model}", resolve) + elif channel != "command": + rows.append({"check": f"model {channel}", "ok": False, "detail": "model is empty"}) + for binary, channel in sorted(binaries): + + def binary_check(binary=binary, channel=channel): + argv = cli_argv(binary, channel) + result = subprocess.run( + [*argv, "--version"], check=True, capture_output=True, text=True, timeout=10 + ) + return (result.stdout or result.stderr).strip() + + check(f"binary {binary or channel}", binary_check) + for name in sorted(referenced): + problems = providers.check(config.provider(name)) + rows.append( + {"check": f"provider {name}", "ok": not problems, "detail": "; ".join(problems) or "ok"} + ) + if args.live: + for (channel, model), spec in sorted(models.items()): + + def live_check(spec=spec): + with ctx.progress(): + reply = Proposer(spec).call('Reply "ok".', None, "doctor.live") + if not reply.ok or reply.text.strip().lower() != "ok": + raise RuntimeError(reply.error or "expected reply ok") + return reply.text + + check(f"live {channel}/{model}", live_check) + ctx.out( + rows, + lambda data: "\n".join( + f"{'ok' if row['ok'] else 'FAIL'} {row['check']}: {row['detail']}" for row in data + ), + ) + return int(any(not row["ok"] for row in rows)) diff --git a/harness2/cli/human.py b/harness2/cli/human.py new file mode 100644 index 00000000..dcd7ea03 --- /dev/null +++ b/harness2/cli/human.py @@ -0,0 +1,23 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Human selector placeholder.""" + +from __future__ import annotations + +from ..core.proposer import Proposer + + +class HumanSelector(Proposer): + pass diff --git a/harness2/cli/init.py b/harness2/cli/init.py new file mode 100644 index 00000000..2acb46c0 --- /dev/null +++ b/harness2/cli/init.py @@ -0,0 +1,28 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Init command placeholder.""" + +from __future__ import annotations + +import sys + + +def add_parser(subparsers): + return subparsers.add_parser("init") + + +def run(args, ctx) -> int: + print("not implemented yet", file=sys.stderr) + return 1 diff --git a/harness2/cli/judge.py b/harness2/cli/judge.py index 6e8be6f0..5074ddac 100644 --- a/harness2/cli/judge.py +++ b/harness2/cli/judge.py @@ -12,32 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Judge the SELECTED run of each finished task in one namespace, and cache the rows. - -Only the run the blind terminal selector chose is scored. A judge failure is recorded as -ungraded (score=None), never as a zero. Re-running re-judges every selection, but -adapters serve cached judge details and a failed re-judge never replaces an existing -graded row. - - python3 -m harness2.cli.judge --bench jb --substrate oc --k 2 --modes parallel - -With --bases it scores this namespace's reference executions instead, so the initial -harness can be reported beside the adapted one. - -Output: //judged.json (judge.write format), consumed by cli.report; -or base_judged.json under --bases, one row per (task, base). -""" +"""Judge finished selections using the saved run settings.""" from __future__ import annotations -import argparse import json -import sys from pathlib import Path -from .. import config from ..core import judge as J +from ..core.runmark import resolve_path +from ..settings import ConfigError from . import _common as C +from ._adopt import open_run _DERIVED = ("frac", "graded") @@ -54,8 +40,8 @@ def _load_rows(path: Path) -> list[J.Scored]: except (OSError, json.JSONDecodeError) as exc: raise ValueError(f"unreadable judged cache {path}: {exc}; move it aside to rebuild") rows = [] - for row in raw: - row = {k: v for k, v in row.items() if k not in _DERIVED} + for row in raw.get("rows", []) if isinstance(raw, dict) else raw: + row = {k: v for k, v in row.items() if k not in (*_DERIVED, "judge_model")} if "mode" not in row and "arm" in row: row["mode"] = row.pop("arm") rows.append(J.Scored(**row)) @@ -96,8 +82,7 @@ def collect_selections(ns_dir: Path, modes: tuple[str, ...]) -> list[dict]: try: summary = json.loads(summary_path.read_text()) except (OSError, json.JSONDecodeError) as exc: - print(f" ! skipping unreadable {summary_path}: {exc}", flush=True) - continue + raise ConfigError(f"cannot read {summary_path}: {exc}") from exc for mode in modes: sel = (summary.get("selections") or {}).get(mode) if sel: @@ -105,100 +90,94 @@ def collect_selections(ns_dir: Path, modes: tuple[str, ...]) -> list[dict]: return out -def judge_bases(adapter, ns_dir: Path) -> int: - """Score every reference execution recorded in this namespace's task summaries.""" - rows, legacy = [], 0 +def add_parser(subparsers): + p = subparsers.add_parser("judge") + p.add_argument("name", nargs="?") + p.add_argument("--bases", action="store_true", help="Judge reference executions.") + return p + + +def _judge_model(path, model): + if not path.is_file(): + return + try: + data = json.loads(path.read_text()) + except (OSError, ValueError) as exc: + raise ConfigError(f"cannot read {path}: {exc}") from exc + old = data.get("judge_model") if isinstance(data, dict) else None + if old is not None and old != model: + raise ConfigError(f"judge model differs: {old!r} -> {model!r}; delete {path} to re-judge") + + +def judge_bases(adapter, ns_dir: Path, *, log, allowed) -> list[dict]: + rows = [] for summary_path in sorted(ns_dir.glob("*/summary.json")): - try: - summary = json.loads(summary_path.read_text()) - except (OSError, json.JSONDecodeError) as exc: - print(f" ! skipping unreadable {summary_path}: {exc}", flush=True) + summary = json.loads(summary_path.read_text()) + task = summary["task_id"] + if not allowed(task): continue - task_id = summary.get("task_id", "") for entry in summary.get("base") or []: if not isinstance(entry, dict) or not entry.get("run_dir"): - legacy += 1 + log(f"{task}: reference execution has no run directory") continue - label, run_dir = entry.get("label", ""), Path(entry["run_dir"]) - try: - got = adapter.score(run_dir, task_id) if run_dir.is_dir() else None - except Exception as exc: - got = None - print(f" {task_id[:52]:<52} {label} ERROR {type(exc).__name__}: {exc}", flush=True) - else: - shown = "-" if got is None else f"{got.get('n_passed')}/{got.get('n_criteria')}" - print(f" {task_id[:52]:<52} {label} {shown}", flush=True) - rows.append( - {"task": task_id, "label": label, "run_id": entry.get("run_id", ""), "score": got} + run_dir = resolve_path(entry["run_dir"], adapter.spec.runs_root) + row = J.judge_one( + adapter, + task, + "base", + summary["k"], + entry.get("label", ""), + entry.get("run_id", ""), + run_dir, + chosen_is_base=True, ) - if legacy: - print( - f" {legacy} reference execution(s) predate run-directory recording and cannot be " - f"scored; re-run those tasks to include them.", - flush=True, - ) - if not rows: - print(f"no reference executions recorded under {ns_dir}; wrote nothing", flush=True) - return 2 - out = ns_dir / "base_judged.json" - out.write_text(json.dumps(rows, indent=1, default=str), encoding="utf-8") - graded = sum(1 for r in rows if r["score"] is not None) - print(f"wrote {out}: {len(rows)} row(s), {graded} graded") - return 0 if graded == len(rows) else 2 - - -def main(argv: list[str] | None = None) -> int: - p = argparse.ArgumentParser( - prog="harness2.cli.judge", description=__doc__, formatter_class=C.HelpFormatter - ) - C.add_cell_args(p.add_argument_group("cell")) - C.add_run_identity_args(p.add_argument_group("run identity (names the sweep)")) - p.add_argument( - "--bases", - action="store_true", - help="score this namespace's reference executions into base_judged.json " - "instead of the selected runs", + rows.append(row.__dict__ | {"graded": row.graded, "frac": row.frac}) + return rows + + +def run(args, ctx) -> int: + with ctx.progress(): + adapter, root = open_run(ctx, args.name) + record = C.load_run(root) + config = C.run_settings(ctx, record) + model = config.role("judge")["model"] + plan = record["schedule"] + ns_dir = C.runs_root_for(adapter) / C.summary_namespace( + adapter, plan["k"], seed=plan.get("seed") ) - args = p.parse_args(argv) - modes = C.parse_modes(args.modes) - - config.validate_models(judge=True) - adapter = C.make_adapter(args.bench, args.subset, args.substrate) - ns = C.summary_namespace(adapter, args.k, seed=args.seed) - ns_dir = C.runs_root_for(adapter) / ns - identity = ns_dir / "config.json" - if identity.is_file() and json.loads(identity.read_text()).get("judge") != config.JUDGE_MODEL: - raise SystemExit("Judge model changed for this experiment; choose a new HARNESS2_RUNS_DIR") - if not ns_dir.is_dir(): - raise SystemExit( - f"nothing to judge: {ns_dir} does not exist — has the sweep `{ns}` run on this machine?" - ) + path = ns_dir / ("base_judged.json" if args.bases else "judged.json") + _judge_model(path, model) + skipped = set() + + def allowed(task): + if not hasattr(adapter, "has_judge") or adapter.has_judge(task): + return True + if task not in skipped: + ctx.log(f"{task}: no judge; skipped") + skipped.add(task) + return False + if args.bases: - return judge_bases(adapter, ns_dir) - - sels = collect_selections(ns_dir, modes) - if not sels: - raise SystemExit( - f"nothing to judge: no selections for modes " - f"{[C.mode_label(m) for m in modes]} under {ns_dir} (tasks may still " - f"be running, or the wrong --k/--seed was given)" - ) - - print(f"judging {len(sels)} selection(s) under {ns} ...", flush=True) - rows = J.judge_selections(adapter, sels, parallel=_PARALLEL) - - out_path = ns_dir / "judged.json" - merged = _merge(_load_rows(out_path), rows) - J.write(merged, out_path) - - graded = sum(1 for r in merged if r.graded) - print( - f"judged.json: {len(merged)} row(s), {graded} graded, " - f"{len(merged) - graded} ungraded (judge failure or no pick — NOT zeros)" + with ctx.progress(): + rows = judge_bases(adapter, ns_dir, log=ctx.log, allowed=allowed) + else: + selections = [ + s + for s in collect_selections(ns_dir, C.parse_modes(plan["modes"])) + if allowed(s["task_id"]) + ] + with ctx.progress(): + new = J.judge_selections(adapter, selections, parallel=_PARALLEL) + merged = _merge([row for row in _load_rows(path) if allowed(row.task_id)], new) + rows = [row.__dict__ | {"graded": row.graded, "frac": row.frac} for row in merged] + if not rows: + ctx.out([], lambda _: "no tasks to judge") + return 0 + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps({"judge_model": model, "rows": rows}, indent=2) + "\n", encoding="utf-8" ) - print(f"wrote {out_path}") - return 0 if rows and all(r.graded for r in rows) else 2 - - -if __name__ == "__main__": - sys.exit(main()) + ctx.out( + rows, lambda data: f"{path}: {len(data)} rows, {sum(row['graded'] for row in data)} graded" + ) + return 0 if all(row["graded"] for row in rows) else 1 diff --git a/harness2/cli/main.py b/harness2/cli/main.py new file mode 100644 index 00000000..a7cb8641 --- /dev/null +++ b/harness2/cli/main.py @@ -0,0 +1,102 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The harness2 command dispatcher.""" + +from __future__ import annotations + +import argparse +import importlib +import sys +from importlib.metadata import version + +from ._common import Context + +COMMANDS = ( + "run", + "doctor", + "setup", + "judge", + "report", + "runs", + "diff", + "task", + "config", + "demo", + "bridge", + "completion", + "init", + "status", + "chat", + "apply", +) +ALIASES = {"harness2-" + name: name for name in ("judge", "report", "setup", "demo", "bridge")} +ALIASES["harness2-check"] = "doctor" + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="harness2") + parser.add_argument("-C", "--project", metavar="DIR", help="Project directory.") + parser.add_argument("--profile", metavar="NAME", help="Settings profile.") + parser.add_argument( + "-c", + "--config", + action="append", + default=[], + metavar="KEY=VALUE", + help="Override a setting.", + ) + parser.add_argument("-q", "--quiet", action="store_true", help="Silence logs.") + parser.add_argument("-v", "--verbose", action="store_true", help="Show debug logs.") + parser.add_argument("--json", action="store_true", help="Print JSON.") + parser.add_argument("--version", action="version", version=version("harness2")) + subparsers = parser.add_subparsers(dest="command", required=True) + for name in COMMANDS: + module = importlib.import_module(f"harness2.cli.{name}") + module.add_parser(subparsers) + subparsers.choices[name].set_defaults(_module=module) + return parser + + +def main(argv: list[str] | None = None) -> int: + argv = list(sys.argv[1:] if argv is None else argv) + alias = ALIASES.get(sys.argv[0].rsplit("/", 1)[-1]) + if alias: + # Aliases still accept global options before their subcommand's arguments. + index = 0 + while index < len(argv): + flag = argv[index].split("=", 1)[0] + if flag in ("-C", "--project", "--profile", "-c", "--config"): + index += 1 if "=" in argv[index] else 2 + elif argv[index].startswith(("-C", "-c")): + index += 1 + elif flag in ("-q", "--quiet", "-v", "--verbose", "--json", "--version"): + index += 1 + else: + break + argv.insert(index, alias) + args = build_parser().parse_args(argv) + ctx = Context(args, getattr(args._module, "FLAGS", {})) + try: + ctx.debug(f"command: {args.command}") + return args._module.run(args, ctx) + except KeyboardInterrupt: + return 130 + except (OSError, RuntimeError, ValueError, KeyError) as exc: + ctx.log(f"error: {exc}") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/harness2/cli/report.py b/harness2/cli/report.py index 8f80fc56..1d725d06 100644 --- a/harness2/cli/report.py +++ b/harness2/cli/report.py @@ -12,24 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Render the score report for one cell from cached judged.json rows — read-only. - -The figure each benchmark reports per (mode, k) — partial credit and all-criteria pass on LAB, -one score elsewhere — with pick provenance, the k=1 noise-floor note and the sanity warnings; -fell-back counts and improver spend come from the task summaries. - - python3 -m harness2.cli.report --bench jb --substrate oc --k 1,2 -""" +"""Render cached scores for one or more runs.""" from __future__ import annotations -import argparse import json -import sys from pathlib import Path from ..core import report as R from ..core import schedule as SCHED +from ..settings import ConfigError from . import _common as C from .judge import _load_rows @@ -42,8 +34,7 @@ def _spend_and_fallbacks(ns_dir: Path, modes: tuple[str, ...], k: int) -> tuple[ try: summary = json.loads(summary_path.read_text()) except (OSError, json.JSONDecodeError) as exc: - print(f" ! skipping unreadable {summary_path}: {exc}", flush=True) - continue + raise ConfigError(f"cannot read {summary_path}: {exc}") from exc records = SCHED._modes_of(summary) for mode in modes: key = (mode, k) @@ -55,60 +46,41 @@ def _spend_and_fallbacks(ns_dir: Path, modes: tuple[str, ...], k: int) -> tuple[ return fell_back, usd -def main(argv: list[str] | None = None) -> int: - p = argparse.ArgumentParser( - prog="harness2.cli.report", description=__doc__, formatter_class=C.HelpFormatter - ) - C.add_cell_args(p.add_argument_group("cell")) - g = p.add_argument_group("run identity (names the sweeps)") - g.add_argument("--k", default="1", help="comma list of scales to report, e.g. 1,2,3") - g.add_argument( - "--seed", - type=int, - default=None, - help="stream-order seed of the sweep; omit for the seedless namespace", - ) - C.add_modes_arg(g) - args = p.parse_args(argv) - modes = C.parse_modes(args.modes) - try: - ks = sorted({int(s) for s in args.k.split(",") if s.strip()}) - except ValueError: - raise SystemExit(f"--k takes a comma list of ints, got {args.k!r}") - if not ks: - raise SystemExit("--k named no scale") +def add_parser(subparsers): + p = subparsers.add_parser("report") + p.add_argument("names", nargs="*") + return p - adapter = C.make_adapter(args.bench, args.subset, args.substrate) - rows, fell_back, usd = [], {}, {} - for k in ks: - ns_dir = C.runs_root_for(adapter) / C.summary_namespace(adapter, k, seed=args.seed) - judged = ns_dir / "judged.json" - got = _load_rows(judged) - if not got: - print(f" ! no judged rows at {judged} — run cli.judge for k={k} first", flush=True) - continue - rows.extend(r for r in got if r.mode in modes) - fb, sp = _spend_and_fallbacks(ns_dir, modes, k) - fell_back.update(fb) - usd.update(sp) - if not rows: - raise SystemExit("nothing to report") - title = ( - f"{adapter.spec.name}-{adapter.substrate.name}" - + (f" seed={args.seed}" if args.seed is not None else "") - + f" modes={','.join(C.mode_label(m) for m in modes)}" - ) - summaries = R.summarise(rows, fell_back=fell_back, usd=usd) - print(R.render(summaries, title=title, bench=adapter.spec.name)) - spend = sum(s.usd for s in summaries) - if spend: - print( - f"\n improver (editors, composer, selection, reflection) spend across reported modes: " - f"${spend:.2f}" +def run(args, ctx) -> int: + reports, rows = [], [] + for name in args.names or [None]: + root = C.resolve_run(ctx, name) + record = C.load_run(root) + scored, fallbacks, spend = [], {}, {} + for path in sorted((root / "adapt").glob("*/judged.json")): + got = _load_rows(path) + scored.extend(got) + for k in {row.k for row in got}: + fb, usd = _spend_and_fallbacks( + path.parent, tuple({row.mode for row in got if row.k == k}), k + ) + for key, value in fb.items(): + fallbacks[key] = fallbacks.get(key, 0) + value + for key, value in usd.items(): + spend[key] = spend.get(key, 0) + value + rows.extend( + {"name": root.name, **row.__dict__, "frac": row.frac, "graded": row.graded} + for row in scored + ) + reports.append( + R.render( + R.summarise(scored, fell_back=fallbacks, usd=spend), + title=root.name, + bench=record["suite"], + ) ) + if not rows: + raise ConfigError("nothing to report") + ctx.out(rows, lambda _: "\n\n".join(reports)) return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/harness2/cli/run.py b/harness2/cli/run.py new file mode 100644 index 00000000..ce19ae1a --- /dev/null +++ b/harness2/cli/run.py @@ -0,0 +1,257 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Configure and execute a resumable task stream.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from .. import core, registry +from ..core import events, ledger, schedule +from ..core.proposer import Proposer +from ..settings import ConfigError +from . import _common as C + +FLAGS = { + "suite": "suite.name", + "executor": "executor.name", + "k": "run.k", + "modes": "run.modes", + "seed": "run.seed", + "workers": "run.workers", + "budget_usd": "run.budget_usd", + "max_calls": "run.max_calls", + "selector": "selector.channel", + "steer": "improver.steer", +} + + +def add_parser(subparsers): + p = subparsers.add_parser("run") + p.add_argument("--suite", help="Suite name.") + p.add_argument("--bench", dest="suite", help=argparse.SUPPRESS) + p.add_argument("--executor", help="Executor name.") + p.add_argument("--name", help="Run name.") + p.add_argument("--k", type=int, help="Recursion scale.") + p.add_argument("--mode", "--modes", dest="modes", help="Comma-separated recursion modes.") + p.add_argument("--seed", type=int, help="Task ordering seed.") + p.add_argument("--split", choices=("all", "train", "test"), default="all", help="Task split.") + p.add_argument("--split-file", type=Path, help="Task split file.") + p.add_argument("--only-domains", default="", help="Comma-separated domains.") + p.add_argument("--only-tasks", default="", help="Comma-separated task IDs.") + p.add_argument("--max-tasks", type=int, help="Tasks per domain.") + p.add_argument("--workers", type=int, help="Concurrent domains.") + p.add_argument("--budget-usd", type=float, help="Spending limit in USD.") + p.add_argument("--max-calls", type=int, help="Model call limit.") + p.add_argument("--selector", choices=("human",), help="Interactive selector.") + p.add_argument("--steer", metavar="TEXT|@FILE", help="Improver guidance.") + p.add_argument("--dry-run", action="store_true", help="Preview tasks and call estimates.") + return p + + +def _csv(raw: str) -> tuple[str, ...]: + return tuple(s.strip() for s in raw.split(",") if s.strip()) + + +def _estimate(tasks: int, k: int, modes: tuple[str, ...], totals: dict) -> dict: + per_mode = { + "rollouts": ledger.expected_rollouts(k), + "improver": ledger.expected_sessions(k), + "selector": int(k > 1), + "reflector": 1, + } + # Reference rollouts are shared by modes; reflection also runs at k=1. + rollouts = tasks * (k + (ledger.expected_rollouts(k) - k) * len(modes)) + sessions = ( + tasks * len(modes) * sum(per_mode[key] for key in ("improver", "selector", "reflector")) + ) + return { + "per_task_per_mode": per_mode, + "rollouts": rollouts, + "sessions": sessions, + "calls": rollouts + sessions, + "average_cost_per_call": totals["spent_usd"] / totals["calls"] if totals["calls"] else None, + } + + +def _print_stream(data) -> str: + lines = [ + f"# {data['suite']}-{data['substrate']} split={data['split']} " + f"seed={data['seed']} k={data['k']} modes={','.join(data['modes'])}" + ] + lines.extend(f"# namespace[{mode}]: {ns}" for mode, ns in data["namespaces"].items()) + lines.extend( + ( + f"# summaries under: {data['summaries']}", + f"# library root: {data['library']}", + f"# domains parallel: {data['workers']}", + ) + ) + lines.extend( + f"{i:>4} {domain:<40} {task}" for i, (domain, task) in enumerate(data["stream"], 1) + ) + lines.append("# domains of this split, with task counts (the legal --only-domains names):") + lines.extend(f"{len(tasks):>4} {domain}" for domain, tasks in data["domains"].items()) + cap = f", max {data['max_tasks']} per domain" if data["max_tasks"] is not None else "" + lines.append( + f"# {len(data['stream'])} tasks over {len(data['domains'])} domains " + f"(domains parallel, tasks within a domain sequential{cap})" + ) + lines.append(f"# call estimate: {data['estimate']}") + return "\n".join(lines) + + +def run(args, ctx) -> int: + events.start(None) + status, reason = "error", "" + try: + project, config = ctx.project, ctx.settings + k, seed = config.get("run.k"), config.get("run.seed") + modes = C.parse_modes(config.get("run.modes")) + name = args.name or C.default_name(config) + root = C.run_path(ctx, name) + suite = registry.load(config.get("suite.name")) + split_file = (project / args.split_file).resolve() if args.split_file else None + suite_ctx = registry.SuiteContext(config, project, root, split_file) + with ctx.progress(): + adapter = suite.make_adapter(suite_ctx) + problems = suite.check(suite_ctx) if hasattr(suite, "check") else [] + if problems: + raise ConfigError("\n".join(problems)) + filters = dict( + split=args.split, + seed=seed, + only_domains=_csv(args.only_domains), + only_tasks=_csv(args.only_tasks), + max_tasks=args.max_tasks, + ) + domains = schedule.task_stream(adapter, **filters) + stream = [(domain, task) for domain, tasks in domains.items() for task in tasks] + if not stream: + raise ConfigError("zero tasks scheduled") + if args.dry_run: + ctx.out( + { + "name": name, + "suite": adapter.spec.name, + "substrate": adapter.substrate.name, + "split": args.split, + "seed": seed, + "max_tasks": args.max_tasks, + "k": k, + "modes": [C.mode_label(m) for m in modes], + "namespaces": { + C.mode_label(m): schedule.ns( + adapter.spec.name, adapter.substrate.name, m, k, seed=seed + ) + for m in modes + }, + "summaries": C.runs_root_for(adapter) + / C.summary_namespace(adapter, k, seed=seed), + "library": C.library_root_for(adapter, k, seed=seed), + "workers": config.get("run.workers"), + "domains": domains, + "stream": stream, + "estimate": _estimate(len(stream), k, modes, C.event_totals(root)), + }, + _print_stream, + ) + status = "ok" + return 0 + C.save_run( + root, + config, + stream, + { + "k": k, + "modes": config.get("run.modes"), + **filters, + "split_file": str(split_file) if split_file else None, + }, + ) + core.configure( + **{ + key: config.get(f"run.{key}") + for key in ("phase_width", "transport_attempts", "proposer_concurrency") + } + ) + events.start( + root / "events.jsonl", + budget_usd=config.get("run.budget_usd"), + max_calls=config.get("run.max_calls"), + ) + audit = root / "adapt/_audit" / C.summary_namespace(adapter, k, seed=seed) + prop = Proposer(adapter.spec.channels["proposer"], audit_dir=audit) + selector_spec = adapter.spec.channels["selector"] + if selector_spec.kind == "human": + from .human import HumanSelector + + selector = HumanSelector(selector_spec, audit_dir=audit) + else: + selector = Proposer(selector_spec, audit_dir=audit) + reflector = Proposer( + adapter.spec.channels.get( + "reflector", + adapter.spec.channels["proposer"] + if selector_spec.kind == "human" + else selector_spec, + ), + audit_dir=audit, + ) + events.emit("run_start", name=name) + with ctx.progress(): + result = schedule.run_sweep( + k, + adapter=adapter, + prop=prop, + selector_prop=selector, + reflector_prop=reflector, + runs_root=C.runs_root_for(adapter), + library_root=C.library_root_for(adapter, k, seed=seed), + modes=modes, + domain_parallel=config.get("run.workers"), + **filters, + ) + failed = sum(int(row.get("failed", 0)) for row in result["domains"]) + errors = [row["error"] for row in result["domains"] if row.get("error")] + status, reason = ( + ("error", "; ".join(errors)) + if errors + else (("failed", "failed tasks") if failed else ("ok", "")) + ) + ctx.out( + {"result": result, "totals": events.totals()}, + lambda data: f"{name}: {status}; {data['totals']}", + ) + return 1 if errors else 3 if failed else 0 + except events.Stopped as exc: + reason = str(exc) + status = "interrupted" if reason == "interrupted" else "stopped" + ctx.out(events.totals(), str) + if status == "interrupted": + ctx.log("run the same command to resume") + return 130 if status == "interrupted" else 4 + except KeyboardInterrupt: + status, reason = "interrupted", "interrupted" + events.request_stop(reason) + ctx.out(events.totals(), str) + ctx.log("run the same command to resume") + return 130 + except Exception as exc: + reason = str(exc) + raise + finally: + events.emit("run_end", status=status, reason=reason) diff --git a/harness2/cli/run_sweep.py b/harness2/cli/run_sweep.py deleted file mode 100644 index 70387e3f..00000000 --- a/harness2/cli/run_sweep.py +++ /dev/null @@ -1,239 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Run one adaptation sweep: every task of a split, domains parallel, tasks in stream order. - -Thin argparse over core.schedule.run_sweep. Costs money when run for real: each task is -R=4k task-executor rollouts and P=3k improver sessions. `--dry-run` prints the exact -(domain, task) stream and namespaces without creating a Proposer, touching a substrate or -validating a model setting. - -A real run checks the judge before the first rollout, because the namespace records the -judge model and `harness2-judge` refuses to grade it under a different one: discovering a -missing judge after the sweep means paying for the sweep twice. The check is offline — a -judge model must be named and a benchmark that grades in its own environment must have it -installed — so it catches an unset or unbuilt judge, not one an account cannot serve. - - python3 -m harness2.cli.run_sweep --bench jb --substrate oc --k 2 --split all - python3 -m harness2.cli.run_sweep --bench wb --subset web --substrate cx \\ - --k 1 --split all --seed 7 --modes parallel --max-tasks 2 --dry-run -""" - -from __future__ import annotations - -import argparse -import json -import sys -from datetime import datetime, timezone -from pathlib import Path - -from .. import config -from ..core import schedule -from ..core.proposer import Proposer -from . import _common as C - - -def _build_parser() -> argparse.ArgumentParser: - p = argparse.ArgumentParser( - prog="harness2.cli.run_sweep", description=__doc__, formatter_class=C.HelpFormatter - ) - C.add_cell_args(p.add_argument_group("cell")) - C.add_run_identity_args(p.add_argument_group("run identity (enters the namespace)")) - g = p.add_argument_group("run control") - g.add_argument("--split", choices=("all", "train", "test"), default="all") - g.add_argument("--split-file", type=Path, help="optional partition from plugins/harness2-benchmarks/scripts/split_tasks.py") - g.add_argument("--workers", type=int, default=1, help="domains to run concurrently") - g.add_argument( - "--only-domains", default="", help="comma list restricting the sweep to these domains" - ) - g.add_argument( - "--max-tasks", - type=int, - default=None, - help="per-domain cap on the stream (the first N tasks of every domain); " - "drop it for the full run", - ) - g.add_argument( - "--dry-run", - action="store_true", - help="print the stream order, the namespaces and the split's domains, then " - "exit — no rollouts, no sessions, nothing written", - ) - return p - - -def _csv(raw: str) -> tuple[str, ...]: - return tuple(s.strip() for s in raw.split(",") if s.strip()) - - -def _stream_for(adapter, args) -> tuple[dict[str, list[str]], list[tuple[str, str]]]: - by_domain = schedule.task_stream( - adapter, - split=args.split, - seed=args.seed, - only_domains=_csv(args.only_domains), - max_tasks=args.max_tasks, - ) - return by_domain, [(domain, task) for domain, tasks in by_domain.items() for task in tasks] - - -def _print_stream(adapter, args, modes) -> int: - by_domain, stream = _stream_for(adapter, args) - print( - f"# {adapter.spec.name}-{adapter.substrate.name} split={args.split} " - f"seed={args.seed} k={args.k} modes={','.join(C.mode_label(m) for m in modes)}" - ) - for m in modes: - namespace = schedule.ns( - adapter.spec.name, adapter.substrate.name, m, args.k, seed=args.seed - ) - print(f"# namespace[{C.mode_label(m)}]: {namespace}") - print( - f"# summaries under: {C.runs_root_for(adapter)}/" - f"{C.summary_namespace(adapter, args.k, seed=args.seed)}" - ) - print(f"# library root: {C.library_root_for(adapter, args.k, seed=args.seed)}") - print(f"# domains parallel: {args.workers}") - for i, (domain, task) in enumerate(stream, 1): - print(f"{i:>4} {domain:<40} {task}") - print("# domains of this split, with task counts (the legal --only-domains names):") - for d, tasks in by_domain.items(): - print(f"{len(tasks):>4} {d}") - cap = f", max {args.max_tasks} per domain" if args.max_tasks is not None else "" - print( - f"# {len(stream)} tasks over {len(by_domain)} domains " - f"(domains parallel, tasks within a domain sequential{cap})" - ) - return 0 if stream else 1 - - -def main(argv: list[str] | None = None) -> int: - """Parse, then run `_main`, printing a misconfiguration as one remediation line.""" - args = _build_parser().parse_args(argv) - try: - return _main(args) - except (config.ConfigError, RuntimeError, ValueError, FileNotFoundError) as e: - print(f"error: {e}", file=sys.stderr) - return 1 - - -def _main(args: argparse.Namespace) -> int: - if args.k < 1 or args.workers < 1: - raise ValueError("--k and --workers must be positive") - modes = C.parse_modes(args.modes) - - if args.dry_run: - return _print_stream( - C.make_adapter(args.bench, args.subset, args.substrate, args.split_file), args, modes - ) - - adapter = C.make_adapter(args.bench, args.subset, args.substrate, args.split_file) - _stream_for(adapter, args) - from harness2_benchmarks.preflight import require_ready - - require_ready(adapter, judge=True) - - runs_root = C.runs_root_for(adapter) - library_root = C.library_root_for(adapter, args.k, seed=args.seed) - summary_ns = C.summary_namespace(adapter, args.k, seed=args.seed) - _check_identity(adapter, args, runs_root / summary_ns) - prop = Proposer(adapter.spec.channels["proposer"], audit_dir=runs_root / "_audit" / summary_ns) - - only = _csv(args.only_domains) - lanes = args.workers - print( - f"sweep {summary_ns}: {lanes} domain(s) in parallel, tasks within a domain in stream order", - flush=True, - ) - rep = schedule.run_sweep( - args.k, - adapter=adapter, - prop=prop, - runs_root=runs_root, - library_root=library_root, - split=args.split, - modes=modes, - seed=args.seed, - tag="", - domain_parallel=lanes, - only_domains=only, - max_tasks=args.max_tasks, - pool_policy="finals", - ) - - stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") - rec = runs_root / "_sweeps" / f"{summary_ns}-{stamp}.json" - rec.parent.mkdir(parents=True, exist_ok=True) - rec.write_text(json.dumps(rep, indent=2, default=str), encoding="utf-8") - - done = sum(int(d.get("done", 0)) for d in rep["domains"]) - failed = sum(int(d.get("failed", 0)) for d in rep["domains"]) - dead = [d["domain"] for d in rep["domains"] if d.get("error")] - if not rep["domains"]: - print( - f"\nsweep {summary_ns}: zero tasks scheduled (split={args.split!r}, " - f"only_domains={only!r}, max_tasks={args.max_tasks!r}); " - f"refusing to report success" - ) - print(f"record: {rec}") - return 1 - print( - f"\nsweep {summary_ns}: {done} tasks ok, {failed} failed, " - f"{len(dead)} domain(s) died{': ' + ', '.join(dead) if dead else ''}" - ) - print(f"record: {rec}") - print( - f"next: python3 -m harness2.cli.judge --bench {args.bench}" - + (f" --subset {args.subset}" if args.subset else "") - + f" --substrate {args.substrate} --k {args.k}" - + (f" --seed {args.seed}" if args.seed is not None else "") - + f" --modes {args.modes}" - ) - - if dead: - return 1 - return 2 if failed else 0 - - -def _check_identity(adapter, args, namespace_dir: Path) -> None: - """Prevent a resumed run or library from silently mixing models or task partitions.""" - identity = { - "method_version": config.METHOD_VERSION, - "executor": config.SOLVER_MODEL_OC if args.substrate == "oc" else config.SOLVER_MODEL_CX, - "improver": config.IMPROVER_MODEL, - "judge": config.JUDGE_MODEL, - "project": config.VERTEX_PROJECT, - "location": config.VERTEX_REGION, - "judge_endpoint": config.JB_JUDGE_API_BASE, - "endpoint": config.CODEX_BASE_URL if args.substrate == "cx" else "", - "split": args.split, - "tasks": _stream_for(adapter, args)[1], - "max_tasks": args.max_tasks, - "domains": sorted(_csv(args.only_domains)), - "seed": args.seed, - } - namespace_dir.mkdir(parents=True, exist_ok=True) - path = namespace_dir / "config.json" - if path.is_file(): - if json.loads(path.read_text()) != json.loads(json.dumps(identity)): - raise ValueError( - "This run directory belongs to different models or tasks. " - "Choose a new HARNESS2_RUNS_DIR for a new experiment." - ) - else: - path.write_text(json.dumps(identity, indent=2) + "\n", encoding="utf-8") - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/harness2/cli/runs.py b/harness2/cli/runs.py new file mode 100644 index 00000000..4aac7571 --- /dev/null +++ b/harness2/cli/runs.py @@ -0,0 +1,93 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""List persisted runs and task progress.""" + +from __future__ import annotations + +from datetime import datetime, timezone + +from . import _common as C + + +def add_parser(subparsers): + p = subparsers.add_parser("runs") + commands = p.add_subparsers(dest="action", required=True) + commands.add_parser("ls") + show = commands.add_parser("show") + show.add_argument("name") + return p + + +def describe(root) -> dict: + record = C.load_run(root) + plan = record["schedule"] + summaries = { + row["task_id"]: row + for _, row in C.summaries(root) + if row.get("k") == plan["k"] and row.get("seed") == plan.get("seed") + } + statuses = [ + {"task": task, "domain": domain, "status": summaries.get(task, {}).get("status", "pending")} + for domain, task in record["stream"] + ] + totals = C.event_totals(root) + return { + "run": record, + "totals": totals, + "tasks": statuses, + "done": sum(row["status"] in ("ok", "skipped") for row in statuses), + "total": len(statuses), + "last_activity": totals["last_activity"] + or datetime.fromtimestamp(root.stat().st_mtime, timezone.utc).isoformat(), + } + + +def run(args, ctx) -> int: + if args.action == "show": + ctx.out(describe(C.resolve_run(ctx, args.name)), str) + else: + ctx.project + root = ctx.settings.path("paths.runs") + paths = ( + sorted( + (p for p in root.iterdir() if (p / "run.json").is_file()), + key=lambda p: p.stat().st_mtime_ns, + reverse=True, + ) + if root.is_dir() + else [] + ) + rows = [] + for path in paths: + detail = describe(path) + rows.append( + { + "name": path.name, + "suite": detail["run"]["suite"], + "executor": detail["run"]["executor"], + "done": detail["done"], + "total": detail["total"], + "spent_usd": detail["totals"]["spent_usd"], + "last_activity": detail["last_activity"], + } + ) + ctx.out( + rows, + lambda data: "\n".join( + f"{row['name']}\t{row['suite']}\t{row['executor']}\t{row['done']}/{row['total']}\t${row['spent_usd']:.6f}\t{row['last_activity']}" + for row in data + ), + ) + return 0 diff --git a/harness2/cli/setup.py b/harness2/cli/setup.py new file mode 100644 index 00000000..7833e9fd --- /dev/null +++ b/harness2/cli/setup.py @@ -0,0 +1,37 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Dispatch setup arguments to an installed suite.""" + +from __future__ import annotations + +import argparse + +from .. import registry +from ..settings import ConfigError + + +def add_parser(subparsers): + p = subparsers.add_parser("setup") + p.add_argument("suite") + p.add_argument("args", nargs=argparse.REMAINDER) + return p + + +def run(args, ctx) -> int: + ctx.settings + suite = registry.load(args.suite) + if not hasattr(suite, "setup"): + raise ConfigError(f"suite {args.suite!r} has no setup command") + return suite.setup(args.args) diff --git a/harness2/cli/status.py b/harness2/cli/status.py new file mode 100644 index 00000000..ecf0b9ff --- /dev/null +++ b/harness2/cli/status.py @@ -0,0 +1,28 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Status command placeholder.""" + +from __future__ import annotations + +import sys + + +def add_parser(subparsers): + return subparsers.add_parser("status") + + +def run(args, ctx) -> int: + print("not implemented yet", file=sys.stderr) + return 1 diff --git a/harness2/cli/task.py b/harness2/cli/task.py new file mode 100644 index 00000000..b3bc31ba --- /dev/null +++ b/harness2/cli/task.py @@ -0,0 +1,59 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Manage task inputs for the built-in suite.""" + +from __future__ import annotations + +from pathlib import Path + + +def add_parser(subparsers): + p = subparsers.add_parser("task") + commands = p.add_subparsers(dest="action", required=True) + new = commands.add_parser("new") + new.add_argument("id") + new.add_argument("--files", action="store_true", help="Create a files workspace.") + new.add_argument("--domain", default="default", help="Task domain.") + new.add_argument("--check", help="Judge command.") + commands.add_parser("ls") + import_parser = commands.add_parser("import") + import_parser.add_argument("file", type=Path) + import_parser.add_argument("--force", action="store_true", help="Replace existing tasks.") + return p + + +def run(args, ctx) -> int: + ctx.project + from .. import suite + + tasks_dir = ctx.settings.path("paths.tasks") + if args.action == "new": + result = suite.new_task( + tasks_dir, args.id, files=args.files, domain=args.domain, check=args.check + ) + ctx.out({"path": result}, lambda data: str(data["path"])) + elif args.action == "ls": + ctx.out( + suite.list_tasks(tasks_dir), + lambda rows: "\n".join( + f"{row['id']}\t{row['domain']}\t{row['split']}\t{row['workspace']}\t{row['has_judge']}" + for row in rows + ), + ) + else: + ctx.out( + suite.import_tasks(tasks_dir, args.file, force=args.force), lambda rows: "\n".join(rows) + ) + return 0 diff --git a/harness2/config.py b/harness2/config.py deleted file mode 100644 index 59e0703d..00000000 --- a/harness2/config.py +++ /dev/null @@ -1,189 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Deployment settings and dependency versions. Shell values override .env.""" - -from __future__ import annotations - -import os -import sys -from pathlib import Path - -PKG = Path(__file__).resolve().parent -ROOT = PKG.parent if (PKG.parent / "pyproject.toml").is_file() else Path.cwd() -THIRD_PARTY = ROOT / "third_party" -METHOD_VERSION = 2 - -DEPENDENCIES = ROOT / "dependencies" - - -class ConfigError(RuntimeError): - """A required deployment value is missing; the message carries the remediation.""" - - -def load_dotenv(path: Path = ROOT / ".env") -> None: - """Load KEY=VALUE lines from `path` into os.environ with setdefault (the shell wins). - - Accepts blank lines, `#` comments, an optional `export ` prefix, and single- or - double-quoted values. A missing or unreadable file is not an error. - """ - try: - text = Path(path).read_text(encoding="utf-8") - except OSError: - return - for raw in text.splitlines(): - line = raw.strip() - if not line or line.startswith("#") or "=" not in line: - continue - if line.startswith("export "): - line = line[len("export ") :].lstrip() - key, _, value = line.partition("=") - key = key.strip() - if not key or not all(c.isalnum() or c == "_" for c in key): - continue - value = value.strip() - if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'): - value = value[1:-1] - elif " #" in value: - value = value.split(" #", 1)[0].rstrip() - os.environ.setdefault(key, value) - - -load_dotenv() - - -def env(name: str, default=None): - """Env override where a set-but-empty variable is the same as an unset one.""" - return os.environ.get(name) or default - - -def env_int(name: str, default: int) -> int: - v = env(name) - if v is None: - return int(default) - try: - return int(v) - except ValueError as e: - raise ConfigError(f"{name} must be an integer, got {v!r}") from e - - -def require(name: str, *, needed_by: str) -> str: - """The value of `name`, or ConfigError with the remediation line when it is unset/empty.""" - v = env(name) - if not v: - raise ConfigError( - f"{name} is required for {needed_by}; set it in .env or the shell (see .env.example)" - ) - return str(v) - - -RUNS = Path(env("HARNESS2_RUNS_DIR", ROOT / "runs")).expanduser().resolve() -VERTEX_PROJECT = env("GOOGLE_CLOUD_PROJECT", env("HARNESS2_VERTEX_PROJECT", "")) -VERTEX_REGION = env("GOOGLE_CLOUD_LOCATION", "global") -if VERTEX_PROJECT: - os.environ.setdefault("GOOGLE_CLOUD_PROJECT", VERTEX_PROJECT) -os.environ.setdefault("GOOGLE_CLOUD_LOCATION", VERTEX_REGION) - - -SOLVER_MODEL_OC = env("HARNESS2_SOLVER_MODEL", "") -SOLVER_MODEL_CX = env("HARNESS2_CODEX_MODEL", SOLVER_MODEL_OC.split("/", 1)[-1]) -IMPROVER_MODEL = env("HARNESS2_IMPROVER_MODEL", SOLVER_MODEL_OC) -JUDGE_MODEL = env("HARNESS2_JUDGE_MODEL", "") - -DEPENDENCY_PINS = { - "job-bench-eval": "fa03a17f9ff662f136798a274e10011c3aa00fd0", - "workbuddy-bench": "b516950be5b56eb3be406c2f76ee1c5111dcb57f", - "harvey-labs": "b58a28ae9fde5ee4b810e53b4fd0a1915325a35d", - "opencode": "23fb5e0516c99ac04a1aa46c193efda2e1b9bb24", -} -DEPENDENCY_ORIGINS = { - "job-bench-eval": "https://github.com/Job-Bench/job-bench-eval.git", - "workbuddy-bench": "https://github.com/Tencent/workbuddy-bench.git", - "harvey-labs": "https://github.com/harveyai/harvey-labs.git", - "opencode": "https://github.com/anomalyco/opencode.git", -} - - -def jb_repo() -> Path: - return DEPENDENCIES / "job-bench-eval" - - -def wb_repo() -> Path: - return DEPENDENCIES / "workbuddy-bench" - - -def lab_repo() -> Path: - return DEPENDENCIES / "harvey-labs" - - -OPENCODE_DIR = DEPENDENCIES / "opencode" -BUN = DEPENDENCIES / "tools" / "node_modules" / ".bin" / "bun" -LAB_PYTHON = lab_repo() / ".venv" / "bin" / "python" -LAB_JUDGE_MODEL = ( - "vertex/" + JUDGE_MODEL if JUDGE_MODEL.startswith(("claude", "gemini")) else JUDGE_MODEL -) -LAB_DELIV_CAP = 60000 -JB_JUDGE_PYTHON = jb_repo() / ".venv" / "bin" / "python" -JB_JUDGE_MODEL = ( - "google/" + JUDGE_MODEL - if JUDGE_MODEL.startswith("gemini") and not env("HARNESS2_JUDGE_BASE_URL") - else JUDGE_MODEL -) -_vertex_host = ( - "aiplatform.googleapis.com" - if VERTEX_REGION == "global" - else f"{VERTEX_REGION}-aiplatform.googleapis.com" -) -VERTEX_OPENAI_URL = ( - f"https://{_vertex_host}/v1/projects/{VERTEX_PROJECT}" - f"/locations/{VERTEX_REGION}/endpoints/openapi" -) -JB_JUDGE_API_BASE = env("HARNESS2_JUDGE_BASE_URL", VERTEX_OPENAI_URL if VERTEX_PROJECT else "") -JB_JUDGE_API_KEY = env("HARNESS2_JUDGE_API_KEY", "") -JB_JUDGE_REASONING_EFFORT = "low" -JB_REPORTING_JUDGE_DETAILS = "" -JB_OBS_TRUNC = 50000 - -CODEX_VERSION = "0.144.6" -CODEX_DIST = str(DEPENDENCIES / "codex" / "dist") -CODEX_BASE_URL = env("HARNESS2_CODEX_BASE_URL", "") -CODEX_CONTAINER_URL = env("HARNESS2_CODEX_CONTAINER_URL", CODEX_BASE_URL) -VERTEX_PROXY_KEY = env("HARNESS2_CODEX_API_KEY", "") -CODEX_HOME_DIR = DEPENDENCIES / "codex" / "home" - -WB_SOLVER_MODEL = "harness2/solver" -WB_WEB_JUDGE = "harness2/judge" -WB_RENDER_PY = sys.executable -WB_OC_HARNESS_VER = "1.14.18" -WB_CX_HARNESS_VER = CODEX_VERSION -WB_MAX_JOBS = {"oc": 1, "cx": 1} -WB_MAX_TRIALS = {"oc": 2, "cx": 2} -WB_MODEL_CONNECTION = "local_proxy" - -DEFAULT_PROPOSER_CHANNEL = "opencode" -PROPOSER_MODEL_BY_CHANNEL = {"opencode": IMPROVER_MODEL, "claude_cli": IMPROVER_MODEL} -PHASE_WIDTH = 2 -TRANSPORT_ATTEMPTS = 2 -PROPOSER_CONCURRENCY = 2 - - -def validate_models(*, judge: bool = False) -> None: - require("HARNESS2_SOLVER_MODEL", needed_by="executor rollouts") - if "/" not in SOLVER_MODEL_OC or "/" not in IMPROVER_MODEL: - raise ConfigError("Executor and improver models must use OpenCode's provider/model syntax") - if any(m.startswith("google-vertex") for m in (SOLVER_MODEL_OC, IMPROVER_MODEL)): - if not VERTEX_PROJECT: - raise ConfigError("Set GOOGLE_CLOUD_PROJECT for Vertex AI") - if judge: - require("HARNESS2_JUDGE_MODEL", needed_by="benchmark judging") diff --git a/pyproject.toml b/pyproject.toml index 09bab60e..dca5f5c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,13 +23,16 @@ dependencies = [] bridge = ["litellm[proxy]==1.95.0", "fastapi==0.139.2", "google-auth==2.56.3"] [project.scripts] -harness2 = "harness2.cli.run_sweep:main" -harness2-judge = "harness2.cli.judge:main" -harness2-report = "harness2.cli.report:main" -harness2-setup = "harness2_benchmarks.setup:main" -harness2-check = "harness2.cli.check:main" -harness2-demo = "harness2.cli.demo:main" -harness2-bridge = "harness2.cli.bridge:main" +harness2 = "harness2.cli.main:main" +harness2-judge = "harness2.cli.main:main" +harness2-report = "harness2.cli.main:main" +harness2-check = "harness2.cli.main:main" +harness2-demo = "harness2.cli.main:main" +harness2-bridge = "harness2.cli.main:main" +harness2-setup = "harness2.cli.main:main" + +[tool.setuptools] +include-package-data = false [tool.setuptools.packages.find] include = ["harness2*"] diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 00000000..acae6d49 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,619 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline command parsing, snapshots, adoption, and demo-backed CLI checks.""" + +from __future__ import annotations + +import contextlib +import io +import json +import os +import subprocess +import sys +import tempfile +import types +import unittest +from dataclasses import replace +from pathlib import Path +from unittest.mock import Mock, patch + +from harness2 import settings +from harness2.cli import _adopt, completion, judge, main, run, runs +from harness2.cli import _common as C +from harness2.core import events +from harness2.core.judge import Scored +from harness2.core.proposer import Reply +from harness2.demo import DemoAdapter, DemoProposer +from harness2.substrate.claude import ClaudeSubstrate + + +class CliTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.project = self.root / "project" + (self.project / ".harness2").mkdir(parents=True) + self.env = patch.dict( + os.environ, {"XDG_CONFIG_HOME": str(self.root / "config")}, clear=True + ) + self.env.start() + self.addCleanup(self.env.stop) + self.addCleanup(events.start, None) + self.config_path = self.project / ".harness2/config.toml" + self.config_path.write_text('[improver]\nmodel="openai/fake"\nchannel="opencode"\n') + self.adapters = [] + + def make_adapter(ctx): + adapter = DemoAdapter(ctx.run_root / "fixture") + adapter.spec = replace(adapter.spec, runs_root=ctx.run_root) + adapter.has_judge = lambda task: task == "ledger-1" + adapter.score = lambda path, task: {"n_passed": 1, "n_criteria": 1} + self.adapters.append(adapter) + return adapter + + self.suite = types.SimpleNamespace(make_adapter=make_adapter, check=lambda ctx: []) + self.registry = patch("harness2.registry.load", return_value=self.suite) + self.registry.start() + self.addCleanup(self.registry.stop) + + def invoke(self, *args, project=True): + stdout, stderr = io.StringIO(), io.StringIO() + argv = (["-C", str(self.project)] if project else []) + list(args) + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + code = main.main(argv) + return code, stdout.getvalue(), stderr.getvalue() + + def context(self, *args): + parsed = main.build_parser().parse_args(["-C", str(self.project), *args]) + return C.Context(parsed, getattr(parsed._module, "FLAGS", {})) + + def execute(self, k=1, *extra): + with patch.object(run, "Proposer", DemoProposer): + return self.invoke("--json", "run", "--k", str(k), *extra) + + def test_all_command_arguments(self): + commands = [ + [ + "run", + "--suite", + "tasks", + "--executor", + "claude", + "--name", + "x", + "--k", + "2", + "--mode", + "parallel,sequential", + "--seed", + "0", + "--split", + "train", + "--split-file", + "split.json", + "--only-domains", + "d", + "--only-tasks", + "a", + "--max-tasks", + "1", + "--workers", + "2", + "--budget-usd", + "1", + "--max-calls", + "20", + "--selector", + "human", + "--steer", + "@guide", + "--dry-run", + ], + ["doctor", "--live"], + ["setup", "tasks", "--flag"], + ["judge", "x", "--bases"], + ["report", "x", "y"], + ["runs", "ls"], + ["runs", "show", "x"], + ["diff", "x", "--task", "a", "--mode", "sequential", "--stat"], + ["task", "new", "a", "--files", "--domain", "d", "--check", "true"], + ["task", "ls"], + ["task", "import", "a.csv", "--force"], + ["config", "show", "--origin"], + ["config", "get", "run.k"], + ["config", "set", "run.k", "2", "--global"], + ["config", "path"], + ["demo", "--output", "demo", "--k", "1", "--mode", "both"], + ["bridge", "--model", "openai/model", "--host", "localhost", "--port", "1234"], + ["completion", "zsh"], + ["init"], + ["status"], + ["chat"], + ["apply"], + ] + parser = main.build_parser() + for argv in commands: + with self.subTest(argv=argv): + args = parser.parse_args(argv) + self.assertEqual(args.command, argv[0]) + self.assertTrue(callable(args._module.run)) + self.assertEqual(parser.parse_args(["run", "--bench", "jb"]).suite, "jb") + self.assertEqual(parser.parse_args(["run", "--modes", "seq"]).modes, "seq") + + def test_context_overrides_origins_and_output(self): + ctx = self.context( + "-c", "run.k=4", "run", "--k", "2", "--selector", "human", "--workers", "3" + ) + self.assertEqual(ctx.settings.get("run.k"), 2) + self.assertEqual(ctx.settings.origin("run.k"), "flag --k") + self.assertEqual(ctx.settings.role("selector")["channel"], "human") + self.assertEqual(ctx.settings.get("run.workers"), 3) + self.assertEqual(C.default_name(ctx.settings), "claude-k2") + ctx = self.context("run", "--seed", "0") + self.assertEqual(C.default_name(ctx.settings), "claude-k1-s0") + code, stdout, _ = self.invoke("--json", "config", "show", "--origin") + self.assertEqual(code, 0) + self.assertEqual(json.loads(stdout)["run.k"]["origin"], "default") + with self.assertRaises(settings.ConfigError): + self.context("-c", "oops", "config", "show").settings + with self.assertRaises(settings.ConfigError): + C.parse_modes("") + self.assertEqual(C.parse_modes("seq,agg,parallel"), ("agg", "seq")) + self.assertEqual(C.mode_label("seq"), "sequential") + + def test_lazy_project_and_errors(self): + with patch.object(settings, "find_project", return_value=None): + code, _, error = self.invoke("runs", "ls", project=False) + self.assertEqual(code, 1) + self.assertIn("not inside a harness2 project", error) + self.assertEqual( + self.invoke("--json", "config", "get", "run.k", project=False)[:2], (0, "1\n") + ) + self.assertEqual(self.invoke("--quiet", "runs", "show", "missing")[2], "") + self.assertIn("command: config", self.invoke("--verbose", "config", "path")[2]) + for command in ("init", "status", "chat", "apply"): + code, _, error = self.invoke(command, project=False) + self.assertEqual((code, error), (1, "not implemented yet\n")) + with self.assertRaises(settings.ConfigError): + C.run_path(self.context("runs", "ls"), "../escape") + for command in ("--help", "--version"): + with contextlib.redirect_stdout(io.StringIO()), self.assertRaises(SystemExit) as raised: + main.main([command]) + self.assertEqual(raised.exception.code, 0) + + def test_run_identity_and_name_resolution(self): + ctx = self.context("run") + root = C.run_path(ctx, C.default_name(ctx.settings)) + plan = {"k": 1, "modes": "parallel", "seed": None} + first = C.save_run(root, ctx.settings, [["d", "t"]], plan) + self.assertNotIn("run.k", first["settings"]) + self.assertNotIn("judge.model", first["settings"]) + self.assertIn("executor.model", first["settings"]) + changed = self.context("run", "--workers", "9", "--k", "2") + second = C.save_run(root, changed.settings, [["d", "t"]], {**plan, "k": 2}) + self.assertEqual(first["created"], second["created"]) + with self.assertRaisesRegex(settings.ConfigError, "executor.name.*claude.*codex"): + C.save_run( + root, self.context("run", "--executor", "codex").settings, [["d", "t"]], plan + ) + with self.assertRaisesRegex(settings.ConfigError, "stream"): + C.save_run(root, ctx.settings, [["d", "different"]], plan) + recent = root.parent / "new" + C.save_run(recent, ctx.settings, [["d", "t"]], plan) + os.utime(root, ns=(1, 1)) + self.assertEqual(C.resolve_run(ctx, None), recent) + self.assertEqual(C.resolve_run(ctx, root.name), root) + (recent / "run.json").write_text("[]") + with self.assertRaises(settings.ConfigError): + C.load_run(recent) + self.assertEqual(run._csv(" a, b, "), ("a", "b")) + + def test_snapshot_ignores_later_identity_settings(self): + ctx = self.context("run") + root = C.run_path(ctx, "snapshot") + record = C.save_run( + root, ctx.settings, [["d", "t"]], {"k": 1, "modes": "parallel", "seed": None} + ) + self.config_path.write_text( + '[executor]\nname="codex"\n[selector]\nchannel="human"\n[improver]\nmodel="openai/changed"\n' + ) + rebuilt = C.run_settings(self.context("-c", "judge.model=openai/new", "judge"), record) + self.assertEqual(C.identity(rebuilt), record["settings"]) + self.assertEqual(rebuilt.role("selector")["channel"], "opencode") + self.assertEqual(rebuilt.role("judge")["model"], "openai/new") + + def test_dry_run_estimate_and_stream_filters(self): + code, stdout, _ = self.invoke( + "--json", + "run", + "--dry-run", + "--k", + "1", + "--mode", + "parallel,sequential", + "--only-tasks", + "ledger-2", + ) + self.assertEqual(code, 0) + data = json.loads(stdout) + self.assertEqual(data["stream"], [["accounting", "ledger-2"]]) + self.assertEqual((data["estimate"]["rollouts"], data["estimate"]["sessions"]), (7, 8)) + self.assertEqual(data["estimate"]["per_task_per_mode"]["reflector"], 1) + self.assertEqual(list((self.project / ".harness2/runs").rglob("run.json")), []) + self.assertEqual(list((self.project / ".harness2/runs").rglob("events.jsonl")), []) + code, stdout, _ = self.invoke("run", "--dry-run", "--k", "2", "--max-tasks", "1") + self.assertEqual(code, 0) + self.assertIn("namespace[parallel]", stdout) + self.assertIn("library root", stdout) + self.assertIn("accounting", stdout) + self.assertIn("call estimate", stdout) + self.assertEqual( + run._estimate(1, 2, ("agg",), {"spent_usd": 4, "calls": 2})["average_cost_per_call"], 2 + ) + self.suite.check = lambda ctx: ["fixture error"] + code, _, error = self.invoke("run", "--dry-run") + self.assertEqual(code, 1) + self.assertIn("fixture error", error) + self.suite.check = lambda ctx: [] + self.suite.make_adapter = lambda ctx: types.SimpleNamespace(tasks=lambda split: []) + self.assertEqual(self.invoke("run", "--dry-run")[0], 1) + + def test_demo_backed_run_judge_report_diff_for_each_scale(self): + for k in (1, 2): + with self.subTest(k=k): + code, stdout, _ = self.execute(k, "--mode", "parallel,sequential") + self.assertEqual(code, 0) + data = json.loads(stdout) + self.assertEqual( + data["totals"]["calls"], 2 * (k + 6 * k + 6 * k + 2 * (1 + int(k > 1))) + ) + name = f"claude-k{k}" + root = self.project / ".harness2/runs" / name + record = C.load_run(root) + self.assertEqual( + record["stream"], [["accounting", "ledger-1"], ["accounting", "ledger-2"]] + ) + self.assertEqual(record["schedule"]["k"], k) + self.assertEqual( + record["settings"], C.identity(self.context("run", "--k", str(k)).settings) + ) + stored_events = [ + json.loads(line) for line in (root / "events.jsonl").read_text().splitlines() + ] + self.assertEqual(stored_events[0]["kind"], "run_start") + self.assertEqual(stored_events[-1]["status"], "ok") + code, stdout, error = self.invoke("--json", "judge", name) + self.assertEqual(code, 0) + self.assertEqual(len(json.loads(stdout)), 2) + self.assertEqual(error.count("no judge; skipped"), 1) + judged = next((root / "adapt").glob("*/judged.json")) + self.assertEqual(json.loads(judged.read_text())["judge_model"], "openai/fake") + self.assertEqual( + self.invoke("-c", "judge.model=openai/different", "judge", name)[0], 1 + ) + code, stdout, _ = self.invoke("--json", "report", name) + self.assertEqual((code, len(json.loads(stdout))), (0, 2)) + code, stdout, _ = self.invoke("--json", "diff", name) + self.assertEqual(code, 0) + diff = json.loads(stdout) + self.assertEqual(diff["task"], "ledger-2") + self.assertIn("-total = sum", diff["diff"]) + self.assertIn("+total = sum", diff["diff"]) + code, stdout, _ = self.invoke( + "--json", "diff", name, "--task", "ledger-1", "--mode", "sequential", "--stat" + ) + self.assertEqual(code, 0) + self.assertIn("scripts/summarize.py", json.loads(stdout)["modified"]) + self.assertEqual(self.invoke("diff", name, "--task", "missing")[0], 1) + self.assertEqual(self.invoke("--json", "judge", name, "--bases")[0], 0) + code, stdout, _ = self.invoke("--json", "runs", "show", name) + self.assertEqual((code, json.loads(stdout)["done"]), (0, 2)) + with patch.object(run, "Proposer", DemoProposer): + code, stdout, _ = self.invoke( + "--json", "run", "--k", str(k), "--mode", "parallel,sequential" + ) + self.assertEqual((code, self.adapters[-1].execution_count), (0, 0)) + code, stdout, _ = self.invoke("--json", "runs", "ls") + self.assertEqual((code, len(json.loads(stdout))), (0, 2)) + self.assertIn("claude-k1", self.invoke("runs", "ls")[1]) + code, stdout, _ = self.invoke("report", "claude-k1", "claude-k2") + self.assertEqual(code, 0) + self.assertIn("claude-k1", stdout) + self.assertIn("claude-k2", stdout) + self.assertIn("score 100.00%", stdout) + + def test_budget_resume_and_failure_exit_events(self): + code, stdout, _ = self.execute(1, "--max-calls", "1") + self.assertEqual(code, 4) + self.assertEqual(json.loads(stdout)["stopped"], "max_calls") + root = self.project / ".harness2/runs/claude-k1" + self.assertEqual( + json.loads((root / "events.jsonl").read_text().splitlines()[-1])["status"], "stopped" + ) + code, _, _ = self.execute(1, "--max-calls", "100") + self.assertEqual(code, 0) + for exception, expected, status in ( + (KeyboardInterrupt(), 130, "interrupted"), + (events.Stopped("interrupted"), 130, "interrupted"), + (RuntimeError("fixture crash"), 1, "error"), + ): + with ( + patch.object(run.schedule, "run_sweep", side_effect=exception), + patch.object(run, "Proposer", DemoProposer), + ): + code, _, error = self.invoke("run") + self.assertEqual(code, expected) + self.assertEqual( + json.loads((root / "events.jsonl").read_text().splitlines()[-1])["status"], status + ) + if expected == 130: + self.assertIn("run the same command to resume", error) + for result, expected in ( + ({"domains": [{"failed": 1}]}, 3), + ({"domains": [{"error": "dead"}]}, 1), + ): + with ( + patch.object(run.schedule, "run_sweep", return_value=result), + patch.object(run, "Proposer", DemoProposer), + ): + self.assertEqual(self.invoke("run")[0], expected) + + def test_human_selector_is_loaded_lazily(self): + self.assertNotIn("harness2.cli.human", sys.modules) + from harness2.core.spec import ChannelSpec + + original = self.suite.make_adapter + + def make(ctx): + adapter = original(ctx) + adapter.spec = replace( + adapter.spec, + channels={**adapter.spec.channels, "selector": ChannelSpec("human", "")}, + ) + return adapter + + self.suite.make_adapter = make + with ( + patch.object(run.schedule, "run_sweep", return_value={"domains": [{"done": 2}]}), + patch.object(run, "Proposer", DemoProposer), + ): + self.assertEqual(self.invoke("run", "--selector", "human")[0], 0) + self.assertIn("harness2.cli.human", sys.modules) + + def test_json_stays_parseable_when_adapter_prints(self): + self.execute() + original = self.suite.make_adapter + + def make(ctx): + print("adapter progress") + adapter = original(ctx) + + def score(path, task): + print("judge progress") + return {"n_passed": 1, "n_criteria": 1} + + adapter.score = score + return adapter + + self.suite.make_adapter = make + code, stdout, stderr = self.invoke("--json", "judge") + self.assertEqual((code, len(json.loads(stdout))), (0, 1)) + self.assertIn("judge progress", stderr) + code, stdout, stderr = self.invoke("--json", "--quiet", "judge") + self.assertEqual((code, len(json.loads(stdout)), stderr), (0, 1, "")) + + def test_config_task_setup_and_json(self): + self.assertEqual(self.invoke("--json", "config", "set", "run.k", "3")[0], 0) + self.assertEqual(self.invoke("--json", "config", "get", "run.k")[:2], (0, "3\n")) + self.assertEqual(self.invoke("config", "set", "run.k", "0")[0], 1) + self.assertEqual(self.invoke("config", "set", "run.k", "4", "--global")[0], 0) + self.assertTrue(settings.global_path().is_file()) + rows = [ + {"id": "x", "domain": "d", "split": "train", "workspace": "files", "has_judge": True} + ] + fake = types.ModuleType("harness2.suite") + fake.new_task = Mock(return_value=self.project / "task.md") + fake.list_tasks = Mock(return_value=rows) + fake.import_tasks = Mock(return_value=["x"]) + with patch.dict(sys.modules, {"harness2.suite": fake}): + self.assertEqual( + self.invoke( + "--json", "task", "new", "x", "--files", "--domain", "d", "--check", "true" + )[0], + 0, + ) + fake.new_task.assert_called_once_with( + self.project / ".harness2/tasks", "x", files=True, domain="d", check="true" + ) + code, stdout, _ = self.invoke("--json", "task", "ls") + self.assertEqual((code, json.loads(stdout)), (0, rows)) + self.assertIn("x\td\ttrain", self.invoke("task", "ls")[1]) + self.assertEqual(self.invoke("--json", "task", "import", "tasks.csv", "--force")[0], 0) + fake.import_tasks.assert_called_once_with( + self.project / ".harness2/tasks", Path("tasks.csv"), force=True + ) + self.assertEqual(self.invoke("setup", "demo")[0], 1) + self.suite.setup = Mock(return_value=7) + self.assertEqual(self.invoke("setup", "demo", "--example")[0], 7) + self.suite.setup.assert_called_once_with(["--example"]) + + def test_doctor_fake_binary_and_live_deduplication(self): + binary = self.root / "bin/opencode" + binary.parent.mkdir() + binary.write_text("#!/bin/sh\nprintf 'fixture 1.0\\n'\n") + binary.chmod(0o755) + os.environ["PATH"] = str(binary.parent) + os.environ["OPENAI_API_KEY"] = "fixture" + self.config_path.write_text( + '[executor]\nname="opencode"\nmodel="openai/fake"\n[improver]\nchannel="opencode"\nmodel="openai/fake"\n' + ) + with patch( + "harness2.cli.doctor.Proposer.call", return_value=Reply("doctor", "ok", True) + ) as call: + code, stdout, _ = self.invoke("--json", "doctor", "--live") + self.assertEqual(code, 0) + self.assertEqual(call.call_count, 1) + rows = json.loads(stdout) + self.assertEqual(sum(row["check"].startswith("binary") for row in rows), 1) + self.assertTrue(any("fixture 1.0" in row["detail"] for row in rows)) + os.environ.pop("OPENAI_API_KEY") + binary.unlink() + code, stdout, _ = self.invoke("--json", "doctor") + self.assertEqual(code, 1) + self.assertTrue( + any(not row["ok"] and row["check"].startswith("binary") for row in json.loads(stdout)) + ) + self.config_path.write_text('[improver]\nmodel="invalid"\n') + self.assertEqual(self.invoke("doctor")[0], 1) + + def test_completion_aliases_and_module_entry(self): + for shell in ("bash", "zsh"): + script = completion.generate(main.build_parser(), shell) + for command in main.COMMANDS: + self.assertIn(":" + command, script) + self.assertIn("--only-tasks", script) + result = subprocess.run( + ["/bin/" + shell, "-n"], input=script, text=True, capture_output=True + ) + self.assertEqual(result.returncode, 0, result.stderr) + if shell == "bash": + probe = ( + script + + '\nCOMP_WORDS=(harness2 run --on); COMP_CWORD=2; _harness2_complete; printf "%s\\n" "${COMPREPLY[@]}"' + ) + else: + probe = ( + 'compdef() { :; }; compadd() { print -l -- "$@"; };\n' + + script + + "\nwords=(harness2 run --on); CURRENT=3; _harness2_complete" + ) + result = subprocess.run(["/bin/" + shell], input=probe, text=True, capture_output=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("--only-tasks", result.stdout) + for executable, command in main.ALIASES.items(): + with ( + patch.object(sys, "argv", [executable]), + patch.object(main, "build_parser", wraps=main.build_parser), + ): + with ( + contextlib.redirect_stdout(io.StringIO()) as stdout, + self.assertRaises(SystemExit), + ): + main.main(["--help"]) + self.assertIn("harness2 " + command, stdout.getvalue()) + with patch.object(sys, "argv", ["harness2-check"]): + self.assertEqual(self.invoke("--json", "--live")[0], 1) + result = subprocess.run( + [sys.executable, "-B", "-m", "harness2", "--version"], capture_output=True, text=True + ) + self.assertEqual((result.returncode, result.stdout.strip()), (0, "0.1.0")) + + def test_judge_legacy_cache_and_merge(self): + good = Scored("t", "agg", 1, "agg1", "id", False, n_passed=1, n_criteria=1) + bad = Scored("t", "agg", 1, "agg1", "id", False, error="judge failed") + self.assertEqual(judge._merge([good], [bad]), [good]) + changed = replace(bad, run_id="new") + self.assertEqual(judge._merge([good], [changed]), [changed]) + path = self.root / "judged.json" + self.assertEqual(judge._load_rows(path), []) + row = good.__dict__ | {"graded": True, "frac": 1.0} + row["arm"] = row.pop("mode") + path.write_text(json.dumps([row])) + self.assertEqual(judge._load_rows(path), [good]) + judge._judge_model(path, "new") + path.write_text(json.dumps({"judge_model": "old", "rows": [row]})) + with self.assertRaises(settings.ConfigError): + judge._judge_model(path, "new") + path.write_text("broken") + with self.assertRaises(ValueError): + judge._load_rows(path) + with self.assertRaises(settings.ConfigError): + judge._judge_model(path, "new") + path = self.root / "ns/t/summary.json" + path.parent.mkdir(parents=True) + path.write_text(json.dumps({"selections": {"agg": row}})) + self.assertEqual(judge.collect_selections(path.parent.parent, ("agg",))[0]["mode"], "agg") + path.write_text("broken") + with self.assertRaises(settings.ConfigError): + judge.collect_selections(path.parent.parent, ("agg",)) + + def test_adoption_changes_diff_and_native_export(self): + before = {"a": "old", "d": "removed\n", "same": "same"} + after = {"a": "new", "b": "added\n", "same": "same"} + self.assertEqual(_adopt.changes(before, after), [("a", "M"), ("b", "A"), ("d", "D")]) + self.assertIn("No newline at end of file", _adopt.unified("a", before, after)) + self.assertIn("--- /dev/null", _adopt.unified("b", before, after)) + self.assertIn("+++ /dev/null", _adopt.unified("d", before, after)) + adapter = types.SimpleNamespace(substrate=ClaudeSubstrate()) + config = self.context("config", "show").settings + (self.project / "CLAUDE.md").write_text("old\n") + base = _adopt.project_tree(adapter, config) + self.assertEqual(base, {"CLAUDE.md": "old\n"}) + self.assertEqual(_adopt.dirty_paths(config, adapter), []) + + def git(*args): + return subprocess.run( + ["/usr/bin/git", "-C", str(self.project), *args], + check=True, + capture_output=True, + text=True, + ) + + git("init", "-q") + git("add", "CLAUDE.md") + git("-c", "user.name=test", "-c", "user.email=test@example.test", "commit", "-qm", "base") + (self.project / "unrelated.txt").write_text("keep") + (self.project / "CLAUDE.md").write_text("dirty") + (self.project / ".claude/scripts").mkdir(parents=True) + (self.project / ".claude/scripts/new.py").write_text("pass\n") + self.assertEqual( + _adopt.dirty_paths(config, adapter), [".claude/scripts/new.py", "CLAUDE.md"] + ) + result = _adopt.perform( + config, adapter, {"CLAUDE.md": "adopted\n"}, base=base, branch="adopt" + ) + self.assertEqual(result, {"written": ["CLAUDE.md"], "deleted": []}) + self.assertEqual(git("branch", "--show-current").stdout.strip(), "adopt") + self.assertEqual((self.project / "unrelated.txt").read_text(), "keep") + with self.assertRaises(settings.ConfigError): + _adopt.perform(config, adapter, {}, base=base, branch="adopt") + self.assertEqual( + _adopt.perform(config, adapter, {}, base=base, branch=None)["deleted"], ["CLAUDE.md"] + ) + + def test_missing_and_corrupt_state(self): + self.assertEqual(self.invoke("report")[0], 1) + self.assertEqual(self.invoke("--json", "runs", "ls")[:2], (0, "[]\n")) + self.execute() + ctx = self.context("runs", "ls") + adapter, root = _adopt.open_run(ctx, None) + data = C.load_run(root) + for _, task in data["stream"]: + path = root / "adapt/demo-oc-agg1" / task / "summary.json" + summary = json.loads(path.read_text()) + summary["selections"]["agg"]["ok"] = False + path.write_text(json.dumps(summary)) + with self.assertRaises(settings.ConfigError): + _adopt.adopted_tree(adapter, root, None, "parallel") + (root / "events.jsonl").write_text("broken") + with self.assertRaises(settings.ConfigError): + C.event_totals(root) + with self.assertRaises(settings.ConfigError): + runs.describe(root) + + +if __name__ == "__main__": + unittest.main() From e9450ba1c578335bbfb672ee1505f5cc900db5ea Mon Sep 17 00:00:00 2001 From: Bae Seongmin <83870598+3ae3ae@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:59:23 +0900 Subject: [PATCH 07/21] feat: built-in tasks suite for your own project tasks (#7) harness2.suite reads tasks from .harness2/tasks// (task.md plus an optional task.toml, files/, check/ and rubric.md) and runs them on the claude, codex or command executor: - project tasks run in a throwaway git worktree at `ref`, with the project's own harness and .harness2/ removed and the candidate harness installed; the result is the patch the agent made, harness files excluded; - files tasks run in a scratch directory seeded from files/, and the result is what the agent saved in output/. The base harness is imported from the project once per run and kept, so a resumed run compares like with like. Rollouts are idempotent by run id and record exit code, time and reported cost. Grading is optional and never used for refinement: a `check` command is run against the result in a fresh workspace (exit status, or a JSON {n_passed, n_criteria} line), or an LLM judge applies rubric.md. Cached grades are reused only while the check, rubric and judge model are unchanged. Also adds task listing, scaffolding and JSONL/CSV import, and the improver prompt profiles for repository and files work. Co-authored-by: Claude Opus 5.5 --- harness2.egg-info/SOURCES.txt | 26 +- harness2.egg-info/entry_points.txt | 14 +- harness2/suite.py | 718 +++++++++++++++++++++++++++++ harness2/workspace.py | 110 +++++ tests/test_cli.py | 5 +- tests/test_suite.py | 670 +++++++++++++++++++++++++++ tests/test_workspace.py | 94 ++++ 7 files changed, 1625 insertions(+), 12 deletions(-) create mode 100644 harness2/suite.py create mode 100644 harness2/workspace.py create mode 100644 tests/test_suite.py create mode 100644 tests/test_workspace.py diff --git a/harness2.egg-info/SOURCES.txt b/harness2.egg-info/SOURCES.txt index dca95e0f..af94ef2d 100644 --- a/harness2.egg-info/SOURCES.txt +++ b/harness2.egg-info/SOURCES.txt @@ -8,11 +8,13 @@ assets/harness2_pipeline.png assets/harness2_recursion_modes.png assets/logo.png harness2/__init__.py -harness2/config.py +harness2/__main__.py harness2/demo.py harness2/providers.py harness2/registry.py harness2/settings.py +harness2/suite.py +harness2/workspace.py harness2.egg-info/PKG-INFO harness2.egg-info/SOURCES.txt harness2.egg-info/dependency_links.txt @@ -20,13 +22,26 @@ harness2.egg-info/entry_points.txt harness2.egg-info/requires.txt harness2.egg-info/top_level.txt harness2/cli/__init__.py +harness2/cli/_adopt.py harness2/cli/_common.py +harness2/cli/apply.py harness2/cli/bridge.py -harness2/cli/check.py +harness2/cli/chat.py +harness2/cli/completion.py +harness2/cli/config.py harness2/cli/demo.py +harness2/cli/diff.py +harness2/cli/doctor.py +harness2/cli/human.py +harness2/cli/init.py harness2/cli/judge.py +harness2/cli/main.py harness2/cli/report.py -harness2/cli/run_sweep.py +harness2/cli/run.py +harness2/cli/runs.py +harness2/cli/setup.py +harness2/cli/status.py +harness2/cli/task.py harness2/core/__init__.py harness2/core/context.py harness2/core/events.py @@ -51,9 +66,12 @@ harness2/substrate/command.py harness2/substrate/opencode.py scripts/render_readme_typing.py tests/test_bridge.py +tests/test_cli.py tests/test_executors.py tests/test_providers.py tests/test_registry.py tests/test_runtime.py tests/test_settings.py -tests/test_workflow.py \ No newline at end of file +tests/test_suite.py +tests/test_workflow.py +tests/test_workspace.py \ No newline at end of file diff --git a/harness2.egg-info/entry_points.txt b/harness2.egg-info/entry_points.txt index 7e26df27..6ac95293 100644 --- a/harness2.egg-info/entry_points.txt +++ b/harness2.egg-info/entry_points.txt @@ -1,8 +1,8 @@ [console_scripts] -harness2 = harness2.cli.run_sweep:main -harness2-bridge = harness2.cli.bridge:main -harness2-check = harness2.cli.check:main -harness2-demo = harness2.cli.demo:main -harness2-judge = harness2.cli.judge:main -harness2-report = harness2.cli.report:main -harness2-setup = harness2_benchmarks.setup:main +harness2 = harness2.cli.main:main +harness2-bridge = harness2.cli.main:main +harness2-check = harness2.cli.main:main +harness2-demo = harness2.cli.main:main +harness2-judge = harness2.cli.main:main +harness2-report = harness2.cli.main:main +harness2-setup = harness2.cli.main:main diff --git a/harness2/suite.py b/harness2/suite.py new file mode 100644 index 00000000..2143948e --- /dev/null +++ b/harness2/suite.py @@ -0,0 +1,718 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The built-in suite for tasks in the user's project or supplied files.""" + +from __future__ import annotations + +import csv +import hashlib +import json +import os +import re +import shutil +import subprocess +import tempfile +import threading +import time +import tomllib +import warnings +from dataclasses import replace +from pathlib import Path + +from . import providers, workspace +from .core import runmark, tree +from .core.library import slug +from .core.proposer import Proposer, ProposerError, _parse_json +from .core.spec import BenchPrompts, BenchSpec, ChannelSpec +from .registry import SuiteContext +from .settings import ConfigError, Settings +from .substrate._native import checked_path, cli_argv +from .substrate.claude import ClaudeSubstrate +from .substrate.codex import CodexSubstrate +from .substrate.command import CommandSubstrate + +DESCRIPTION = "Tasks" + +PROJECT_PROMPTS = BenchPrompts( + work=( + "Software engineering in this repository: the changes, fixes and investigations a " + "maintainer asks for. The request controls. The result must be a change a maintainer " + "could merge: complete, consistent with the codebase's conventions, and checked by " + "running it." + ), + domain_word="area", + deliverable_word="change", + inv_levers=( + "- locate the code, tests and conventions that govern the request before editing; read " + "before writing\n" + "- find and use the project's own way to build, run and test; run it before finishing " + "and read what it reports\n" + "- keep the change scoped to the request; do not reformat, rename or refactor what the " + "request did not ask for\n" + "- when a command fails, read the error and change approach instead of repeating it" + ), + dom_levers=( + "- the codebase's architecture, module boundaries and the patterns existing code follows\n" + "- the invariants a change in this area must keep: interfaces, data formats, error " + "handling, compatibility\n" + "- where the tests for this area live and what a convincing test for this kind of " + "change looks like" + ), + yardstick=( + "Did the run find the code that governs the request, make the change the request asked " + "for and nothing else, follow the codebase's conventions, and verify the change by " + "running it before stopping?" + ), + selection_checks=( + "Prefer the change that does what the request asks, completely; fits the surrounding " + "code's conventions; keeps the behaviour the request did not ask to change; and shows " + "it was verified. A smaller change that fully solves the request beats a larger one." + ), +) + +FILES_PROMPTS = BenchPrompts( + work=( + "Producing the documents, analyses or data a request asks for from the supplied files. " + "The request and the source files control. The result must be ready to use: complete, " + "accurate to the sources, and in exactly the files and format requested." + ), + domain_word="area", + deliverable_word="deliverable", + inv_levers=( + "- inventory the supplied files and map each requested output to the sources that " + "control it\n" + "- write every requested file early enough that it exists before the budget runs out, " + "then reopen it and check that it is complete and parses\n" + "- use code for counting, arithmetic and format conversion instead of doing it by hand\n" + "- when a tool fails on a file, read the error and try one bounded alternative" + ), + dom_levers=( + "- what a complete deliverable of this kind contains and how its audience will use it\n" + "- which source governs when two sources disagree, and how to state what the sources " + "do not say\n" + "- the checks a practitioner runs on this kind of work before handing it over" + ), + yardstick=( + "Did the run read the sources that control the request, produce every requested file in " + "usable form, keep every figure and claim traceable to a source, and check the finished " + "files before stopping?" + ), + selection_checks=( + "Prefer the deliverable that answers every part of the request, stays accurate to the " + "supplied files, is in exactly the requested files and format, and needs no repair " + "before use." + ), +) + +FILES_PROMPT_SUFFIX = "\n\nSave your deliverables in the output/ directory." + +RUBRIC_MISSION = """\ +You are grading one result of a task against a rubric. + +context/task.md is the request the agent received. context/rubric.md lists the criteria. +context/deliverables/ holds the result: the files it produced or its patch, and its final +message. + +Read the request, the rubric and the whole result. For each criterion, decide whether the +result meets it, judging only what the result actually contains -- not what it claims or +promises. A criterion the result does not address is not met. + +Give your reasoning per criterion, then end with exactly one JSON object on the last line: +{"n_passed": , "n_criteria": , "notes": ""} +""" + +TASK_TEMPLATE = "TODO: describe the task the way you would ask the agent to do it.\n" +# suite.check reports a task whose task.md still starts with "TODO:": +# "task {id}: task.md still holds the template text" + +BASE_DRIFT_WARNING = ( + "the project's harness has changed since this run started; the run keeps using the copy " + "it started with in {path}" +) + +_TASK_KEYS = {"domain", "split", "workspace", "ref", "deliverable", "check", "timeout"} + + +def _task_id(task_id: str) -> str: + if not isinstance(task_id, str) or not re.fullmatch(r"[a-z0-9][a-z0-9._-]*", task_id): + raise ConfigError(f"invalid task id {task_id!r}; use [a-z0-9][a-z0-9._-]*") + return task_id + + +def _task_config(task_id: str, values: dict, *, files: bool) -> dict: + _task_id(task_id) + for key in values: + if key not in _TASK_KEYS: + raise ConfigError(f"task {task_id}: unknown task.toml key {key!r}") + for key, value in values.items(): + if key == "timeout": + if type(value) is not int or value <= 0: + raise ConfigError(f"task {task_id}: timeout must be a positive integer") + elif not isinstance(value, str) or not value.strip(): + raise ConfigError(f"task {task_id}: {key} must be a non-empty string") + config = {"domain": "default", "workspace": "files" if files else "project", "ref": "HEAD"} + config.update(values) + config.setdefault("deliverable", "diff" if config["workspace"] == "project" else "files") + for key, choices in ( + ("split", ("train", "test")), + ("workspace", ("project", "files")), + ("deliverable", ("diff", "files", "message")), + ): + if key in config and config[key] not in choices: + raise ConfigError(f"task {task_id}: {key} must be one of {', '.join(choices)}") + return config + + +def _read_task(tasks_dir: Path, task_id: str) -> dict: + _task_id(task_id) + task_dir = checked_path(tasks_dir, task_id) + try: + checked_path(task_dir, "task.md").read_text(encoding="utf-8") + config_path = checked_path(task_dir, "task.toml") + values = ( + tomllib.loads(config_path.read_text(encoding="utf-8")) if config_path.exists() else {} + ) + except (OSError, ValueError, UnicodeError) as exc: + raise ConfigError(f"task {task_id}: cannot read task: {exc}") from exc + return _task_config(task_id, values, files=(task_dir / "files").is_dir()) + + +def list_tasks(tasks_dir: Path) -> list[dict]: + if not tasks_dir.exists(): + return [] + rows = [] + for path in sorted(tasks_dir.iterdir()): + if not path.is_dir(): + continue + config = _read_task(tasks_dir, path.name) + rows.append( + { + "id": path.name, + "domain": config["domain"], + "split": config.get("split"), + "workspace": config["workspace"], + "has_judge": bool(config.get("check") or (path / "rubric.md").is_file()), + } + ) + return rows + + +def _write_task(task_dir: Path, prompt: str, values: dict) -> None: + prompt_path = checked_path(task_dir, "task.md") + config_path = checked_path(task_dir, "task.toml") + files_path = checked_path(task_dir, "files") + task_dir.mkdir(parents=True, exist_ok=True) + prompt_path.write_text(prompt, encoding="utf-8") + config_path.write_text( + "".join( + f"{key} = {json.dumps(value, ensure_ascii=False)}\n" for key, value in values.items() + ), + encoding="utf-8", + ) + if values.get("workspace") == "files": + files_path.mkdir(exist_ok=True) + + +def new_task(tasks_dir: Path, task_id: str, *, files: bool, domain: str, check: str | None) -> Path: + values = {"domain": domain, "workspace": "files" if files else "project"} + if check is not None: + values["check"] = check + _task_config(task_id, values, files=files) + task_dir = checked_path(tasks_dir, task_id) + if task_dir.exists(): + raise ConfigError(f"task {task_id} already exists") + _write_task(task_dir, TASK_TEMPLATE, values) + return task_dir + + +def import_tasks(tasks_dir: Path, source: Path, *, force: bool) -> list[str]: + try: + with source.open(encoding="utf-8", newline="") as stream: + if source.suffix.lower() == ".jsonl": + records = [json.loads(line) for line in stream if line.strip()] + elif source.suffix.lower() == ".csv": + records = list(csv.DictReader(stream)) + else: + raise ConfigError("task imports must use .jsonl or .csv") + except (OSError, ValueError, UnicodeError, csv.Error) as exc: + raise ConfigError(f"cannot import {source}: {exc}") from exc + pending = {} + for record in records: + if not isinstance(record, dict): + raise ConfigError("each imported task must be an object") + task_id = _task_id(record.get("id")) + prompt = record.get("prompt") + if not isinstance(prompt, str) or not prompt.strip(): + raise ConfigError(f"task {task_id}: prompt is required") + values = {k: v for k, v in record.items() if k not in ("id", "prompt") and v != ""} + if source.suffix.lower() == ".csv" and "timeout" in values: + try: + values["timeout"] = int(values["timeout"]) + except (TypeError, ValueError) as exc: + raise ConfigError(f"task {task_id}: timeout must be a positive integer") from exc + _task_config(task_id, values, files=(tasks_dir / task_id / "files").is_dir()) + if task_id in pending: + raise ConfigError(f"duplicate task id {task_id!r} in import") + task_dir = checked_path(tasks_dir, task_id) + if task_dir.exists() and not force: + raise ConfigError(f"task {task_id} already exists; use --force to replace it") + pending[task_id] = (prompt, values) + for task_id, (prompt, values) in pending.items(): + _write_task(tasks_dir / task_id, prompt, values) + return list(pending) + + +def _channel(settings: Settings, role: str) -> ChannelSpec: + values = settings.role(role) + kind = values["channel"] + if kind == "human" and role != "selector": + raise ConfigError("the human channel is only allowed for selector") + provider = None + if kind != "human": + provider_name, _ = providers.split_model(values["model"]) + provider = settings.provider(provider_name) + try: + return ChannelSpec( + kind=kind, + model=values["model"], + bin=values["bin"], + effort=values["effort"], + timeout=values["timeout"], + steps=values["steps"], + provider=provider, + ) + except ValueError as exc: + raise ConfigError(str(exc)) from exc + + +class GenericAdapter: + """Run local task definitions through the shared adaptation method.""" + + def __init__(self, ctx: SuiteContext): + self.ctx = ctx + self.settings = ctx.settings + self.tasks_dir = self.settings.path("paths.tasks") + self.executor = self.settings.section("executor") + name = self.executor["name"] + if name == "claude": + self.substrate = ClaudeSubstrate(bin=self.executor["bin"]) + install_mode = "claude_root" + elif name == "codex": + self.substrate = CodexSubstrate(bin=self.executor["bin"]) + install_mode = "codex_root" + elif name == "command": + try: + self.substrate = CommandSubstrate( + self.executor["command"], tuple(self.executor["editable"]) + ) + except ValueError as exc: + raise ConfigError(str(exc)) from exc + install_mode = "command_files" + else: + raise ConfigError("the tasks suite runs on claude, codex or command") + self.spec = BenchSpec( + name="tasks", + runs_root=ctx.run_root, + install_mode=install_mode, + prompts=PROJECT_PROMPTS, + channels={ + "proposer": _channel(self.settings, "improver"), + "selector": _channel(self.settings, "selector"), + "reflector": _channel(self.settings, "reflector"), + }, + domain_word="area", + leak_globs=("rubric.md", "check/**"), + allow_subagents=name == "claude", + obs_trunc=self.settings.get("run.obs_trunc"), + ) + self._warned_drift = False + self._base_lock = threading.Lock() + + def tasks(self, split: str = "all") -> list[str]: + if split not in ("all", "train", "test"): + raise ConfigError(f"unknown task split {split!r}; use all, train or test") + return [ + row["id"] + for row in list_tasks(self.tasks_dir) + if split == "all" or row["split"] == split + ] + + def domain_of(self, task_id: str) -> str: + return _read_task(self.tasks_dir, task_id)["domain"] + + def task_prompt(self, task_id: str) -> str: + _read_task(self.tasks_dir, task_id) + return (self.tasks_dir / task_id / "task.md").read_text(encoding="utf-8") + + def task_files(self, task_id: str) -> Path | None: + _read_task(self.tasks_dir, task_id) + path = checked_path(self.tasks_dir / task_id, "files") + return path if path.is_dir() else None + + def prompts_for(self, task_id: str) -> BenchPrompts: + config = _read_task(self.tasks_dir, task_id) + prompts = PROJECT_PROMPTS if config["workspace"] == "project" else FILES_PROMPTS + steer = self.settings.get("improver.steer") + if steer.startswith("@"): + path = self.ctx.project / Path(steer[1:]).expanduser() + try: + steer = path.read_text(encoding="utf-8") + except (OSError, UnicodeError) as exc: + raise ConfigError(f"cannot read improver.steer file {path}: {exc}") from exc + try: + return replace( + prompts, + work=self.settings.get("suite.work") or prompts.work, + deliverable_word="answer" + if config["deliverable"] == "message" + else prompts.deliverable_word, + domain_guidance=steer, + ) + except ValueError as exc: + raise ConfigError("improver.steer / suite.work: " + str(exc)) from exc + + def base_harness(self) -> Path: + dest = self.spec.runs_root / "base_harness" + surface = self.substrate.surface() + with self._base_lock: + current = self.substrate.import_tree(self.settings.path("paths.harness")) + if not dest.exists(): + tree.write_tree(surface, current, dest, stubs=True) + elif ( + not self._warned_drift + and tree.read_tree(surface, dest) != {a: "" for a in surface.always_on} | current + ): + warnings.warn(BASE_DRIFT_WARNING.format(path=dest), stacklevel=2) + self._warned_drift = True + return dest + + def _workspace(self, task_id: str, config: dict): + if config["workspace"] == "project": + return workspace.project_workspace(self.ctx.project, config["ref"]) + return workspace.files_workspace(self.task_files(task_id)) + + def execute( + self, task_id: str, harness_dir: Path | None, run_id: str, *, timeout: int | None = None + ) -> Path: + config = _read_task(self.tasks_dir, task_id) + harness_dir = harness_dir if harness_dir is not None else self.base_harness() + run_name = slug(run_id) + if not run_name or run_name in (".", "..") or "\\" in run_name: + raise ConfigError(f"invalid rollout id {run_id!r}") + run_dir = self.spec.runs_root / "rollouts" / run_name + if runmark.is_complete(run_dir, harness_dir=harness_dir): + return run_dir + if run_dir.exists(): + shutil.rmtree(run_dir) + run_dir.mkdir(parents=True) + prompt_file = run_dir / "prompt.md" + prompt = self.task_prompt(task_id) + if config["workspace"] == "files": + prompt += FILES_PROMPT_SUFFIX + prompt_file.write_text(prompt, encoding="utf-8") + wall = config.get("timeout") or timeout or self.executor["timeout"] + started = time.monotonic() + timed_out = False + with self._workspace(task_id, config) as ws: + self.substrate.strip_native(ws) + if config["workspace"] == "project": + state = ws / ".harness2" + if state.is_symlink() or state.is_file(): + state.unlink() + else: + shutil.rmtree(state, ignore_errors=True) + install = self.substrate.materialise(harness_dir, self.spec, {}) + try: + self.substrate.install_into(install, ws) + finally: + shutil.rmtree(install.staging_dir, ignore_errors=True) + if config["workspace"] == "project": + workspace.commit_all(ws, "harness2: candidate harness") + base = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=ws, text=True + ).strip() + env = dict(os.environ) + if self.executor["name"] != "command": + provider_name, model = providers.split_model(self.executor["model"]) + provider = self.settings.provider(provider_name) + if self.executor["name"] == "claude": + env = self.substrate.env(env, provider, run_dir / "claude-home") + else: + home = run_dir / "codex-home" + self.substrate.write_home(home, provider, model, self.executor["effort"]) + env = self.substrate.env(env, home) + argv = self.substrate.argv(prompt_file, ws, model, self.executor["effort"]) + try: + if self.executor["name"] == "command": + exit_code = self.substrate.run(prompt_file, ws, run_dir, wall, env) + else: + with ( + prompt_file.open("rb") as stdin, + (run_dir / "stream.jsonl").open("wb") as stdout, + (run_dir / "stderr.log").open("wb") as stderr, + ): + exit_code = subprocess.run( + argv, + cwd=ws, + env=env, + stdin=stdin, + stdout=stdout, + stderr=stderr, + timeout=wall, + ).returncode + except subprocess.TimeoutExpired: + exit_code, timed_out = -1, True + except OSError as exc: + (run_dir / "stderr.log").write_text(str(exc), encoding="utf-8") + exit_code = -1 + if config["workspace"] == "project": + (run_dir / "deliverable.patch").write_text( + workspace.diff_since(ws, base, (*self.substrate.NATIVE_PATHS, ".harness2")), + encoding="utf-8", + ) + if (ws / "output").is_dir(): + shutil.copytree(ws / "output", run_dir / "output") + stream = run_dir / "stream.jsonl" + (run_dir / "final.md").write_text( + self.substrate.final_message(stream), encoding="utf-8" + ) + meta = { + "task_id": task_id, + "deliverable": config["deliverable"], + "ok": exit_code == 0 and not timed_out, + "exit_code": exit_code, + "wall_s": round(time.monotonic() - started, 3), + "cost_usd": self.substrate.cost_usd(stream), + "timed_out": timed_out, + } + (run_dir / "meta.json").write_text(json.dumps(meta, indent=2), encoding="utf-8") + runmark.mark_complete(run_dir, run_id=run_id, harness_dir=harness_dir) + return run_dir + + def rollout_cost(self, run_dir: Path) -> float: + return float(json.loads((run_dir / "meta.json").read_text())["cost_usd"]) + + def delivered(self, run_dir: Path) -> bool: + if not (run_dir / "meta.json").is_file(): + return False + kind = json.loads((run_dir / "meta.json").read_text())["deliverable"] + if kind == "files": + return any(path.is_file() for path in (run_dir / "output").rglob("*")) + path = run_dir / ("deliverable.patch" if kind == "diff" else "final.md") + return path.is_file() and bool(path.read_text(encoding="utf-8").strip()) + + def trajectory(self, run_dir: Path, *, obs_trunc: int) -> str: + return self.substrate.read_trajectory( + run_dir / "stream.jsonl", obs_trunc=obs_trunc, max_chars=200_000 + ) + + def deliverable_files(self, run_dir: Path) -> dict[str, str]: + kind = json.loads((run_dir / "meta.json").read_text())["deliverable"] + if kind == "files": + paths = { + p.relative_to(run_dir / "output").as_posix(): p + for p in sorted((run_dir / "output").rglob("*")) + if p.is_file() + } + else: + name = "deliverable.patch" if kind == "diff" else "final.md" + paths = {name: run_dir / name} + files = {} + for name, path in paths.items(): + if not path.is_file(): + continue + data = path.read_bytes() + try: + files[name] = data.decode("utf-8") + except UnicodeDecodeError: + files[name] = f"(binary file, {len(data)} bytes)" + return files + + def deliverable_text(self, run_dir: Path) -> str: + files = self.deliverable_files(run_dir) + text = "\n\n".join(files.values()) + if "deliverable.patch" in files: + text += "\n\n" + (run_dir / "final.md").read_text(encoding="utf-8") + return text + + def validate_tree(self, tree: dict[str, str]) -> None: + pass + + def has_judge(self, task_id: str) -> bool: + config = _read_task(self.tasks_dir, task_id) + return bool(config.get("check") or (self.tasks_dir / task_id / "rubric.md").is_file()) + + def score(self, run_dir: Path, task_id: str) -> dict | None: + config = _read_task(self.tasks_dir, task_id) + task_dir = self.tasks_dir / task_id + if config.get("check"): + check_dir = task_dir / "check" + digest = hashlib.sha256() + for path in sorted(check_dir.rglob("*")): + digest.update(path.relative_to(check_dir).as_posix().encode() + b"\0") + digest.update(b"d" if path.is_dir() else b"f" + path.read_bytes()) + digest.update(b"\0") + identity = {"check": config["check"], "check_sha256": digest.hexdigest()} + elif (task_dir / "rubric.md").is_file(): + judge_channel = _channel(self.settings, "judge") + identity = { + "channel": judge_channel.kind, + "model": judge_channel.model, + "rubric_sha256": hashlib.sha256((task_dir / "rubric.md").read_bytes()).hexdigest(), + } + else: + return None + cache = run_dir / "judge.json" + if cache.is_file(): + try: + cached = json.loads(cache.read_text(encoding="utf-8")) + except (OSError, ValueError): + cached = None + if isinstance(cached, dict) and cached.get("identity") == identity: + return cached + cache.unlink(missing_ok=True) + result = None + if config.get("check"): + with self._workspace(task_id, config) as ws: + if config["workspace"] == "project": + try: + workspace.apply_patch(ws, (run_dir / "deliverable.patch").read_text()) + except (OSError, subprocess.CalledProcessError): + return None + elif (run_dir / "output").is_dir(): + shutil.copytree(run_dir / "output", ws / "output", dirs_exist_ok=True) + if (task_dir / "check").is_dir(): + shutil.copytree(task_dir / "check", ws / "check", dirs_exist_ok=True) + try: + process = subprocess.run( + config["check"], + shell=True, + cwd=ws, + text=True, + capture_output=True, + timeout=config.get("timeout") or self.executor["timeout"], + env=os.environ + | {"HARNESS2_RUN_DIR": str(run_dir), "HARNESS2_TASK_DIR": str(task_dir)}, + ) + code, output, stderr = process.returncode, process.stdout, process.stderr + except subprocess.TimeoutExpired as exc: + code = -1 + output = (exc.stdout or b"").decode("utf-8", errors="replace") + stderr = (exc.stderr or b"").decode("utf-8", errors="replace") + lines = output.strip().splitlines() + try: + counts = _judge_counts(json.loads(lines[-1])) if lines else None + except ValueError: + counts = None + result = counts or {"n_passed": int(code == 0), "n_criteria": 1} + result.update(exit_code=code, output_tail=(output + stderr)[-2000:]) + elif (task_dir / "rubric.md").is_file(): + with tempfile.TemporaryDirectory(prefix="harness2-judge-") as tmp: + context = Path(tmp) + shutil.copyfile(task_dir / "task.md", context / "task.md") + shutil.copyfile(task_dir / "rubric.md", context / "rubric.md") + dest = context / "deliverables" + dest.mkdir() + if config["deliverable"] == "files" and (run_dir / "output").is_dir(): + shutil.copytree(run_dir / "output", dest, dirs_exist_ok=True) + else: + name = "deliverable.patch" if config["deliverable"] == "diff" else "final.md" + if (run_dir / name).is_file(): + shutil.copyfile(run_dir / name, dest / name) + shutil.copyfile(run_dir / "final.md", dest / "final.md") + try: + reply = Proposer(judge_channel).call( + mission=RUBRIC_MISSION, + context_dir=context, + label=f"judge.{task_id}.{slug(run_dir.name)}", + ) + except ProposerError: + return None + blocks = re.findall(r"```json\s*(.*?)```", reply.text, flags=re.S) + result = _judge_counts(_parse_json(blocks[-1] if blocks else reply.text)) + if result is not None: + result["all_pass"] = result["n_passed"] == result["n_criteria"] + result["identity"] = identity + cache.write_text(json.dumps(result, indent=2), encoding="utf-8") + return result + + +def _judge_counts(value: object) -> dict | None: + if not isinstance(value, dict): + return None + passed, total = value.get("n_passed"), value.get("n_criteria") + if type(passed) is not int or type(total) is not int or not 0 <= passed <= total or total <= 0: + return None + result = {"n_passed": passed, "n_criteria": total} + if isinstance(value.get("notes"), str): + result["notes"] = value["notes"] + return result + + +def make_adapter(ctx: SuiteContext) -> GenericAdapter: + return GenericAdapter(ctx) + + +def check(ctx: SuiteContext) -> list[str]: + problems = [] + settings = ctx.settings + executor = settings.get("executor.name") + if executor not in ("claude", "codex", "command"): + problems.append("the tasks suite runs on claude, codex or command") + elif executor != "command": + try: + cli_argv(settings.get("executor.bin"), executor) + except ConfigError as exc: + problems.append(str(exc)) + elif not settings.get("executor.command") or not settings.get("executor.editable"): + problems.append("command executor needs executor.command and non-empty executor.editable") + for role in ("executor", "improver", "selector", "reflector", "judge"): + if role == "executor": + if executor == "command": + continue + values = settings.section(role) + kind = executor + else: + values = settings.role(role) + kind = values["channel"] + if kind == "human": + if role != "selector": + problems.append("the human channel is only allowed for selector") + continue + try: + if role != "executor": + _channel(settings, role) + provider_name, _ = providers.split_model(values["model"]) + problems.extend(providers.check(settings.provider(provider_name))) + except ConfigError as exc: + problems.append(f"{role}: {exc}") + tasks_dir = settings.path("paths.tasks") + for path in sorted(tasks_dir.iterdir()) if tasks_dir.is_dir() else []: + if not path.is_dir(): + continue + try: + config = _read_task(tasks_dir, path.name) + if (path / "task.md").read_text(encoding="utf-8").startswith("TODO:"): + problems.append(f"task {path.name}: task.md still holds the template text") + if config["workspace"] == "project": + result = subprocess.run( + ["git", "-C", str(ctx.project), "rev-parse", "--show-toplevel"], + capture_output=True, + ) + if result.returncode: + problems.append(f"task {path.name}: project workspace needs a git repository") + except (ConfigError, OSError) as exc: + problems.append(str(exc)) + return list(dict.fromkeys(problems)) diff --git a/harness2/workspace.py b/harness2/workspace.py new file mode 100644 index 00000000..51b46414 --- /dev/null +++ b/harness2/workspace.py @@ -0,0 +1,110 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Temporary project worktrees and file workspaces for rollouts and judges.""" + +from __future__ import annotations + +import shutil +import subprocess +import tempfile +from collections.abc import Iterator, Sequence +from contextlib import contextmanager +from pathlib import Path + +from .settings import ConfigError + + +@contextmanager +def project_workspace(project: Path, ref: str) -> Iterator[Path]: + ws = Path(tempfile.mkdtemp(prefix="harness2-ws-")).resolve() + try: + result = subprocess.run( + ["git", "-C", str(project), "worktree", "add", "--detach", "--", str(ws), ref], + capture_output=True, + text=True, + ) + if result.returncode: + raise ConfigError( + f"cannot create project workspace at {ref!r}: {result.stderr.strip()}" + ) + yield ws + finally: + subprocess.run( + ["git", "-C", str(project), "worktree", "remove", "--force", str(ws)], + capture_output=True, + ) + shutil.rmtree(ws, ignore_errors=True) + + +@contextmanager +def files_workspace(src: Path | None) -> Iterator[Path]: + ws = Path(tempfile.mkdtemp(prefix="harness2-ws-")).resolve() + try: + if src is not None: + shutil.copytree(src, ws, dirs_exist_ok=True) + (ws / "output").mkdir(exist_ok=True) + yield ws + finally: + shutil.rmtree(ws, ignore_errors=True) + + +def commit_all(ws: Path, message: str) -> None: + subprocess.run(["git", "add", "-A"], cwd=ws, check=True, capture_output=True) + subprocess.run( + [ + "git", + "-c", + "user.name=harness2", + "-c", + "user.email=harness2@localhost", + "-c", + "commit.gpgsign=false", + "commit", + "--no-verify", + "--allow-empty", + "-qm", + message, + ], + cwd=ws, + check=True, + capture_output=True, + ) + + +def diff_since(ws: Path, base: str, exclude: Sequence[str]) -> str: + subprocess.run(["git", "add", "-A"], cwd=ws, check=True, capture_output=True) + pathspecs = [ + p.replace(":(", ":(exclude,", 1) if p.startswith(":(") else f":(exclude){p}" + for p in exclude + ] + return subprocess.run( + ["git", "diff", "--cached", "--binary", base, "--", ".", *pathspecs], + cwd=ws, + check=True, + capture_output=True, + text=True, + ).stdout + + +def apply_patch(ws: Path, patch: str) -> None: + if patch.strip(): + subprocess.run( + ["git", "apply", "--whitespace=nowarn"], + cwd=ws, + input=patch, + text=True, + check=True, + capture_output=True, + ) diff --git a/tests/test_cli.py b/tests/test_cli.py index acae6d49..f4ee54af 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -428,7 +428,10 @@ def test_config_task_setup_and_json(self): fake.new_task = Mock(return_value=self.project / "task.md") fake.list_tasks = Mock(return_value=rows) fake.import_tasks = Mock(return_value=["x"]) - with patch.dict(sys.modules, {"harness2.suite": fake}): + with ( + patch.dict(sys.modules, {"harness2.suite": fake}), + patch("harness2.suite", fake, create=True), + ): self.assertEqual( self.invoke( "--json", "task", "new", "x", "--files", "--domain", "d", "--check", "true" diff --git a/tests/test_suite.py b/tests/test_suite.py new file mode 100644 index 00000000..daa3bb72 --- /dev/null +++ b/tests/test_suite.py @@ -0,0 +1,670 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The generic suite against fake CLIs, local git projects, and the real scheduler.""" + +from __future__ import annotations + +import csv +import json +import os +import shlex +import shutil +import subprocess +import sys +import tempfile +import unittest +import warnings +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from pathlib import Path +from unittest.mock import patch + +from harness2 import registry, suite, workspace +from harness2.core import events, runmark, schedule, tree +from harness2.core.proposer import Proposer +from harness2.core.spec import BenchmarkAdapter +from harness2.settings import ConfigError, load + +FAKE_CLI = r"""import json +import os +import sys +import time +from pathlib import Path + +if "--version" in sys.argv: + print("fake cli 1.0") + sys.exit(0) +cwd = Path.cwd() +prompt = sys.stdin.read() +record = {"cwd": str(cwd), "argv": sys.argv[1:], "prompt": prompt, + "claude_home": os.environ.get("CLAUDE_CONFIG_DIR"), + "codex_home": os.environ.get("CODEX_HOME"), + "harness": {name: (cwd / name).read_text() for name in ("CLAUDE.md", "AGENTS.md") + if (cwd / name).is_file()}} +if record["codex_home"]: + record["config"] = (Path(record["codex_home"]) / "config.toml").read_text() +with open(os.environ["FAKE_RECORD"], "a") as stream: + stream.write(json.dumps(record) + "\n") +if "--output-format" in sys.argv and sys.argv[sys.argv.index("--output-format") + 1] == "json": + context = cwd / "context" + if (context / "rubric.md").is_file(): + assert (context / "task.md").is_file() + assert (context / "deliverables/final.md").is_file() + text = os.environ.get("FAKE_JUDGE", '{"n_passed": 2, "n_criteria": 3, "notes": "one missing"}') + elif (context / "candidates").is_dir(): + chosen = sorted((context / "candidates").glob("*.md"))[0].stem + text = "Complete change.\n```choice\n" + chosen + "\n```" + elif (context / "adoption.json").is_file(): + text = "NO_ENTRY" + else: + harness = context / "harness/CLAUDE.md" + harness.write_text(harness.read_text() + "Run the project tests.\n") + text = "Use the project's tests." + print(json.dumps({"type": "result", "subtype": "success", "result": text, + "total_cost_usd": 0.05})) + sys.exit(0) +if Path(sys.argv[0]).name == "codex" and (cwd / "context/rubric.md").is_file(): + print(json.dumps({"type": "item.completed", "item": { + "id": "judge", "type": "agent_message", "text": os.environ["FAKE_JUDGE"]}})) + sys.exit(0) +assert not (cwd / ".harness2").exists(), "project state leaked" +assert not (cwd / "check").exists(), "checks leaked" +assert not (cwd / "rubric.md").exists(), "rubric leaked" +(cwd / "code.txt").write_text("after\n") +for name in ("CLAUDE.md", "AGENTS.md"): + if (cwd / name).is_file(): + (cwd / name).write_text("agent changed instructions\n") +(cwd / ".harness2").mkdir() +(cwd / ".harness2/leak").write_text("private state") +(cwd / "output").mkdir(exist_ok=True) +(cwd / "output/report.txt").write_text("report\n") +if Path(sys.argv[0]).name == "codex": + print(json.dumps({"type": "item.completed", "item": { + "id": "1", "type": "agent_message", "text": "Finished the change."}}), flush=True) + print(json.dumps({"type": "turn.completed", "usage": { + "input_tokens": 10, "cached_input_tokens": 0, "output_tokens": 5}}), flush=True) +else: + print(json.dumps({"type": "system", "subtype": "init", "model": "fake"}), flush=True) + print(json.dumps({"type": "assistant", "message": {"content": [ + {"type": "tool_use", "name": "Write", "input": {"file_path": "code.txt"}}, + {"type": "text", "text": "Finished the change."}]}}), flush=True) + print(json.dumps({"type": "result", "subtype": "success", "result": "Finished the change.", + "total_cost_usd": 0.125}), flush=True) +time.sleep(float(os.environ.get("FAKE_SLEEP", "0"))) +sys.exit(int(os.environ.get("FAKE_RC", "0"))) +""" + + +class SuiteTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name).resolve() + self.project = self.root / "project" + self.project.mkdir() + (self.project / ".harness2/tasks").mkdir(parents=True) + self.tasks_dir = self.project / ".harness2/tasks" + (self.project / "code.txt").write_text("before\n") + (self.project / "CLAUDE.md").write_text("Read the code first.\n") + (self.project / "AGENTS.md").write_text("Read the code first.\n") + subprocess.run(["git", "init", "-q", str(self.project)], check=True) + binary = self.root / "bin" + binary.mkdir() + for name in ("claude", "codex"): + path = binary / name + path.write_text(f"#!{sys.executable}\n" + FAKE_CLI) + path.chmod(0o755) + self.record = self.root / "records.jsonl" + env = {k: v for k, v in os.environ.items() if not k.startswith(("HARNESS2_", "FAKE_"))} + env.update( + PATH=f"{binary}{os.pathsep}{env['PATH']}", + FAKE_RECORD=str(self.record), + XDG_CONFIG_HOME=str(self.root / "config"), + ANTHROPIC_API_KEY="fake-key", + OPENAI_API_KEY="fake-key", + ) + environment = patch.dict(os.environ, env, clear=True) + environment.start() + self.addCleanup(environment.stop) + self.addCleanup(events.start, None) + events.start(None) + self.task("one", domain="coding", split="train") + workspace.commit_all(self.project, "initial") + self.ctx = self.context() + self.adapter = suite.make_adapter(self.ctx) + + def context(self, **overrides): + values = {"executor.model": "anthropic/fake", "improver.model": "anthropic/fake"} + values.update(overrides) + settings = load( + self.project, overrides={key: (value, "test") for key, value in values.items()} + ) + return registry.SuiteContext(settings, self.project, self.project / ".harness2/runs/test") + + def task(self, task_id, *, files=False, **values): + task_dir = suite.new_task( + self.tasks_dir, + task_id, + files=files, + domain=values.pop("domain", "default"), + check=values.pop("check", None), + ) + (task_dir / "task.md").write_text("Change code.txt as requested.\n") + with (task_dir / "task.toml").open("a") as stream: + for key, value in values.items(): + stream.write(f"{key} = {json.dumps(value)}\n") + return task_dir + + def records(self): + return [json.loads(line) for line in self.record.read_text().splitlines()] + + def test_module_task_protocol_splits_and_prompt_profiles(self): + self.assertIs(registry.load("tasks"), suite) + self.assertIsInstance(self.adapter, BenchmarkAdapter) + self.assertEqual(self.adapter.tasks(), ["one"]) + self.assertEqual(self.adapter.tasks("train"), ["one"]) + self.assertEqual(self.adapter.tasks("test"), []) + self.assertEqual(self.adapter.domain_of("one"), "coding") + self.assertIsNone(self.adapter.task_files("one")) + self.assertEqual(self.adapter.task_prompt("one"), "Change code.txt as requested.\n") + self.assertIs(self.adapter.spec.prompts, suite.PROJECT_PROMPTS) + self.assertEqual(self.adapter.prompts_for("one"), suite.PROJECT_PROMPTS) + self.adapter.validate_tree({}) + files = self.task("docs", files=True, deliverable="message") + self.assertEqual(self.adapter.task_files("docs"), files / "files") + (self.project / "steer.txt").write_text("Use local conventions.") + ctx = self.context(**{"improver.steer": "@steer.txt", "suite.work": "Maintain the project"}) + prompts = suite.make_adapter(ctx).prompts_for("docs") + self.assertEqual(prompts.work, "Maintain the project") + self.assertEqual(prompts.deliverable_word, "answer") + self.assertEqual(prompts.domain_guidance, "Use local conventions.") + self.assertEqual(prompts.yardstick, suite.FILES_PROMPTS.yardstick) + for overrides in ({"suite.work": "rubric"}, {"improver.steer": "score"}): + with self.assertRaisesRegex(ConfigError, "improver.steer / suite.work:"): + suite.make_adapter(self.context(**overrides)).prompts_for("one") + with self.assertRaisesRegex(ConfigError, "cannot read improver.steer"): + suite.make_adapter(self.context(**{"improver.steer": "@missing"})).prompts_for("one") + with self.assertRaises(ConfigError): + self.adapter.tasks("missing") + + def test_claude_execute_capture_idempotence_and_isolation(self): + self.assertFalse(self.adapter.delivered(self.root / "missing")) + run_dir = self.adapter.execute("one", None, "nested/run") + self.assertEqual(run_dir.name, "nested__run") + self.assertTrue(runmark.is_complete(run_dir, harness_dir=self.adapter.base_harness())) + self.assertTrue(self.adapter.delivered(run_dir)) + self.assertEqual(self.adapter.rollout_cost(run_dir), 0.125) + artifacts = self.adapter.deliverable_files(run_dir) + patch_text = artifacts["deliverable.patch"] + self.assertIn("+after", patch_text) + self.assertNotIn("CLAUDE.md", patch_text) + self.assertNotIn(".harness2", patch_text) + self.assertEqual( + self.adapter.deliverable_text(run_dir), patch_text + "\n\nFinished the change." + ) + self.assertIn("[ACTION] Write:", self.adapter.trajectory(run_dir, obs_trunc=10000)) + meta = json.loads((run_dir / "meta.json").read_text()) + self.assertTrue(meta["ok"]) + self.assertEqual(meta["exit_code"], 0) + first = self.records()[0] + self.assertEqual(first["prompt"], self.adapter.task_prompt("one")) + self.assertEqual(first["claude_home"], str(run_dir / "claude-home")) + self.assertFalse(Path(first["cwd"]).exists()) + self.assertEqual(self.adapter.execute("one", None, "nested/run"), run_dir) + self.assertEqual(len(self.records()), 1) + self.assertEqual((self.project / "code.txt").read_text(), "before\n") + (run_dir / "deliverable.patch").write_text("") + self.assertFalse(self.adapter.delivered(run_dir)) + + def test_changed_candidate_reexecutes_and_drops_stale_judge(self): + candidate = self.root / "candidate" + surface = self.adapter.substrate.surface() + tree.write_tree(surface, {"CLAUDE.md": "First candidate\n"}, candidate) + run_dir = self.adapter.execute("one", candidate, "candidate") + (run_dir / "judge.json").write_text('{"n_passed": 0}') + (candidate / "CLAUDE.md").write_text("Second candidate\n") + self.adapter.execute("one", candidate, "candidate") + self.assertEqual(len(self.records()), 2) + self.assertFalse((run_dir / "judge.json").exists()) + + def test_codex_stream_and_provider_home(self): + ctx = self.context(**{"executor.name": "codex", "executor.model": "openai/fake"}) + adapter = suite.make_adapter(ctx) + run_dir = adapter.execute("one", None, "codex") + self.assertTrue(adapter.delivered(run_dir)) + self.assertEqual(adapter.rollout_cost(run_dir), 0) + self.assertNotIn("AGENTS.md", adapter.deliverable_text(run_dir)) + self.assertIn("Finished the change.", adapter.deliverable_text(run_dir)) + self.assertIn("tokens in=10", adapter.trajectory(run_dir, obs_trunc=10000)) + record = self.records()[0] + self.assertEqual(record["codex_home"], str(run_dir / "codex-home")) + self.assertIn('model = "fake"', record["config"]) + self.assertEqual(record["argv"][-1], "-") + self.assertFalse(adapter.spec.allow_subagents) + + def test_agent_failure_and_timeout_are_completed(self): + for run_id, env, kwargs, expected in ( + ("failed", {"FAKE_RC": "7"}, {}, 7), + ("timeout", {"FAKE_SLEEP": "2"}, {"timeout": 0.1}, -1), + ): + with self.subTest(run_id=run_id), patch.dict(os.environ, env): + run_dir = self.adapter.execute("one", None, run_id, **kwargs) + meta = json.loads((run_dir / "meta.json").read_text()) + self.assertFalse(meta["ok"]) + self.assertEqual(meta["exit_code"], expected) + self.assertEqual(meta["timed_out"], run_id == "timeout") + self.assertTrue( + runmark.is_complete(run_dir, harness_dir=self.adapter.base_harness()) + ) + count = len(self.records()) + self.adapter.execute("one", None, run_id, **kwargs) + self.assertEqual(len(self.records()), count) + + def test_task_timeout_takes_precedence(self): + self.task("bounded", timeout=1) + with patch.dict(os.environ, {"FAKE_SLEEP": "0.02"}): + run_dir = self.adapter.execute("bounded", None, "bounded", timeout=0.001) + self.assertTrue(json.loads((run_dir / "meta.json").read_text())["ok"]) + + def test_files_command_executor_and_binary_deliverables(self): + task_dir = self.task("files", files=True, check="test -s output/report.txt") + (task_dir / "files/source.txt").write_text("supplied source\n") + script = self.root / "command.py" + script.write_text( + "import pathlib, sys\n" + "ws, prompt, output = map(pathlib.Path, sys.argv[1:])\n" + "assert ws == pathlib.Path.cwd()\n" + "assert 'Save your deliverables in the output/ directory.' in prompt.read_text()\n" + "(output / 'report.txt').write_text((ws / 'source.txt').read_text())\n" + "(output / 'binary.dat').write_bytes(b'\\xff\\x00')\n" + "print('Saved report.')\n" + ) + command = f"{shlex.quote(sys.executable)} {shlex.quote(str(script))} {{workspace}} {{prompt_file}} {{output}}" + adapter = suite.make_adapter( + self.context( + **{ + "executor.name": "command", + "executor.command": command, + "executor.editable": ["instructions.txt"], + } + ) + ) + (self.project / "instructions.txt").write_text("Use the source.\n") + run_dir = adapter.execute("files", None, "files") + self.assertTrue(adapter.delivered(run_dir)) + self.assertEqual(adapter.rollout_cost(run_dir), 0) + self.assertEqual( + adapter.deliverable_files(run_dir), + { + "binary.dat": "(binary file, 2 bytes)", + "report.txt": "supplied source\n", + }, + ) + self.assertIn("supplied source", adapter.deliverable_text(run_dir)) + self.assertIn("Saved report.", adapter.trajectory(run_dir, obs_trunc=10000)) + self.assertTrue(adapter.score(run_dir, "files")["all_pass"]) + self.assertTrue(adapter.has_judge("files")) + shutil.rmtree(run_dir / "output") + self.assertFalse(adapter.delivered(run_dir)) + + def test_message_deliverable(self): + self.task("message", deliverable="message") + run_dir = self.adapter.execute("message", None, "message") + self.assertTrue(self.adapter.delivered(run_dir)) + self.assertEqual(self.adapter.deliverable_text(run_dir), "Finished the change.") + self.assertEqual( + self.adapter.deliverable_files(run_dir), {"final.md": "Finished the change."} + ) + self.assertFalse(self.adapter.has_judge("message")) + self.assertIsNone(self.adapter.score(run_dir, "message")) + self.assertFalse((run_dir / "judge.json").exists()) + + def test_check_judge_fresh_patch_json_exit_code_cache_and_failure(self): + task_dir = self.tasks_dir / "one" + (task_dir / "check").mkdir() + checker = task_dir / "check/probe.py" + checker.write_text( + "import os, pathlib\n" + "assert pathlib.Path('code.txt').read_text() == 'after\\n'\n" + "assert pathlib.Path('CLAUDE.md').read_text() == 'Read the code first.\\n'\n" + "assert pathlib.Path(os.environ['HARNESS2_RUN_DIR']).is_dir()\n" + "assert pathlib.Path(os.environ['HARNESS2_TASK_DIR']).is_dir()\n" + "print('detail')\n" + 'print(\'{"n_passed": 2, "n_criteria": 3}\')\n' + ) + with (task_dir / "task.toml").open("a") as stream: + stream.write(f"check = {json.dumps(shlex.quote(sys.executable) + ' check/probe.py')}\n") + self.assertTrue(self.adapter.has_judge("one")) + run_dir = self.adapter.execute("one", None, "judge") + result = self.adapter.score(run_dir, "one") + self.assertEqual((result["n_passed"], result["n_criteria"], result["exit_code"]), (2, 3, 0)) + self.assertFalse(result["all_pass"]) + with patch("harness2.suite.workspace.project_workspace") as create_workspace: + self.assertEqual(self.adapter.score(run_dir, "one"), result) + create_workspace.assert_not_called() + checker.write_text("raise RuntimeError('changed checker')") + changed = self.adapter.score(run_dir, "one") + self.assertEqual(changed["n_passed"], 0) + self.assertNotEqual(changed["identity"]["check_sha256"], result["identity"]["check_sha256"]) + for command, expected in ( + ("exit 0", 1), + ("exit 3", 0), + ('echo \'{"n_passed": true, "n_criteria": 1}\'; exit 4', 0), + ): + (task_dir / "task.toml").write_text(f"check = {json.dumps(command)}\n") + result = self.adapter.score(run_dir, "one") + self.assertEqual((result["n_passed"], result["n_criteria"]), (expected, 1)) + self.assertEqual(result["identity"]["check"], command) + (run_dir / "judge.json").unlink() + (run_dir / "deliverable.patch").write_text("not a patch") + self.assertIsNone(self.adapter.score(run_dir, "one")) + self.assertFalse((run_dir / "judge.json").exists()) + + def test_check_timeout_is_a_failed_judgment(self): + task_dir = self.task("slow", timeout=1) + command = f"exec {shlex.quote(sys.executable)} -c " + shlex.quote( + "import time; print('started', flush=True); time.sleep(2)" + ) + with (task_dir / "task.toml").open("a") as stream: + stream.write(f"check = {json.dumps(command)}\n") + run_dir = self.adapter.execute("slow", None, "slow") + result = self.adapter.score(run_dir, "slow") + self.assertEqual( + (result["n_passed"], result["n_criteria"], result["exit_code"]), (0, 1, -1) + ) + self.assertIn("started", result["output_tail"]) + + def test_check_cache_hashes_contents_names_and_empty_directories(self): + task_dir = self.task("check-identity", check="test -f check/ready") + check_dir = task_dir / "check" + check_dir.mkdir() + ready = check_dir / "ready" + ready.write_bytes(b"first\0content") + run_dir = self.adapter.execute("check-identity", None, "check-identity") + first = self.adapter.score(run_dir, "check-identity") + self.assertEqual(json.loads((run_dir / "judge.json").read_text()), first) + ready.write_bytes(b"second\0content") + second = self.adapter.score(run_dir, "check-identity") + self.assertNotEqual(first["identity"], second["identity"]) + ready.rename(check_dir / "renamed") + renamed = self.adapter.score(run_dir, "check-identity") + self.assertFalse(renamed["all_pass"]) + self.assertNotEqual(second["identity"], renamed["identity"]) + (check_dir / "empty").mkdir() + empty = self.adapter.score(run_dir, "check-identity") + self.assertNotEqual(renamed["identity"], empty["identity"]) + (check_dir / "compiled.pyc").write_bytes(b"compiled check") + compiled = self.adapter.score(run_dir, "check-identity") + self.assertNotEqual(empty["identity"], compiled["identity"]) + for legacy in ('{"n_passed": 1, "n_criteria": 1}', "invalid json", "[]"): + (run_dir / "judge.json").write_text(legacy) + self.assertEqual(self.adapter.score(run_dir, "check-identity"), compiled) + + def test_rubric_cache_regrades_after_model_channel_or_rubric_changes(self): + task_dir = self.tasks_dir / "one" + rubric = task_dir / "rubric.md" + rubric.write_text("Three criteria.\n") + run_dir = self.adapter.execute("one", None, "rubric-identity") + first = self.adapter.score(run_dir, "one") + self.assertEqual(first["identity"]["channel"], "claude") + self.assertEqual(first["identity"]["model"], "anthropic/fake") + calls = len(self.records()) + output = '{"n_passed": 3, "n_criteria": 3}' + with patch.dict(os.environ, {"FAKE_JUDGE": output}): + self.assertEqual(self.adapter.score(run_dir, "one"), first) + self.assertEqual(len(self.records()), calls) + changed_model = suite.make_adapter(self.context(**{"judge.model": "anthropic/other"})) + second = changed_model.score(run_dir, "one") + self.assertEqual(second["identity"]["model"], "anthropic/other") + self.assertTrue(second["all_pass"]) + self.assertEqual(len(self.records()), calls + 1) + self.assertEqual( + second["identity"]["rubric_sha256"], first["identity"]["rubric_sha256"] + ) + with patch.dict(os.environ, {"FAKE_JUDGE": '{"n_passed": 0, "n_criteria": 3}'}): + rubric.write_text("Three changed criteria.\n") + third = changed_model.score(run_dir, "one") + self.assertEqual(third["n_passed"], 0) + self.assertNotEqual( + third["identity"]["rubric_sha256"], second["identity"]["rubric_sha256"] + ) + changed_channel = suite.make_adapter( + self.context( + **{ + "judge.channel": "codex", + "judge.model": "openai/fake", + } + ) + ) + fourth = changed_channel.score(run_dir, "one") + self.assertEqual(fourth["identity"]["channel"], "codex") + self.assertEqual(fourth["identity"]["model"], "openai/fake") + self.assertEqual(fourth["n_passed"], 0) + self.assertEqual(len(self.records()), calls + 3) + + def test_rubric_judge_json_fence_bare_and_unparseable(self): + task_dir = self.tasks_dir / "one" + (task_dir / "rubric.md").write_text("Three criteria.\n") + self.assertTrue(self.adapter.has_judge("one")) + run_dir = self.adapter.execute("one", None, "rubric") + for text, expected in ( + ('{"n_passed": 2, "n_criteria": 3, "notes": "one missing"}', 2), + ( + 'Reasoning\n```json\n{"n_passed": 0, "n_criteria": 3}\n```\n' + '```json\n{"n_passed": 3, "n_criteria": 3}\n```', + 3, + ), + ('Reasoning\n{"n_passed": 1, "n_criteria": 3}', 1), + ('{"n_passed": 4, "n_criteria": 3}', None), + ("not json", None), + ): + with self.subTest(text=text), patch.dict(os.environ, {"FAKE_JUDGE": text}): + (run_dir / "judge.json").unlink(missing_ok=True) + result = self.adapter.score(run_dir, "one") + if expected is None: + self.assertIsNone(result) + self.assertFalse((run_dir / "judge.json").exists()) + else: + self.assertEqual(result["n_passed"], expected) + + def test_base_harness_is_stored_warns_once_and_survives_relocation(self): + base = self.adapter.base_harness() + original = tree.read_tree(self.adapter.substrate.surface(), base) + (self.project / "CLAUDE.md").write_text("changed harness\n") + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.assertEqual(self.adapter.base_harness(), base) + self.adapter.base_harness() + self.assertEqual(len(caught), 1) + self.assertEqual(str(caught[0].message), suite.BASE_DRIFT_WARNING.format(path=base)) + self.assertEqual(tree.read_tree(self.adapter.substrate.surface(), base), original) + self.adapter.execute("one", None, "stored-base") + self.assertEqual(self.records()[-1]["harness"]["CLAUDE.md"], original["CLAUDE.md"]) + other = suite.make_adapter(self.ctx) + with self.assertWarns(UserWarning): + self.assertEqual(other.base_harness(), base) + new_root = self.project / ".harness2/runs/moved" + shutil.move(self.ctx.run_root, new_root) + moved = suite.make_adapter(replace(self.ctx, run_root=new_root)) + with self.assertWarns(UserWarning): + self.assertEqual(moved.base_harness(), new_root / "base_harness") + (self.project / "CLAUDE.md").unlink() + empty = suite.make_adapter(replace(self.ctx, run_root=self.root / "empty")) + empty.base_harness() + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + empty.base_harness() + self.assertEqual(caught, []) + + def test_concurrent_base_harness_has_one_complete_snapshot(self): + with ThreadPoolExecutor(max_workers=4) as pool: + paths = list(pool.map(lambda _: self.adapter.base_harness(), range(8))) + self.assertEqual(len(set(paths)), 1) + self.assertEqual((paths[0] / "CLAUDE.md").read_text(), "Read the code first.\n") + + def test_task_validation_new_import_jsonl_csv_and_force(self): + self.assertEqual(suite.list_tasks(self.root / "missing"), []) + with self.assertRaisesRegex(ConfigError, "already exists"): + suite.new_task(self.tasks_dir, "one", files=False, domain="x", check=None) + for task_id in ("../escape", "Upper", "", "/absolute"): + with self.assertRaises(ConfigError): + suite.new_task(self.tasks_dir, task_id, files=False, domain="x", check=None) + source = self.root / "tasks.jsonl" + source.write_text( + "\n".join( + json.dumps(row) + for row in ( + { + "id": "two", + "prompt": "Second request", + "domain": "docs", + "split": "test", + "workspace": "files", + "deliverable": "files", + "timeout": 10, + }, + {"id": "three", "prompt": "Third request", "check": "exit 0"}, + ) + ) + ) + self.assertEqual(suite.import_tasks(self.tasks_dir, source, force=False), ["two", "three"]) + self.assertEqual((self.tasks_dir / "two/task.md").read_text(), "Second request") + self.assertEqual(self.adapter.tasks("test"), ["two"]) + self.assertEqual(self.adapter.task_files("two"), self.tasks_dir / "two/files") + with self.assertRaisesRegex(ConfigError, "already exists"): + suite.import_tasks(self.tasks_dir, source, force=False) + self.assertEqual(suite.import_tasks(self.tasks_dir, source, force=True), ["two", "three"]) + source = self.root / "tasks.csv" + with source.open("w", newline="") as stream: + writer = csv.DictWriter( + stream, fieldnames=["id", "prompt", "timeout", "split", "check"] + ) + writer.writeheader() + writer.writerow({"id": "four", "prompt": "First line\nSecond line", "timeout": 12}) + self.assertEqual(suite.import_tasks(self.tasks_dir, source, force=False), ["four"]) + self.assertEqual((self.tasks_dir / "four/task.md").read_text(), "First line\nSecond line") + self.assertEqual(suite._read_task(self.tasks_dir, "four")["timeout"], 12) + for values in ( + {"unknown": 1}, + {"split": "all"}, + {"workspace": "bad"}, + {"deliverable": "bad"}, + {"timeout": 0}, + {"timeout": True}, + {"domain": 1}, + ): + with self.subTest(values=values): + (self.tasks_dir / "four/task.toml").write_text( + "".join(f"{k} = {json.dumps(v)}\n" for k, v in values.items()) + ) + with self.assertRaises(ConfigError): + suite.list_tasks(self.tasks_dir) + (self.tasks_dir / "four/task.toml").write_text("bad toml") + with self.assertRaises(ConfigError): + suite.list_tasks(self.tasks_dir) + source = self.root / "bad.jsonl" + for content in ( + '{"id": "bad"}', + '{"id": "../escape", "prompt": "x"}', + "[1]", + "not json", + '{"id":"dup", "prompt":"x"}\n{"id":"dup", "prompt":"y"}', + ): + source.write_text(content) + with self.assertRaises(ConfigError): + suite.import_tasks(self.tasks_dir, source, force=False) + self.assertFalse((self.tasks_dir / "dup").exists()) + + def test_force_import_rejects_symlinks_and_malformed_csv(self): + outside = self.root / "outside.md" + outside.write_text("keep this") + (self.tasks_dir / "one/task.md").unlink() + (self.tasks_dir / "one/task.md").symlink_to(outside) + source = self.root / "force.jsonl" + source.write_text('{"id": "one", "prompt": "replace"}') + with self.assertRaisesRegex(ConfigError, "symlink"): + suite.import_tasks(self.tasks_dir, source, force=True) + self.assertEqual(outside.read_text(), "keep this") + source = self.root / "missing-column.csv" + source.write_text("id,prompt,timeout\nnew,request\n") + with self.assertRaisesRegex(ConfigError, "timeout must be a positive integer"): + suite.import_tasks(self.tasks_dir, source, force=False) + + def test_preflight_reports_all_invalid_tasks_and_config(self): + self.assertEqual(suite.check(self.ctx), []) + self.task("template") + (self.tasks_dir / "template/task.md").write_text(suite.TASK_TEMPLATE) + self.task("invalid") + (self.tasks_dir / "invalid/task.toml").write_text('unknown = "key"\n') + problems = suite.check( + self.context(**{"executor.bin": "nonexistent-executor", "judge.model": "bad"}) + ) + self.assertTrue(any("binary not found" in p for p in problems)) + self.assertTrue(any("judge: model" in p for p in problems)) + self.assertTrue( + any("task template: task.md still holds the template text" == p for p in problems) + ) + self.assertTrue(any("unknown task.toml key 'unknown'" in p for p in problems)) + human = self.context(**{"selector.channel": "human"}) + self.assertEqual(suite.make_adapter(human).spec.channels["selector"].kind, "human") + with self.assertRaisesRegex(ConfigError, "only allowed for selector"): + suite.make_adapter(self.context(**{"improver.channel": "human"})) + with self.assertRaisesRegex(ConfigError, "tasks suite runs"): + suite.make_adapter(self.context(**{"executor.name": "opencode"})) + with self.assertRaisesRegex(ConfigError, "non-empty editable"): + suite.make_adapter(self.context(**{"executor.name": "command"})) + non_git = self.root / "not-git" + non_git.mkdir() + problems = suite.check(replace(self.ctx, project=non_git)) + self.assertTrue(any("needs a git repository" in p for p in problems)) + + def test_full_two_task_schedule_k1_k2_and_resume(self): + self.task("two", domain="coding") + for k in (1, 2): + with self.subTest(k=k): + run_root = self.project / f".harness2/runs/k{k}" + adapter = suite.make_adapter(replace(self.ctx, run_root=run_root)) + proposer = Proposer(adapter.spec.channels["proposer"], audit_dir=run_root / "audit") + events.start(run_root / "events.jsonl") + kwargs = dict( + adapter=adapter, + prop=proposer, + runs_root=run_root, + library_root=run_root / "library", + modes=("agg", "seq"), + ) + result = schedule.run_domain("coding", adapter.tasks(), k, **kwargs) + self.assertEqual((result["done"], result["failed"]), (2, 0), result) + summaries = list(run_root.glob("tasks-*/**/summary.json")) + self.assertEqual(len(summaries), 2) + for path in summaries: + summary = json.loads(path.read_text()) + for mode in ("agg", "seq"): + ledger = summary["modes"][mode]["ledger"] + self.assertEqual((ledger["R"], ledger["P"]), (4 * k, 3 * k)) + count = len(self.records()) + resumed = schedule.run_domain("coding", adapter.tasks(), k, **kwargs) + self.assertEqual( + [row["status"] for row in resumed["tasks"]], ["skipped", "skipped"] + ) + self.assertEqual(len(self.records()), count) + records = [ + json.loads(line) + for line in (run_root / "events.jsonl").read_text().splitlines() + ] + rollouts = [row for row in records if row["kind"] == "rollout"] + self.assertTrue(rollouts) + self.assertTrue(all(row["cost_usd"] == 0.125 for row in rollouts)) + self.assertTrue(any(row["kind"] == "session" for row in records)) diff --git a/tests/test_workspace.py b/tests/test_workspace.py new file mode 100644 index 00000000..a423a4dc --- /dev/null +++ b/tests/test_workspace.py @@ -0,0 +1,94 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline workspace isolation, binary patches, and git cleanup.""" + +from __future__ import annotations + +import subprocess +import tempfile +import unittest +from pathlib import Path + +from harness2 import workspace +from harness2.settings import ConfigError + + +class WorkspaceTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name).resolve() + self.project = self.root / "project" + self.project.mkdir() + subprocess.run(["git", "init", "-q", str(self.project)], check=True) + (self.project / "code.txt").write_text("before\n") + (self.project / "CLAUDE.md").write_text("instructions\n") + workspace.commit_all(self.project, "initial") + + def test_project_diff_apply_and_cleanup(self): + with workspace.project_workspace(self.project, "HEAD") as ws: + self.assertFalse(ws.is_relative_to(self.project)) + workspace.commit_all(ws, "empty candidate") + base = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ws, text=True).strip() + (ws / "code.txt").write_text("after\n") + (ws / "new.bin").write_bytes(b"\0\xffbinary") + (ws / "CLAUDE.md").write_text("private changes\n") + (ws / ".harness2").mkdir() + (ws / ".harness2/secret").write_text("secret") + patch = workspace.diff_since(ws, base, ["CLAUDE.md", ".harness2"]) + self.assertIn("GIT binary patch", patch) + self.assertNotIn("CLAUDE.md", patch) + self.assertNotIn(".harness2", patch) + self.assertFalse(ws.exists()) + with workspace.project_workspace(self.project, "HEAD") as judge: + workspace.apply_patch(judge, patch) + workspace.apply_patch(judge, "") + self.assertEqual((judge / "code.txt").read_text(), "after\n") + self.assertEqual((judge / "new.bin").read_bytes(), b"\0\xffbinary") + self.assertEqual((judge / "CLAUDE.md").read_text(), "instructions\n") + with self.assertRaises(subprocess.CalledProcessError): + workspace.apply_patch(judge, "broken patch") + self.assertEqual((self.project / "code.txt").read_text(), "before\n") + listing = subprocess.check_output( + ["git", "worktree", "list", "--porcelain"], cwd=self.project, text=True + ) + self.assertEqual(listing.count("worktree "), 1) + + def test_glob_exclusions_and_cleanup_on_error(self): + with self.assertRaisesRegex(RuntimeError, "test"): + with workspace.project_workspace(self.project, "HEAD") as ws: + (ws / "CLAUDE.md").write_text("new instructions") + (ws / "code.txt").write_text("after") + patch = workspace.diff_since(ws, "HEAD", [":(glob)*.md"]) + self.assertNotIn("CLAUDE.md", patch) + self.assertIn("code.txt", patch) + raise RuntimeError("test") + self.assertFalse(ws.exists()) + for project, ref in ((self.project, "missing-ref"), (self.root, "HEAD")): + with self.assertRaises(ConfigError): + with workspace.project_workspace(project, ref): + self.fail("invalid project/ref accepted") + + def test_files_copy_output_and_cleanup(self): + src = self.root / "files" + src.mkdir() + (src / "data.csv").write_text("amount\n1\n") + with workspace.files_workspace(src) as ws: + self.assertTrue((ws / "output").is_dir()) + (ws / "data.csv").write_text("changed") + self.assertFalse(ws.exists()) + self.assertEqual((src / "data.csv").read_text(), "amount\n1\n") + with workspace.files_workspace(None) as ws: + self.assertEqual([p.name for p in ws.iterdir()], ["output"]) From 151503825a15fb3cba108853315b5601ad2e24a8 Mon Sep 17 00:00:00 2001 From: Bae Seongmin <83870598+3ae3ae@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:05:41 +0900 Subject: [PATCH 08/21] test: end-to-end run of the CLI on the built-in tasks suite (#8) Drives `harness2` in-process against a temporary git project with a CLAUDE.md, a skill and a .claude/settings.json, using a fake `claude` that plays both executor and improver. Covers a project task graded by a check command and a files task, runs at k=1 and k=2, judge, report, diff and adoption back into the project (the unrelated settings key and the skill survive), and a resumed run that makes no new model calls. Co-authored-by: Claude Opus 5.5 --- tests/test_e2e.py | 343 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 343 insertions(+) create mode 100644 tests/test_e2e.py diff --git a/tests/test_e2e.py b/tests/test_e2e.py new file mode 100644 index 00000000..a89b733e --- /dev/null +++ b/tests/test_e2e.py @@ -0,0 +1,343 @@ +# Copyright 2026 The Harness2 Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Real CLI and tasks-suite workflow against one offline Claude executable.""" + +from __future__ import annotations + +import contextlib +import io +import json +import os +import shlex +import subprocess +import sys +import tempfile +import unittest +from collections import Counter +from pathlib import Path +from unittest.mock import patch + +from harness2.cli import _adopt, _common +from harness2.cli.main import build_parser, main +from harness2.core import events, runmark + +FAKE_CLAUDE = r"""import json +import os +import sys +from pathlib import Path + +if "--version" in sys.argv: + print("fake claude 1.0") + sys.exit(0) + +prompt = sys.stdin.read() +assert prompt.strip(), "missing stdin prompt" +assert os.environ["ANTHROPIC_API_KEY"] == os.environ["E2E_API_KEY"] +cwd = Path.cwd() +output_format = sys.argv[sys.argv.index("--output-format") + 1] +model = sys.argv[sys.argv.index("--model") + 1] +context = cwd / "context" +if output_format == "stream-json": + role = "executor" +elif (context / "candidates").is_dir(): + role = "selector" +elif (context / "adoption.json").is_file(): + role = "reflector" +else: + role = "improver" +with open(os.environ["E2E_INVOCATIONS"], "a") as log: + log.write(json.dumps({"role": role, "model": model, "cwd": str(cwd)}) + "\n") + +if output_format == "stream-json": + assert model == "x" + assert not (cwd / ".harness2").exists(), "project state leaked into the workspace" + assert (cwd / ".claude/skills/checks/SKILL.md").is_file() + hooks = json.loads((cwd / ".claude/settings.json").read_text())["hooks"] + assert "PreToolUse" in hooks + if (cwd / "tiny_pkg/arithmetic.py").is_file(): + assert "Fix add" in prompt + target = cwd / "tiny_pkg/arithmetic.py" + target.write_text(target.read_text().replace("return a - b", "return a + b")) + result = "Repaired add in tiny_pkg/arithmetic.py." + else: + assert "Uppercase" in prompt + target = cwd / "output/result.txt" + target.parent.mkdir(exist_ok=True) + target.write_text((cwd / "message.txt").read_text().upper()) + result = "Saved output/result.txt." + for event in ( + {"type": "system", "subtype": "init", "model": model, "cwd": str(cwd)}, + {"type": "assistant", "message": {"role": "assistant", "content": [ + {"type": "tool_use", "id": "edit-1", "name": "Write", + "input": {"file_path": str(target)}}]}}, + {"type": "user", "message": {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "edit-1", "content": "File saved."}]}}, + {"type": "assistant", "message": {"role": "assistant", "content": [ + {"type": "text", "text": result}]}}, + {"type": "result", "subtype": "success", "is_error": False, + "result": result, "total_cost_usd": 0.125}, + ): + print(json.dumps(event)) +else: + assert output_format == "json" and model == "y" + harness = context / "harness/CLAUDE.md" + if harness.is_file(): + guidance = "Use the checks skill before finishing.\n" + if guidance not in harness.read_text(): + harness.write_text(harness.read_text() + guidance) + if role == "selector": + chosen = sorted((context / "candidates").glob("*.md"))[0].stem + text = "The requested change is present.\n```choice\n" + chosen + "\n```" + elif role == "reflector": + text = ( + "\ncomponent: CLAUDE.md\ncandidates: inv1\n" + "change: named the checks skill\nobserved: the requested output was produced\n" + "verdict: helped\n\n\n" + "\ncue: When completing a task\n" + "lesson: Use the checks skill before finishing.\n" + "evidence: CLAUDE.md named the checks skill and the run produced its output.\n" + "confidence: specific to this task\n" + ) + else: + text = "Name the checks skill in CLAUDE.md." + print(json.dumps({"type": "result", "subtype": "success", "is_error": False, + "result": text, "total_cost_usd": 0.01})) +""" + + +class EndToEndTests(unittest.TestCase): + def test_tasks_cli_run_judge_adopt_and_resume(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + project = root / "project" + project.mkdir() + binary_dir = root / "bin" + binary_dir.mkdir() + binary = binary_dir / "claude" + binary.write_text(f"#!{sys.executable}\n" + FAKE_CLAUDE) + binary.chmod(0o755) + invocation_log = root / "invocations.jsonl" + env = { + "PATH": f"{binary_dir}{os.pathsep}{os.defpath}", + "XDG_CONFIG_HOME": str(root / "config"), + "XDG_CACHE_HOME": str(root / "cache"), + "E2E_API_KEY": "offline-fixture", + "E2E_INVOCATIONS": str(invocation_log), + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": os.devnull, + "PYTHONDONTWRITEBYTECODE": "1", + } + self.addCleanup(events.start, None) + with patch.dict(os.environ, env, clear=True): + package = project / "tiny_pkg" + package.mkdir() + (package / "__init__.py").write_text("") + (package / "arithmetic.py").write_text("def add(a, b):\n return a - b\n") + tests = project / "tests" + tests.mkdir() + (tests / "test_arithmetic.py").write_text( + "import unittest\nfrom tiny_pkg.arithmetic import add\n\n" + "class ArithmeticTests(unittest.TestCase):\n" + " def test_add(self):\n self.assertEqual(add(2, 3), 5)\n" + ) + base_instructions = "Read the source before editing.\n" + (project / "CLAUDE.md").write_text(base_instructions) + skill = project / ".claude/skills/checks/SKILL.md" + skill.parent.mkdir(parents=True) + skill_text = ( + "---\nname: checks\ndescription: Run the project tests.\n---\n" + "Run python -m unittest discover -s tests.\n" + ) + skill.write_text(skill_text) + native_settings = { + "hooks": { + "PreToolUse": [ + {"matcher": "Bash", "hooks": [{"type": "command", "command": "true"}]} + ] + }, + "permissions": {"allow": ["Read"]}, + } + native_path = project / ".claude/settings.json" + native_path.write_text(json.dumps(native_settings)) + subprocess.run(["git", "init", "-q", str(project)], check=True) + subprocess.run(["git", "add", "."], cwd=project, check=True) + subprocess.run( + [ + "git", + "-c", + "user.name=fixture", + "-c", + "user.email=fixture@localhost", + "-c", + "commit.gpgsign=false", + "commit", + "-qm", + "fixture", + ], + cwd=project, + check=True, + ) + state = project / ".harness2" + state.mkdir() + (state / "config.toml").write_text( + '[executor]\nname = "claude"\nmodel = "anthropic/x"\n' + '[improver]\nchannel = "claude"\nmodel = "anthropic/y"\n' + '[providers.anthropic]\napi_key_env = "E2E_API_KEY"\n' + ) + + def invoke(*argv): + stdout, stderr = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + code = main(["-C", str(project), *argv]) + self.assertEqual(code, 0, f"{argv!r}\n{stdout.getvalue()}\n{stderr.getvalue()}") + return stdout.getvalue() + + project_check = shlex.join( + [sys.executable, "-B", "-m", "unittest", "discover", "-s", "tests", "-v"] + ) + invoke("task", "new", "fix-add", "--check", project_check) + (state / "tasks/fix-add/task.md").write_text("Fix add so 2 plus 3 returns 5.\n") + files_check = shlex.join( + [ + sys.executable, + "-B", + "-c", + "from pathlib import Path; assert Path('output/result.txt').read_text() == 'HELLO\\n'", + ] + ) + invoke("task", "new", "write-file", "--files", "--check", files_check) + (state / "tasks/write-file/task.md").write_text( + "Uppercase message.txt and save output/result.txt.\n" + ) + (state / "tasks/write-file/files/message.txt").write_text("hello\n") + + invoke("run", "--k", "1") + invoke("run", "--k", "2", "--name", "k2") + all_calls = [json.loads(line) for line in invocation_log.read_text().splitlines()] + self.assertEqual( + Counter(call["role"] for call in all_calls), + { + "executor": 24, + "improver": 18, + "selector": 2, + "reflector": 4, + }, + ) + self.assertTrue( + all( + call["model"] == ("x" if call["role"] == "executor" else "y") + for call in all_calls + ) + ) + for name, k in (("claude-k1", 1), ("k2", 2)): + run_dir = state / "runs" / name + record = _common.load_run(run_dir) + self.assertEqual(record["schedule"]["k"], k) + self.assertEqual( + record["stream"], [["default", "fix-add"], ["default", "write-file"]] + ) + event_rows = [ + json.loads(line) + for line in (run_dir / "events.jsonl").read_text().splitlines() + ] + self.assertEqual( + (event_rows[0]["kind"], event_rows[-1]["status"]), ("run_start", "ok") + ) + self.assertEqual(sum(row["kind"] == "rollout" for row in event_rows), 8 * k) + self.assertTrue( + all( + row["cost_usd"] == 0.125 + for row in event_rows + if row["kind"] == "rollout" + ) + ) + self.assertTrue( + all( + row["cost_usd"] == 0.01 + for row in event_rows + if row["kind"] == "session" + ) + ) + run_rows = json.loads(invoke("--json", "runs", "ls")) + self.assertEqual( + { + row["name"]: (row["done"], row["total"], row["spent_usd"]) + for row in run_rows + }, + { + "claude-k1": (2, 2, 1.08), + "k2": (2, 2, 2.16), + }, + ) + invoke("judge") + rows = json.loads(invoke("--json", "report")) + self.assertEqual({row["task_id"] for row in rows}, {"fix-add", "write-file"}) + self.assertTrue( + all(row["name"] == "k2" and row["graded"] and row["all_pass"] for row in rows) + ) + self.assertTrue(all(row["raw"]["exit_code"] == 0 for row in rows)) + self.assertTrue( + any( + "Ran 1 test" in row["raw"]["output_tail"] + for row in rows + if row["task_id"] == "fix-add" + ) + ) + diff = invoke("diff") + self.assertIn("--- a/CLAUDE.md", diff) + self.assertIn("+Use the checks skill before finishing.", diff) + + args = build_parser().parse_args(["-C", str(project), "diff"]) + ctx = _common.Context(args, {}) + adapter, run_dir = _adopt.open_run(ctx, None) + self.assertEqual(run_dir.name, "k2") + chosen_task, adopted = _adopt.adopted_tree(adapter, run_dir, None, "parallel") + self.assertEqual(chosen_task, "write-file") + base = _adopt.base_tree(adapter) + self.assertEqual(base["CLAUDE.md"], base_instructions) + self.assertEqual(_adopt.changes(base, adopted), [("CLAUDE.md", "M")]) + for _, summary in _common.summaries(run_dir): + selected = summary["selections"]["agg"] + rollout = runmark.resolve_path(selected["chosen_run_dir"], run_dir) + self.assertTrue(adapter.has_judge(summary["task_id"])) + self.assertEqual( + json.loads((rollout / "judge.json").read_text())["n_passed"], 1 + ) + if summary["task_id"] == "fix-add": + deliverable = (rollout / "deliverable.patch").read_text() + self.assertIn("+ return a + b", deliverable) + self.assertNotIn("CLAUDE.md", deliverable) + self.assertNotIn(".claude/", deliverable) + self.assertNotIn(".harness2/", deliverable) + else: + self.assertEqual((rollout / "output/result.txt").read_text(), "HELLO\n") + applied = _adopt.perform(ctx.settings, adapter, adopted, base=base, branch=None) + self.assertEqual(applied, {"written": ["CLAUDE.md"], "deleted": []}) + self.assertEqual((project / "CLAUDE.md").read_text(), adopted["CLAUDE.md"]) + self.assertEqual(json.loads(native_path.read_text()), native_settings) + self.assertEqual(skill.read_text(), skill_text) + self.assertIn("return a - b", (package / "arithmetic.py").read_text()) + before_resume = invocation_log.read_bytes() + invoke("run", "--k", "1") + self.assertEqual(invocation_log.read_bytes(), before_resume) + worktrees = subprocess.check_output( + ["git", "worktree", "list", "--porcelain"], cwd=project, text=True + ) + self.assertEqual(worktrees.count("worktree "), 1) + print("E2E: 2 task types; k=1/k=2; 48 calls; 2 graded; adopted; resume=0 calls") + + +if __name__ == "__main__": + unittest.main() From ebda822c98b9526a00d9c8c6b41a080edb8c4090 Mon Sep 17 00:00:00 2001 From: Bae Seongmin <83870598+3ae3ae@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:15:19 +0900 Subject: [PATCH 09/21] feat: interactive init, status, apply, chat and a human selector (#9) - `harness2 init` asks which agent CLI runs the tasks (detecting claude and codex on PATH), which models to use and where the harness lives, writes a commented .harness2/config.toml, a .gitignore and the tasks directory, and says whether the needed API key is visible. --yes takes the defaults. - `harness2 status [--watch]` shows each task's state, the harness it picked, rollouts and spend, from run.json and events.jsonl only, so it works during a run, after it stopped and from another terminal. - `harness2 apply` applies only what the run changed relative to the harness it started from, so edits made to the project since are kept; a file changed on both sides is skipped unless --force. It lists the changes first and asks, or reviews file by file with -i, refuses to write over uncommitted harness files, and can create a branch. - `harness2 chat` starts the user's own Claude Code or Codex in the project with a guide to harness2 and the project's tasks and runs. - `harness2 run --selector human` hands the final pick to the person at the terminal, showing the anonymised candidates. Every subcommand now has help text, and run, doctor, config, task and judge print readable results (doctor also flags a model whose provider its CLI cannot use). Untracks the bytecode and egg-info that slipped into the repository and adds a .gitignore. Co-authored-by: Claude Opus 5.5 --- .gitignore | 6 + harness2.egg-info/PKG-INFO | 227 ------------------ harness2.egg-info/SOURCES.txt | 77 ------ harness2.egg-info/dependency_links.txt | 1 - harness2.egg-info/entry_points.txt | 8 - harness2.egg-info/requires.txt | 5 - harness2.egg-info/top_level.txt | 1 - harness2/__pycache__/__init__.cpython-312.pyc | Bin 243 -> 0 bytes .../__pycache__/providers.cpython-312.pyc | Bin 7462 -> 0 bytes harness2/__pycache__/settings.cpython-312.pyc | Bin 23181 -> 0 bytes harness2/cli/apply.py | 146 ++++++++++- harness2/cli/bridge.py | 26 +- harness2/cli/chat.py | 102 +++++++- harness2/cli/completion.py | 10 +- harness2/cli/config.py | 47 ++-- harness2/cli/demo.py | 20 +- harness2/cli/diff.py | 19 +- harness2/cli/doctor.py | 70 ++++-- harness2/cli/human.py | 70 +++++- harness2/cli/init.py | 216 ++++++++++++++++- harness2/cli/judge.py | 21 +- harness2/cli/main.py | 19 +- harness2/cli/report.py | 9 +- harness2/cli/run.py | 132 ++++++---- harness2/cli/runs.py | 8 +- harness2/cli/setup.py | 11 +- harness2/cli/status.py | 199 ++++++++++++++- harness2/cli/task.py | 46 ++-- .../core/__pycache__/__init__.cpython-312.pyc | Bin 1452 -> 0 bytes .../core/__pycache__/prompts.cpython-312.pyc | Bin 76063 -> 0 bytes .../core/__pycache__/runmark.cpython-312.pyc | Bin 6126 -> 0 bytes .../core/__pycache__/spec.cpython-312.pyc | Bin 11596 -> 0 bytes .../core/__pycache__/tree.cpython-312.pyc | Bin 25937 -> 0 bytes .../__pycache__/__init__.cpython-312.pyc | Bin 718 -> 0 bytes .../__pycache__/_native.cpython-312.pyc | Bin 5433 -> 0 bytes .../__pycache__/base.cpython-312.pyc | Bin 3794 -> 0 bytes .../__pycache__/claude.cpython-312.pyc | Bin 17276 -> 0 bytes .../__pycache__/codex.cpython-312.pyc | Bin 25842 -> 0 bytes .../__pycache__/command.cpython-312.pyc | Bin 6789 -> 0 bytes .../__pycache__/opencode.cpython-312.pyc | Bin 19071 -> 0 bytes .../harness2_benchmarks.egg-info/PKG-INFO | 11 - .../harness2_benchmarks.egg-info/SOURCES.txt | 77 ------ .../dependency_links.txt | 1 - .../entry_points.txt | 6 - .../harness2_benchmarks.egg-info/requires.txt | 5 - .../top_level.txt | 1 - tests/test_bridge.py | 7 +- tests/test_cli.py | 20 +- tests/test_ui.py | 154 ++++++++++++ 49 files changed, 1192 insertions(+), 586 deletions(-) create mode 100644 .gitignore delete mode 100644 harness2.egg-info/PKG-INFO delete mode 100644 harness2.egg-info/SOURCES.txt delete mode 100644 harness2.egg-info/dependency_links.txt delete mode 100644 harness2.egg-info/entry_points.txt delete mode 100644 harness2.egg-info/requires.txt delete mode 100644 harness2.egg-info/top_level.txt delete mode 100644 harness2/__pycache__/__init__.cpython-312.pyc delete mode 100644 harness2/__pycache__/providers.cpython-312.pyc delete mode 100644 harness2/__pycache__/settings.cpython-312.pyc delete mode 100644 harness2/core/__pycache__/__init__.cpython-312.pyc delete mode 100644 harness2/core/__pycache__/prompts.cpython-312.pyc delete mode 100644 harness2/core/__pycache__/runmark.cpython-312.pyc delete mode 100644 harness2/core/__pycache__/spec.cpython-312.pyc delete mode 100644 harness2/core/__pycache__/tree.cpython-312.pyc delete mode 100644 harness2/substrate/__pycache__/__init__.cpython-312.pyc delete mode 100644 harness2/substrate/__pycache__/_native.cpython-312.pyc delete mode 100644 harness2/substrate/__pycache__/base.cpython-312.pyc delete mode 100644 harness2/substrate/__pycache__/claude.cpython-312.pyc delete mode 100644 harness2/substrate/__pycache__/codex.cpython-312.pyc delete mode 100644 harness2/substrate/__pycache__/command.cpython-312.pyc delete mode 100644 harness2/substrate/__pycache__/opencode.cpython-312.pyc delete mode 100644 plugins/harness2-benchmarks/harness2_benchmarks.egg-info/PKG-INFO delete mode 100644 plugins/harness2-benchmarks/harness2_benchmarks.egg-info/SOURCES.txt delete mode 100644 plugins/harness2-benchmarks/harness2_benchmarks.egg-info/dependency_links.txt delete mode 100644 plugins/harness2-benchmarks/harness2_benchmarks.egg-info/entry_points.txt delete mode 100644 plugins/harness2-benchmarks/harness2_benchmarks.egg-info/requires.txt delete mode 100644 plugins/harness2-benchmarks/harness2_benchmarks.egg-info/top_level.txt create mode 100644 tests/test_ui.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..fb7bbf5c --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +__pycache__/ +*.py[cod] +*.egg-info/ +.venv/ +build/ +dist/ diff --git a/harness2.egg-info/PKG-INFO b/harness2.egg-info/PKG-INFO deleted file mode 100644 index 11273dff..00000000 --- a/harness2.egg-info/PKG-INFO +++ /dev/null @@ -1,227 +0,0 @@ -Metadata-Version: 2.4 -Name: harness2 -Version: 0.1.0 -Summary: Harness²: Recursive Agent Harnessing for an Open World -Author: Google LLC -License-Expression: Apache-2.0 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Operating System :: POSIX -Requires-Python: <3.14,>=3.12 -Description-Content-Type: text/markdown -License-File: LICENSE -Provides-Extra: bridge -Requires-Dist: litellm[proxy]==1.95.0; extra == "bridge" -Requires-Dist: fastapi==0.139.2; extra == "bridge" -Requires-Dist: google-auth==2.56.3; extra == "bridge" -Dynamic: license-file - -