From aac945f33eaa60fc394a9718a885e6c66a29e6f5 Mon Sep 17 00:00:00 2001 From: scgopi Date: Sun, 13 Sep 2026 22:02:14 -0700 Subject: [PATCH 1/2] Add a Harbor adapter that runs Terminal-Bench tasks as goal loops Tools/harbor-graphcode installs graphcode, graphcoded and zmx into a task container, creates one goal loop per task and waits for it to resolve, reusing Harbor's ClaudeCode install and trajectory conversion. The done check is either the task's own test.sh (an oracle) or the session's own node done. build-linux-bundle.sh builds the binaries in swift:6.2; docs/benchmarks/terminal-bench-trial.md holds the trial plan and cost estimate, not yet run. Co-Authored-By: Claude Opus 5 (1M context) --- Tools/harbor-graphcode/.gitignore | 4 + Tools/harbor-graphcode/README.md | 84 +++++ Tools/harbor-graphcode/build-in-container.sh | 30 ++ Tools/harbor-graphcode/build-linux-bundle.sh | 28 ++ .../harbor_graphcode/__init__.py | 3 + .../harbor_graphcode/agent.py | 313 ++++++++++++++++++ Tools/harbor-graphcode/pyproject.toml | 25 ++ Tools/harbor-graphcode/tests/test_agent.py | 217 ++++++++++++ docs/benchmarks/terminal-bench-trial.md | 63 ++++ 9 files changed, 767 insertions(+) create mode 100644 Tools/harbor-graphcode/.gitignore create mode 100644 Tools/harbor-graphcode/README.md create mode 100755 Tools/harbor-graphcode/build-in-container.sh create mode 100755 Tools/harbor-graphcode/build-linux-bundle.sh create mode 100644 Tools/harbor-graphcode/harbor_graphcode/__init__.py create mode 100644 Tools/harbor-graphcode/harbor_graphcode/agent.py create mode 100644 Tools/harbor-graphcode/pyproject.toml create mode 100644 Tools/harbor-graphcode/tests/test_agent.py create mode 100644 docs/benchmarks/terminal-bench-trial.md diff --git a/Tools/harbor-graphcode/.gitignore b/Tools/harbor-graphcode/.gitignore new file mode 100644 index 00000000..7722a2cd --- /dev/null +++ b/Tools/harbor-graphcode/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +.venv/ +dist/ +uv.lock diff --git a/Tools/harbor-graphcode/README.md b/Tools/harbor-graphcode/README.md new file mode 100644 index 00000000..3af91e0d --- /dev/null +++ b/Tools/harbor-graphcode/README.md @@ -0,0 +1,84 @@ +# harbor-graphcode + +A [Harbor](https://github.com/harbor-framework/harbor) installed agent that runs each +Terminal-Bench task as one GraphCode goal loop, for private trials against Harbor's plain +`claude-code` agent on the same model. It is loaded by import path; nothing is added to +Harbor itself, and nothing here submits to a leaderboard. + +## How a task runs + +1. **install** — installs zsh (graphcoded launches sessions through `/bin/zsh -i -l`), + Claude Code via the inherited `ClaudeCode.install`, then uploads + `graphcode-linux-.tar.gz` and unpacks `graphcode`, `graphcoded` and `zmx` into + `/opt/graphcode/bin`. +2. **run** — starts graphcoded with `GRAPHCODE_SUPPORT_DIR=/tmp/gcd`, Claude permission + mode `bypassPermissions` (`IS_SANDBOX=1`), and every model alias pinned to Harbor's + `-m`. It creates one goal loop in the task's working directory with the instruction as + its goal, then polls `graphcode status` until the loop is `succeeded`, `failed`, + `stalled` or `stopped`. +3. **trajectory** — Claude Code transcripts are mirrored into + `/logs/agent/sessions/projects`, where the inherited converter writes `trajectory.json` + and token counts. `/logs/agent/graphcode/` keeps the daemon log, status snapshots and + loop memory. + +### Done check + +| `done_check` | Loop predicate | Fair against `claude-code`? | +|---|---|---| +| `tests` (default) | The task's own `tests/test.sh`, uploaded to `/tests` before the run; passes when it writes reward ≥ 1 | ❌ An oracle: the loop sees the verifier's verdict, plain `claude-code` does not. Only an upper bound. | +| `agent` | None; the loop resolves when the session runs `graphcode node done` | ✅ The same information both agents get | + +The tests directory is read from the trial's `config.json` (or the `tests_dir` kwarg). +Harbor empties `/tests` and re-uploads it before verification, and the check deletes the +reward it wrote, so verification itself is unaffected. + +## Build the Linux bundle + +Needs docker on the host; not verified yet (no container runtime on the machine this was +written on). + +```sh +ARCH=x86_64 Tools/harbor-graphcode/build-linux-bundle.sh +``` + +Inside `swift:6.2` this runs `swift build -c release --static-swift-stdlib`, builds zmx at +the `ThirdParty/zmx` submodule pin (scgopi/zmx) with zig 0.15.2 `ReleaseFast`, and writes +`Tools/harbor-graphcode/dist/graphcode-linux-x86_64.tar.gz`. The binaries link glibc, so +Alpine task images are refused at install. + +## Run + +```sh +cd Tools/harbor-graphcode +uv sync +export ANTHROPIC_API_KEY=... # never commit or echo it +export GRAPHCODE_LINUX_BUNDLE_DIR=$PWD/dist + +# Smoke: 2 tasks x 1 attempt, both agents +uv run harbor run -d terminal-bench/terminal-bench-2 -e docker -m anthropic/claude-opus-5 \ + -a claude-code -l 2 -k 1 +uv run harbor run -d terminal-bench/terminal-bench-2 -e docker -m anthropic/claude-opus-5 \ + -a harbor_graphcode:GraphCode --ak done_check=agent -l 2 -k 1 +``` + +Check `harbor run --help` for the task-selection flag your Harbor version uses before +spending. Useful kwargs: `done_check=tests|agent`, `budget_tokens=`, +`poll_interval_sec=`, `bundle_dir=`. + +## Tests + +No Docker or API key needed: + +```sh +uv run --group dev pytest +``` + +## What is verified + +| Piece | Status | +|---|---| +| Adapter logic, generated shell (`bash -n`), awk/Python parser agreement, oracle reward check | ✅ unit tests | +| `graphcode status` line format and a loop reaching `stopped` | ✅ against a real macOS graphcoded (see PR) | +| Linux bundle build in `swift:6.2` | ⚠️ unverified, no container runtime | +| graphcoded launching a Claude session on Linux inside a task image | ⚠️ unverified; CI only smoke-tests CLI verbs on Linux | +| Claude Code's first-run prompts suppressed by the seeded `~/.claude.json` | ⚠️ unverified | diff --git a/Tools/harbor-graphcode/build-in-container.sh b/Tools/harbor-graphcode/build-in-container.sh new file mode 100755 index 00000000..2164a5a8 --- /dev/null +++ b/Tools/harbor-graphcode/build-in-container.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# Runs inside swift:6.2 with the repo mounted at /src; see build-linux-bundle.sh. +set -eu +: "${ARCH:?}" "${ZMX_COMMIT:?}" "${ZIG_VERSION:?}" +work=/tmp/gcbundle +rm -rf "$work" +mkdir -p "$work/bundle/bin" + +apt-get update -qq +DEBIAN_FRONTEND=noninteractive apt-get install -y -qq curl git xz-utils >/dev/null + +# A separate build path keeps the host checkout's macOS .build untouched. The static +# stdlib spares task images from needing the Swift runtime; glibc is still required, so +# musl images (Alpine) are out of reach for this bundle. +swift build -c release --static-swift-stdlib --build-path "$work/swift" +cp "$work/swift/release/graphcode" "$work/swift/release/graphcoded" "$work/bundle/bin/" + +curl -fsSL "https://ziglang.org/download/$ZIG_VERSION/zig-$ARCH-linux-$ZIG_VERSION.tar.xz" \ + | tar -xJ -C "$work" +git clone -q https://github.com/scgopi/zmx.git "$work/zmx" +git -C "$work/zmx" checkout -q "$ZMX_COMMIT" +# ReleaseFast: a Debug zmx is visibly slower than a plain PTY. +(cd "$work/zmx" && "$work/zig-$ARCH-linux-$ZIG_VERSION/zig" build -Doptimize=ReleaseFast) +cp "$work/zmx/zig-out/bin/zmx" "$work/bundle/bin/" + +for binary in "$work"/bundle/bin/*; do + echo "== $(basename "$binary")" + ldd "$binary" || true +done +tar -czf "/src/Tools/harbor-graphcode/dist/graphcode-linux-$ARCH.tar.gz" -C "$work/bundle" bin diff --git a/Tools/harbor-graphcode/build-linux-bundle.sh b/Tools/harbor-graphcode/build-linux-bundle.sh new file mode 100755 index 00000000..6b6748a2 --- /dev/null +++ b/Tools/harbor-graphcode/build-linux-bundle.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# Builds dist/graphcode-linux-.tar.gz — bin/graphcode, bin/graphcoded and bin/zmx — +# the bundle the Harbor adapter uploads into each task container. +# +# ARCH=x86_64 Tools/harbor-graphcode/build-linux-bundle.sh # Terminal-Bench images +# ARCH=aarch64 Tools/harbor-graphcode/build-linux-bundle.sh +# +# Needs docker (or a docker-compatible CLI) on the host. The arch must match the task +# images, not the host: Terminal-Bench images are linux/amd64, so on Apple Silicon this +# runs emulated and is slow. +set -eu +arch="${ARCH:-x86_64}" +case "$arch" in + x86_64) platform=linux/amd64 ;; + aarch64) platform=linux/arm64 ;; + *) echo "build-linux-bundle: ARCH must be x86_64 or aarch64, got $arch" >&2; exit 2 ;; +esac +root="$(cd "$(dirname "$0")/../.." && pwd)" +# The zmx graphcode ships is the submodule pin (scgopi/zmx, ahead of upstream 0.7.0 with +# the kill-process-group fix), not an upstream release. +zmx_commit="$(git -C "$root" ls-tree HEAD ThirdParty/zmx | awk '{print $3}')" +[ -n "$zmx_commit" ] || { echo "build-linux-bundle: no ThirdParty/zmx pin in HEAD" >&2; exit 1; } +mkdir -p "$root/Tools/harbor-graphcode/dist" +docker run --rm --platform "$platform" \ + -v "$root:/src" -w /src \ + -e ARCH="$arch" -e ZMX_COMMIT="$zmx_commit" -e ZIG_VERSION="${ZIG_VERSION:-0.15.2}" \ + swift:6.2 sh /src/Tools/harbor-graphcode/build-in-container.sh +ls -l "$root/Tools/harbor-graphcode/dist/graphcode-linux-$arch.tar.gz" diff --git a/Tools/harbor-graphcode/harbor_graphcode/__init__.py b/Tools/harbor-graphcode/harbor_graphcode/__init__.py new file mode 100644 index 00000000..bab3401e --- /dev/null +++ b/Tools/harbor-graphcode/harbor_graphcode/__init__.py @@ -0,0 +1,3 @@ +from harbor_graphcode.agent import GraphCode, GraphCodeOptions + +__all__ = ["GraphCode", "GraphCodeOptions"] diff --git a/Tools/harbor-graphcode/harbor_graphcode/agent.py b/Tools/harbor-graphcode/harbor_graphcode/agent.py new file mode 100644 index 00000000..4163bc51 --- /dev/null +++ b/Tools/harbor-graphcode/harbor_graphcode/agent.py @@ -0,0 +1,313 @@ +"""GraphCode as a Harbor installed agent: each task becomes one goal loop. + +The loop runs Claude Code under graphcoded inside the task container, so the comparison +with Harbor's plain ``claude-code`` agent isolates what the loop adds. The transcript +Claude Code writes is mirrored into the agent logs, where the inherited ``ClaudeCode`` +converter turns it into the ATIF trajectory. +""" + +import json +import os +import shlex +from pathlib import Path +from typing import Literal, override + +from pydantic import Field + +from harbor.agents.installed.base import with_prompt_template +from harbor.agents.installed.claude_code import ClaudeCode, ClaudeCodeOptions +from harbor.environments.base import BaseEnvironment +from harbor.models.agent.context import AgentContext +from harbor.models.trial.config import TaskConfig + +BUNDLE_DIR_ENV = "GRAPHCODE_LINUX_BUNDLE_DIR" +INSTALL_ROOT = "/opt/graphcode" +# Short on purpose: the daemon's socket lives inside it and sun_path is bounded. +SUPPORT_DIR = "/tmp/gcd" +LOOP_TITLE = "TerminalBenchTask" +ORACLE_CHECK_PATH = "/opt/graphcode-oracle/check.sh" +RESOLVED_STATES = ("succeeded", "failed", "stalled", "stopped") + +_ARCH_ALIASES = { + "x86_64": "x86_64", + "amd64": "x86_64", + "aarch64": "aarch64", + "arm64": "aarch64", +} + +# graphcoded launches every session through `/bin/zsh -i -l`, and task images rarely +# ship zsh. +_SYSTEM_PACKAGES_COMMAND = """set -eu +missing=0 +for tool in zsh tar ps; do command -v "$tool" >/dev/null 2>&1 || missing=1; done +if [ "$missing" = 1 ]; then + if command -v apt-get >/dev/null 2>&1; then + DEBIAN_FRONTEND=noninteractive apt-get update -qq + DEBIAN_FRONTEND=noninteractive apt-get install -y -qq zsh tar procps ca-certificates >/dev/null + elif command -v dnf >/dev/null 2>&1; then + dnf install -y -q zsh tar procps-ng + elif command -v yum >/dev/null 2>&1; then + yum install -y -q zsh tar procps-ng + elif command -v apk >/dev/null 2>&1; then + echo "graphcode: musl images are unsupported (the bundle links glibc)" >&2 + exit 1 + else + echo "graphcode: no package manager to install zsh with" >&2 + exit 1 + fi +fi +[ -x /bin/zsh ] || ln -sf "$(command -v zsh)" /bin/zsh +""" + +# Harbor scores a task by the reward test.sh writes, not by its exit status, so the done +# check reads the reward back and removes it: the verifier must start from a clean slate. +ORACLE_CHECK_SCRIPT = """#!/bin/bash +mkdir -p /logs/verifier +bash /tests/test.sh > /tmp/graphcode-oracle.out 2>&1 || true +reward="$(cat /logs/verifier/reward.txt 2>/dev/null || echo 0)" +rm -f /logs/verifier/reward.txt /logs/verifier/reward.json +tail -n 40 /tmp/graphcode-oracle.out +awk -v r="$reward" 'BEGIN { exit !(r + 0 >= 1) }' +""" + + +class GraphCodeOptions(ClaudeCodeOptions): + bundle_dir: str | None = Field( + default=None, + description=( + "Host directory holding graphcode-linux-.tar.gz " + f"(falls back to ${BUNDLE_DIR_ENV})." + ), + ) + done_check: Literal["tests", "agent"] = Field( + default="tests", + description=( + "tests: the loop's predicate is the task's own test.sh (an oracle plain " + "claude-code does not get). agent: no predicate; the loop resolves when the " + "session runs `graphcode node done`." + ), + ) + tests_dir: str | None = Field( + default=None, + description="Host tests directory; defaults to the trial's task tests/.", + ) + poll_interval_sec: int = Field(default=10, ge=1) + budget_tokens: int | None = Field(default=None, ge=1) + + +def normalize_arch(uname: str) -> str: + machine = uname.strip().splitlines()[-1].strip() if uname.strip() else "" + try: + return _ARCH_ALIASES[machine] + except KeyError: + raise ValueError(f"Unsupported container architecture: {machine!r}") from None + + +def parse_node_id(status_output: str, title: str = LOOP_TITLE) -> str | None: + """The id of the goal loop titled ``title`` in ``graphcode status`` output.""" + for line in status_output.splitlines(): + fields = line.split() + if len(fields) >= 4 and fields[2] == "goalBased" and fields[3] == title: + return fields[0] + return None + + +def parse_node_state(status_output: str, node_id: str) -> str | None: + for line in status_output.splitlines(): + fields = line.split() + if len(fields) >= 2 and fields[0] == node_id: + return fields[1] + return None + + +class GraphCode(ClaudeCode): + options_model = GraphCodeOptions + options: GraphCodeOptions + + @staticmethod + @override + def name() -> str: + return "graphcode" + + @override + def get_version_command(self) -> str | None: + return None + + def _bundle_dir(self) -> Path: + configured = self.options.bundle_dir or os.environ.get(BUNDLE_DIR_ENV) + if not configured: + raise ValueError( + f"Set the bundle_dir agent kwarg or ${BUNDLE_DIR_ENV} to the directory " + "build-linux-bundle.sh wrote" + ) + return Path(configured).expanduser() + + def bundle_path(self, arch: str) -> Path: + path = self._bundle_dir() / f"graphcode-linux-{arch}.tar.gz" + if not path.is_file(): + raise FileNotFoundError(f"No GraphCode Linux bundle at {path}") + return path + + def resolve_tests_dir(self) -> Path: + if self.options.tests_dir: + tests = Path(self.options.tests_dir).expanduser() + else: + config_path = self.logs_dir.parent / "config.json" + if not config_path.is_file(): + raise FileNotFoundError( + f"No trial config at {config_path}; pass the tests_dir agent kwarg" + ) + task = TaskConfig.model_validate(json.loads(config_path.read_text())["task"]) + tests = task.get_local_path() / "tests" + if not (tests / "test.sh").is_file(): + raise FileNotFoundError(f"No test.sh in {tests}") + return tests + + @override + async def install(self, environment: BaseEnvironment) -> None: + await self.exec_as_root(environment, command=_SYSTEM_PACKAGES_COMMAND) + await super().install(environment) + + uname = await environment.exec(command="uname -m", user="root") + bundle = self.bundle_path(normalize_arch(uname.stdout or "")) + remote_bundle = "/tmp/graphcode-linux.tar.gz" + await environment.upload_file(bundle, remote_bundle) + await self.exec_as_root( + environment, + command=( + "set -eu; " + f"mkdir -p {INSTALL_ROOT} && " + f"tar -xzf {remote_bundle} -C {INSTALL_ROOT} && " + f"rm -f {remote_bundle} && " + f"chmod 755 {INSTALL_ROOT}/bin/* && " + f"ln -sf {INSTALL_ROOT}/bin/graphcode /usr/local/bin/graphcode && " + f"ln -sf {INSTALL_ROOT}/bin/graphcoded /usr/local/bin/graphcoded && " + f"test -x {INSTALL_ROOT}/bin/zmx" + ), + ) + + def session_env(self) -> dict[str, str]: + env = self._resolve_auth_env() + model = self._resolved_model_name() + if model: + # graphcode passes `--model haiku|sonnet|opus` by loop tier; pinning every + # alias keeps the comparison on the one model Harbor was asked for. + env["ANTHROPIC_MODEL"] = model + env["ANTHROPIC_DEFAULT_HAIKU_MODEL"] = model + env["ANTHROPIC_DEFAULT_SONNET_MODEL"] = model + env["ANTHROPIC_DEFAULT_OPUS_MODEL"] = model + env["CLAUDE_CODE_SUBAGENT_MODEL"] = model + env["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC"] = "1" + env["IS_SANDBOX"] = "1" + env["GRAPHCODE_SUPPORT_DIR"] = SUPPORT_DIR + return env + + def start_daemon_command(self) -> str: + settings = json.dumps( + { + "defaultBackend": "claudeCode", + "claudePermissionMode": "bypassPermissions", + } + ) + claude_settings = json.dumps({"skipDangerousModePermissionPrompt": True}) + return f"""set -eu +export PATH="$HOME/.local/bin:{INSTALL_ROOT}/bin:$PATH" +mkdir -p "$GRAPHCODE_SUPPORT_DIR/bin" /logs/agent/graphcode "$HOME/.claude" +cp {INSTALL_ROOT}/bin/zmx "$GRAPHCODE_SUPPORT_DIR/bin/zmx" +printf '%s\\n' {shlex.quote(settings)} > "$GRAPHCODE_SUPPORT_DIR/settings.json" +printf 'export PATH="$HOME/.local/bin:{INSTALL_ROOT}/bin:$PATH"\\n' >> "$HOME/.zshenv" +[ -f "$HOME/.claude/settings.json" ] || printf '%s\\n' {shlex.quote(claude_settings)} > "$HOME/.claude/settings.json" +if [ ! -f "$HOME/.claude.json" ]; then + key_tail="$(printf '%s' "${{ANTHROPIC_API_KEY:-}}" | tail -c 20)" + printf '{{"hasCompletedOnboarding":true,"bypassPermissionsModeAccepted":true,"customApiKeyResponses":{{"approved":["%s"],"rejected":[]}}}}\\n' "$key_tail" > "$HOME/.claude.json" +fi +setsid nohup graphcoded > /logs/agent/graphcode/graphcoded.log 2>&1 < /dev/null & +for _ in $(seq 1 60); do + [ -S "$GRAPHCODE_SUPPORT_DIR/graphcoded.sock" ] && exit 0 + sleep 0.5 +done +echo "graphcode: graphcoded never listened" >&2 +cat /logs/agent/graphcode/graphcoded.log >&2 +exit 1 +""" + + def create_and_wait_command(self, predicate: str | None) -> str: + create = [ + "graphcode", + "node", + "create", + '"$project"', + "--title", + LOOP_TITLE, + "--type", + "goal", + "--backend", + "claudeCode", + "--goal", + '"$GRAPHCODE_TB_GOAL"', + ] + if predicate: + create += ["--predicate", shlex.quote(predicate)] + if self.options.budget_tokens: + create += ["--budget", str(self.options.budget_tokens)] + resolved = "|".join(RESOLVED_STATES) + return f"""set -u +export PATH="$HOME/.local/bin:{INSTALL_ROOT}/bin:$PATH" +project="$(pwd)" +logs=/logs/agent/graphcode +mirror() {{ + if [ -d "$HOME/.claude/projects" ]; then + mkdir -p /logs/agent/sessions/projects + cp -R "$HOME/.claude/projects/." /logs/agent/sessions/projects/ 2>/dev/null || true + fi + graphcode status "$project" > "$logs/status.txt" 2>&1 || true + [ -d "$GRAPHCODE_SUPPORT_DIR/memory" ] && cp -R "$GRAPHCODE_SUPPORT_DIR/memory" "$logs/" 2>/dev/null || true +}} +{" ".join(create)} > "$logs/create.txt" 2>&1 || {{ cat "$logs/create.txt" >&2; exit 1; }} +node="$(awk '$3 == "goalBased" && $4 == "{LOOP_TITLE}" {{ print $1; exit }}' "$logs/create.txt")" +[ -n "$node" ] || {{ echo "graphcode: no loop id in create output" >&2; cat "$logs/create.txt" >&2; exit 1; }} +echo "$node" > "$logs/node-id.txt" +exited=0 +while :; do + sleep {self.options.poll_interval_sec} + mirror + line="$(awk -v id="$node" '$1 == id' "$logs/status.txt")" + state="$(printf '%s\\n' "$line" | awk '{{ print $2 }}')" + case "$state" in {resolved}) break ;; esac + # A session that exited without resolving the loop would otherwise hold the trial + # until Harbor's timeout. + case "$line" in *"session exited"*) exited=$((exited + 1)) ;; *) exited=0 ;; esac + [ "$exited" -ge 6 ] && break +done +echo "$state" > "$logs/final-state.txt" +mirror +""" + + @override + @with_prompt_template + async def run( + self, instruction: str, environment: BaseEnvironment, context: AgentContext + ) -> None: + predicate = None + if self.options.done_check == "tests": + await environment.upload_dir(self.resolve_tests_dir(), "/tests") + await self.exec_as_root( + environment, command=f"mkdir -p {Path(ORACLE_CHECK_PATH).parent}" + ) + await self._upload_config_text( + environment, + content=ORACLE_CHECK_SCRIPT, + remote_path=ORACLE_CHECK_PATH, + filename="check.sh", + ) + predicate = f"bash {ORACLE_CHECK_PATH}" + + env = self.session_env() + await self.exec_as_agent( + environment, command=self.start_daemon_command(), env=env + ) + await self.exec_as_agent( + environment, + command=self.create_and_wait_command(predicate), + env={**env, "GRAPHCODE_TB_GOAL": instruction}, + ) diff --git a/Tools/harbor-graphcode/pyproject.toml b/Tools/harbor-graphcode/pyproject.toml new file mode 100644 index 00000000..d8df38b4 --- /dev/null +++ b/Tools/harbor-graphcode/pyproject.toml @@ -0,0 +1,25 @@ +[project] +name = "harbor-graphcode" +version = "0.1.0" +description = "GraphCode goal loops as a Harbor installed agent, for private Terminal-Bench trials" +requires-python = ">=3.12" +dependencies = [ + "harbor @ git+https://github.com/harbor-framework/harbor@09e555a148f7c0c9995341513efda18449645a5e", +] + +[dependency-groups] +dev = ["pytest>=8", "pytest-asyncio>=0.23"] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.build.targets.wheel] +packages = ["harbor_graphcode"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] diff --git a/Tools/harbor-graphcode/tests/test_agent.py b/Tools/harbor-graphcode/tests/test_agent.py new file mode 100644 index 00000000..9210f9e5 --- /dev/null +++ b/Tools/harbor-graphcode/tests/test_agent.py @@ -0,0 +1,217 @@ +import json +import os +import subprocess +from pathlib import Path +from unittest.mock import AsyncMock, patch + +import pytest + +from harbor.agents.installed.claude_code import ClaudeCode +from harbor_graphcode.agent import ( + BUNDLE_DIR_ENV, + GraphCode, + ORACLE_CHECK_PATH, + ORACLE_CHECK_SCRIPT, + normalize_arch, + parse_node_id, + parse_node_state, +) + +STATUS = """app (running) + 8209FF6E-3116-42E6-9A5F-46386D43CE57 running goalBased TerminalBenchTask + 11111111-2222-3333-4444-555555555555 idle timeBased Watcher +""" + + +def make_agent(tmp_path: Path, **kwargs) -> GraphCode: + logs = tmp_path / "trial" / "agent" + logs.mkdir(parents=True) + return GraphCode(logs_dir=logs, model_name="anthropic/claude-opus-5", **kwargs) + + +def ok_env(stdout: str = "") -> AsyncMock: + environment = AsyncMock() + environment.default_user = None + environment.exec.return_value = AsyncMock(return_code=0, stdout=stdout, stderr="") + return environment + + +def commands(environment: AsyncMock) -> list[str]: + return [call.kwargs["command"] for call in environment.exec.call_args_list] + + +class TestParsing: + def test_node_id_is_the_goal_loop_with_the_task_title(self): + assert parse_node_id(STATUS) == "8209FF6E-3116-42E6-9A5F-46386D43CE57" + + def test_node_id_absent_when_no_goal_loop(self): + assert parse_node_id("app (idle)\n no loops yet") is None + + def test_state_reads_display_state_column(self): + assert parse_node_state(STATUS, "11111111-2222-3333-4444-555555555555") == "idle" + assert parse_node_state(STATUS, "missing") is None + + @pytest.mark.parametrize( + ("uname", "arch"), + [("x86_64\n", "x86_64"), ("amd64", "x86_64"), ("aarch64", "aarch64"), ("arm64\n", "aarch64")], + ) + def test_arch_aliases(self, uname, arch): + assert normalize_arch(uname) == arch + + def test_unknown_arch_is_refused(self): + with pytest.raises(ValueError): + normalize_arch("riscv64") + + +class TestShellScripts: + """The container scripts run under bash; the awk the adapter relies on must agree + with the Python parsers on real status output.""" + + def test_generated_scripts_parse(self, tmp_path): + agent = make_agent(tmp_path) + for script in ( + agent.start_daemon_command(), + agent.create_and_wait_command("bash /opt/graphcode-oracle/check.sh"), + ORACLE_CHECK_SCRIPT, + ): + subprocess.run(["bash", "-n"], input=script, text=True, check=True) + + def test_awk_node_id_matches_python(self, tmp_path): + agent = make_agent(tmp_path) + script = agent.create_and_wait_command(None) + awk_line = next(line for line in script.splitlines() if line.startswith("node=")) + program = awk_line.split("awk '", 1)[1].split("'", 1)[0] + status = tmp_path / "create.txt" + status.write_text(STATUS) + out = subprocess.run( + ["awk", program, str(status)], capture_output=True, text=True, check=True + ).stdout.strip() + assert out == parse_node_id(STATUS) + + @pytest.mark.parametrize(("reward", "passes"), [("1", True), ("1.0", True), ("0", False), ("0.5", False)]) + def test_oracle_check_reads_reward_and_clears_it(self, tmp_path, reward, passes): + verifier = tmp_path / "verifier" + tests = tmp_path / "tests" + tests.mkdir() + (tests / "test.sh").write_text(f"echo {reward} > {verifier}/reward.txt\n") + script = ( + ORACLE_CHECK_SCRIPT.replace("/logs/verifier", str(verifier)) + .replace("/tests/test.sh", str(tests / "test.sh")) + .replace("/tmp/graphcode-oracle.out", str(tmp_path / "out")) + ) + result = subprocess.run(["bash", "-c", script]) + assert (result.returncode == 0) is passes + assert not (verifier / "reward.txt").exists() + + +class TestCreateCommand: + def test_tests_mode_uses_the_oracle_predicate(self, tmp_path): + agent = make_agent(tmp_path) + script = agent.create_and_wait_command(f"bash {ORACLE_CHECK_PATH}") + assert f"--predicate 'bash {ORACLE_CHECK_PATH}'" in script + assert '--goal "$GRAPHCODE_TB_GOAL"' in script + assert "--backend claudeCode" in script + + def test_agent_mode_has_no_predicate(self, tmp_path): + agent = make_agent(tmp_path, done_check="agent") + assert "--predicate" not in agent.create_and_wait_command(None) + + def test_budget_is_passed_through(self, tmp_path): + agent = make_agent(tmp_path, budget_tokens=2_000_000) + assert "--budget 2000000" in agent.create_and_wait_command(None) + + def test_transcripts_are_mirrored_where_claude_code_looks(self, tmp_path): + script = make_agent(tmp_path).create_and_wait_command(None) + assert "/logs/agent/sessions/projects" in script + + +class TestSessionEnv: + def test_every_tier_alias_is_pinned_to_the_harbor_model(self, tmp_path): + agent = make_agent(tmp_path) + with patch.dict(os.environ, {"ANTHROPIC_API_KEY": "sk-test"}, clear=False): + env = agent.session_env() + for key in ( + "ANTHROPIC_MODEL", + "ANTHROPIC_DEFAULT_HAIKU_MODEL", + "ANTHROPIC_DEFAULT_SONNET_MODEL", + "ANTHROPIC_DEFAULT_OPUS_MODEL", + ): + assert env[key] == "claude-opus-5" + assert env["IS_SANDBOX"] == "1" + assert env["GRAPHCODE_SUPPORT_DIR"] == "/tmp/gcd" + + def test_daemon_script_never_prints_the_key(self, tmp_path): + with patch.dict(os.environ, {"ANTHROPIC_API_KEY": "sk-secret-value"}, clear=False): + script = make_agent(tmp_path).start_daemon_command() + assert "sk-secret-value" not in script + assert '"claudePermissionMode": "bypassPermissions"' in script + + +class TestInstall: + async def test_uploads_the_bundle_for_the_container_arch(self, tmp_path): + bundles = tmp_path / "dist" + bundles.mkdir() + (bundles / "graphcode-linux-x86_64.tar.gz").write_bytes(b"bundle") + agent = make_agent(tmp_path, bundle_dir=str(bundles)) + environment = ok_env(stdout="x86_64\n") + with patch.object(ClaudeCode, "install", AsyncMock()) as claude_install: + await agent.install(environment) + + claude_install.assert_awaited_once() + environment.upload_file.assert_awaited_once_with( + bundles / "graphcode-linux-x86_64.tar.gz", "/tmp/graphcode-linux.tar.gz" + ) + issued = commands(environment) + assert any("apt-get install" in c and "zsh" in c for c in issued) + assert any("ln -sf /opt/graphcode/bin/graphcode /usr/local/bin/graphcode" in c for c in issued) + + async def test_missing_bundle_fails_before_upload(self, tmp_path): + agent = make_agent(tmp_path, bundle_dir=str(tmp_path)) + environment = ok_env(stdout="aarch64") + with patch.object(ClaudeCode, "install", AsyncMock()): + with pytest.raises(FileNotFoundError): + await agent.install(environment) + environment.upload_file.assert_not_awaited() + + async def test_bundle_dir_falls_back_to_the_environment(self, tmp_path): + (tmp_path / "graphcode-linux-aarch64.tar.gz").write_bytes(b"bundle") + agent = make_agent(tmp_path) + with patch.dict(os.environ, {BUNDLE_DIR_ENV: str(tmp_path)}): + assert agent.bundle_path("aarch64").is_file() + + +class TestRun: + def write_task(self, tmp_path: Path) -> Path: + task = tmp_path / "task" + (task / "tests").mkdir(parents=True) + (task / "tests" / "test.sh").write_text("#!/bin/bash\n") + return task + + def test_tests_dir_comes_from_the_trial_config(self, tmp_path): + task = self.write_task(tmp_path) + agent = make_agent(tmp_path) + (tmp_path / "trial" / "config.json").write_text( + json.dumps({"task": {"path": str(task)}, "trial_name": "t"}) + ) + assert agent.resolve_tests_dir() == task / "tests" + + async def test_tests_mode_uploads_tests_then_creates_a_predicated_loop(self, tmp_path): + task = self.write_task(tmp_path) + agent = make_agent(tmp_path, tests_dir=str(task / "tests")) + environment = ok_env() + with patch.dict(os.environ, {"ANTHROPIC_API_KEY": "sk-test"}, clear=False): + await agent.run("Make the tests pass", environment, AsyncMock()) + + environment.upload_dir.assert_awaited_once_with(task / "tests", "/tests") + last = environment.exec.call_args_list[-1].kwargs + assert "--predicate" in last["command"] + assert last["env"]["GRAPHCODE_TB_GOAL"] == "Make the tests pass" + assert "Make the tests pass" not in last["command"] + + async def test_agent_mode_uploads_nothing(self, tmp_path): + agent = make_agent(tmp_path, done_check="agent") + environment = ok_env() + with patch.dict(os.environ, {"ANTHROPIC_API_KEY": "sk-test"}, clear=False): + await agent.run("Do the task", environment, AsyncMock()) + environment.upload_dir.assert_not_awaited() + assert "--predicate" not in environment.exec.call_args_list[-1].kwargs["command"] diff --git a/docs/benchmarks/terminal-bench-trial.md b/docs/benchmarks/terminal-bench-trial.md new file mode 100644 index 00000000..9ee79da6 --- /dev/null +++ b/docs/benchmarks/terminal-bench-trial.md @@ -0,0 +1,63 @@ +# Terminal-Bench trial: GraphCode goal loop vs. plain Claude Code + +A private comparison, never submitted to a leaderboard. Question: on the same model, does +a GraphCode goal loop solve more Terminal-Bench tasks than Harbor's `claude-code` agent? + +## Status: not run + +| Prerequisite | State (2026-09-13) | +|---|---| +| Adapter (`Tools/harbor-graphcode`) | ✅ written, unit tests pass without Docker | +| Linux bundle recipe (`build-linux-bundle.sh`) | ⚠️ written, unverified | +| Container runtime (Docker or compatible) | ❌ not installed on the trial machine | +| `ANTHROPIC_API_KEY` | ❌ not set | + +No tasks have been run, so there are no results below. Installing a container runtime and +providing a key is a human decision; this trial does not do either. + +## Plan + +| Step | Scope | Gate | +|---|---|---| +| 1. Build bundle | `ARCH=x86_64 Tools/harbor-graphcode/build-linux-bundle.sh` | Docker present | +| 2. Smoke | 2 tasks × 1 attempt, both agents | API key present | +| 3. Subset trial | 15 tasks × 1 attempt, both agents | Cost estimate approved | + +- Dataset: `terminal-bench/terminal-bench-2` (pin the version at run time) +- Model: `anthropic/claude-opus-5` for both agents +- GraphCode arm runs with `done_check=agent`. `done_check=tests` hands the loop the + verifier's own test, an oracle `claude-code` lacks, so it is reported only as a + separate upper bound, never as the comparison. + +## Cost estimate + +Opus 5 list prices: $5 / MTok input, $25 / MTok output. Cache reads and writes are assumed +at the standard 0.1× and 1.25× input multipliers ($0.50 and $6.25 / MTok). + +| Per task (assumed) | Tokens | Cost | +|---|---|---| +| Cache reads | 1.8M | $0.90 | +| Cache writes | 150K | $0.94 | +| Uncached input | 10K | $0.05 | +| Output | 40K | $1.00 | +| **`claude-code`** | | **≈ $3** | +| **GraphCode loop** (≈1.7× for re-wakes) | | **≈ $5** | + +| Run | Trials | Estimate | +|---|---|---| +| Smoke | 2 tasks × 2 agents | ≈ $16 | +| Subset | 15 tasks × 2 agents | ≈ $120 | +| Subset + oracle arm | 15 tasks × 3 arms | ≈ $195 | + +These are assumptions, good to roughly 2× either way. The smoke run's `harbor` token +counts replace them before the subset is approved. + +## Results + +| Task | `claude-code` | GraphCode (`agent`) | GraphCode (`tests`, oracle) | +|---|---|---|---| +| — | not run | not run | not run | + +## Conclusion + +None yet: nothing has been run, so this trial cannot say whether the goal loop helps. From 49c59b5cd2a1bbd09e8da314298ecc19735dfef5 Mon Sep 17 00:00:00 2001 From: scgopi Date: Sun, 13 Sep 2026 22:06:28 -0700 Subject: [PATCH 2/2] Default the Harbor adapter to the fair agent done check done_check=tests makes the task's own test.sh the loop's predicate, an oracle plain claude-code never gets, so it stays an opt-in upper-bound arm and done_check=agent becomes the default. Co-Authored-By: Claude Opus 5 (1M context) --- Tools/harbor-graphcode/README.md | 10 +++++++--- Tools/harbor-graphcode/harbor_graphcode/agent.py | 11 ++++++----- Tools/harbor-graphcode/tests/test_agent.py | 9 ++++++--- docs/benchmarks/terminal-bench-trial.md | 2 +- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/Tools/harbor-graphcode/README.md b/Tools/harbor-graphcode/README.md index 3af91e0d..d53eedc1 100644 --- a/Tools/harbor-graphcode/README.md +++ b/Tools/harbor-graphcode/README.md @@ -25,8 +25,8 @@ Harbor itself, and nothing here submits to a leaderboard. | `done_check` | Loop predicate | Fair against `claude-code`? | |---|---|---| -| `tests` (default) | The task's own `tests/test.sh`, uploaded to `/tests` before the run; passes when it writes reward ≥ 1 | ❌ An oracle: the loop sees the verifier's verdict, plain `claude-code` does not. Only an upper bound. | -| `agent` | None; the loop resolves when the session runs `graphcode node done` | ✅ The same information both agents get | +| `agent` (default) | None; the loop resolves when the session runs `graphcode node done` | ✅ The same information both agents get | +| `tests` (opt-in) | The task's own `tests/test.sh`, uploaded to `/tests` before the run; passes when it writes reward ≥ 1 | ❌ An oracle: the loop sees the verifier's verdict, plain `claude-code` does not. Only an upper bound. | The tests directory is read from the trial's `config.json` (or the `tests_dir` kwarg). Harbor empties `/tests` and re-uploads it before verification, and the check deletes the @@ -58,7 +58,11 @@ export GRAPHCODE_LINUX_BUNDLE_DIR=$PWD/dist uv run harbor run -d terminal-bench/terminal-bench-2 -e docker -m anthropic/claude-opus-5 \ -a claude-code -l 2 -k 1 uv run harbor run -d terminal-bench/terminal-bench-2 -e docker -m anthropic/claude-opus-5 \ - -a harbor_graphcode:GraphCode --ak done_check=agent -l 2 -k 1 + -a harbor_graphcode:GraphCode -l 2 -k 1 + +# Oracle upper bound, reported separately, never as the comparison +uv run harbor run -d terminal-bench/terminal-bench-2 -e docker -m anthropic/claude-opus-5 \ + -a harbor_graphcode:GraphCode --ak done_check=tests -l 2 -k 1 ``` Check `harbor run --help` for the task-selection flag your Harbor version uses before diff --git a/Tools/harbor-graphcode/harbor_graphcode/agent.py b/Tools/harbor-graphcode/harbor_graphcode/agent.py index 4163bc51..91054220 100644 --- a/Tools/harbor-graphcode/harbor_graphcode/agent.py +++ b/Tools/harbor-graphcode/harbor_graphcode/agent.py @@ -79,12 +79,13 @@ class GraphCodeOptions(ClaudeCodeOptions): f"(falls back to ${BUNDLE_DIR_ENV})." ), ) - done_check: Literal["tests", "agent"] = Field( - default="tests", + done_check: Literal["agent", "tests"] = Field( + default="agent", description=( - "tests: the loop's predicate is the task's own test.sh (an oracle plain " - "claude-code does not get). agent: no predicate; the loop resolves when the " - "session runs `graphcode node done`." + "agent: no predicate; the loop resolves when the session runs `graphcode " + "node done` — the same information plain claude-code gets. tests (opt-in): " + "the loop's predicate is the task's own test.sh, an oracle claude-code does " + "not get, so it only measures an upper bound." ), ) tests_dir: str | None = Field( diff --git a/Tools/harbor-graphcode/tests/test_agent.py b/Tools/harbor-graphcode/tests/test_agent.py index 9210f9e5..313f56d9 100644 --- a/Tools/harbor-graphcode/tests/test_agent.py +++ b/Tools/harbor-graphcode/tests/test_agent.py @@ -112,8 +112,11 @@ def test_tests_mode_uses_the_oracle_predicate(self, tmp_path): assert '--goal "$GRAPHCODE_TB_GOAL"' in script assert "--backend claudeCode" in script + def test_default_done_check_is_the_fair_agent_arm(self, tmp_path): + assert make_agent(tmp_path).options.done_check == "agent" + def test_agent_mode_has_no_predicate(self, tmp_path): - agent = make_agent(tmp_path, done_check="agent") + agent = make_agent(tmp_path) assert "--predicate" not in agent.create_and_wait_command(None) def test_budget_is_passed_through(self, tmp_path): @@ -197,7 +200,7 @@ def test_tests_dir_comes_from_the_trial_config(self, tmp_path): async def test_tests_mode_uploads_tests_then_creates_a_predicated_loop(self, tmp_path): task = self.write_task(tmp_path) - agent = make_agent(tmp_path, tests_dir=str(task / "tests")) + agent = make_agent(tmp_path, done_check="tests", tests_dir=str(task / "tests")) environment = ok_env() with patch.dict(os.environ, {"ANTHROPIC_API_KEY": "sk-test"}, clear=False): await agent.run("Make the tests pass", environment, AsyncMock()) @@ -209,7 +212,7 @@ async def test_tests_mode_uploads_tests_then_creates_a_predicated_loop(self, tmp assert "Make the tests pass" not in last["command"] async def test_agent_mode_uploads_nothing(self, tmp_path): - agent = make_agent(tmp_path, done_check="agent") + agent = make_agent(tmp_path) environment = ok_env() with patch.dict(os.environ, {"ANTHROPIC_API_KEY": "sk-test"}, clear=False): await agent.run("Do the task", environment, AsyncMock()) diff --git a/docs/benchmarks/terminal-bench-trial.md b/docs/benchmarks/terminal-bench-trial.md index 9ee79da6..eaeacc36 100644 --- a/docs/benchmarks/terminal-bench-trial.md +++ b/docs/benchmarks/terminal-bench-trial.md @@ -25,7 +25,7 @@ providing a key is a human decision; this trial does not do either. - Dataset: `terminal-bench/terminal-bench-2` (pin the version at run time) - Model: `anthropic/claude-opus-5` for both agents -- GraphCode arm runs with `done_check=agent`. `done_check=tests` hands the loop the +- GraphCode arm runs with the default `done_check=agent`. The opt-in `done_check=tests` hands the loop the verifier's own test, an oracle `claude-code` lacks, so it is reported only as a separate upper bound, never as the comparison.