diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a4f91f63..77e7386d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -55,7 +55,9 @@ jobs: cache-dependency-path: runner/requirements.txt # httpx: the runner suite drives the app through FastAPI's TestClient, which needs it. The # runner itself does not, so it stays out of requirements.txt and lives with pytest here. - - run: pip install -r runner/requirements.txt pytest pytest-asyncio httpx + # The System One Harness is on the runner's venv in the image (docker/entrypoint.sh pins the tag); + # the driver's tests import it, and skip without it, so the suite installs the same pin. + - run: pip install -r runner/requirements.txt pytest pytest-asyncio httpx "systemone-harness @ git+https://github.com/HarnessRouter/SystemOneHarness@v0.3.1" - run: python -m pytest runner/tests -q conformance: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 7f1e97c4..babd2e5f 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -177,7 +177,7 @@ export HOSTNAME=0.0.0.0 TOOLS="$DATA_DIR/agent-tools" export PATH="$TOOLS/bin:$PATH" export NODE_PATH="$TOOLS/lib/node_modules" -export HR_BACKENDS="${HR_BACKENDS:-claude,codex,hermes,pi,dsh,opencode,qwen,gemini,cline,omp,goose,kimi,aider,openhands}" +export HR_BACKENDS="${HR_BACKENDS:-claude,codex,hermes,pi,dsh,opencode,qwen,gemini,cline,omp,goose,kimi,aider,openhands,systemone}" wanted() { [[ ",$HR_BACKENDS," == *",$1,"* ]]; } # The executable IS the definition of "installed" — an installer that exits 0 without producing @@ -199,6 +199,7 @@ backend_bin() { kimi) echo "$TOOLS/bin/kimi" ;; aider) echo "$TOOLS/aider-venv/bin/aider" ;; openhands) echo "$TOOLS/openhands-venv/bin/python" ;; + systemone) echo "$TOOLS/systemone-venv/bin/python" ;; esac } @@ -341,6 +342,55 @@ KIMI_PIN="${HR_KIMI_VERSION:-2.0.0}"; KIMI_PIN="${KIMI_PIN#v}" # # Own venv, the dsh/hermes precedent: it pins litellm, fastmcp, pydantic and a browser stack, and # must not share the runner's interpreter. +# The System One Harness (github.com/HarnessRouter/SystemOneHarness, Apache-2.0): a loop over a +# decision model, pinned by git tag. Its own venv on the data volume like openhands and aider: three +# small dependencies (httpx, pyyaml, the MCP SDK), installed once and rebuilt when the pin moves. +# The executable is the venv's python, which the runner hands runner/systemone_driver.py. +SYSTEMONE_PIN="${HR_SYSTEMONE_VERSION:-0.3.1}"; SYSTEMONE_PIN="${SYSTEMONE_PIN#v}" +# HR_SYSTEMONE_SPEC overrides where pip takes the package from (a mirror, a fork, a local tree +# copied into a derived image); the version proven below is the pin either way. The browser extra +# brings Browser Use, and with it the page and game environments; the Chromium they drive is +# installed beside the venv on the data volume (Playwright's, world-readable), because the image +# ships no browser and a session process cannot read root's cache. +SYSTEMONE_SPEC="${HR_SYSTEMONE_SPEC:-systemone-harness[browser] @ git+https://github.com/HarnessRouter/SystemOneHarness@v${SYSTEMONE_PIN}}" +export PLAYWRIGHT_BROWSERS_PATH="$TOOLS/ms-playwright" +# The libraries that Chromium links against live in the container, not on the volume the browser +# is on, so a container recreated over a volume that already holds the browser has the binary and +# none of its libraries (measured: libatk, libatspi, libXcomposite "not found", and every launch +# died before CDP came up). They are installed per container, keyed on a marker in the container's +# own filesystem: on the first install, and on every start that finds the venv already there. +CHROMIUM_LIBS_MARK=/var/lib/harnessrouter/chromium-libs +chromium_libs() { + [ -f "$CHROMIUM_LIBS_MARK" ] && return 0 + echo "[harnessrouter] installing the system libraries Chromium links against (this container)…" + "$TOOLS/systemone-venv/bin/playwright" install-deps chromium >/dev/null 2>&1 || return 1 + mkdir -p "$(dirname "$CHROMIUM_LIBS_MARK")" && : > "$CHROMIUM_LIBS_MARK" +} +install_systemone() { + # A failed install leaves NO venv behind: the executable is the definition of installed, and a + # venv whose pip step failed reported the base as available on a box where it could not run. + "${HR_SYSTEMONE_BASE_PYTHON:-python3}" -m venv "$TOOLS/systemone-venv" || { rm -rf "$TOOLS/systemone-venv"; return 1; } + "$TOOLS/systemone-venv/bin/pip" install -q --disable-pip-version-check "$SYSTEMONE_SPEC" playwright \ + || { rm -rf "$TOOLS/systemone-venv"; return 1; } + if ! ls "$PLAYWRIGHT_BROWSERS_PATH"/chromium-*/chrome-linux*/chrome >/dev/null 2>&1; then + echo "[harnessrouter] installing a Chromium for the System One base (Playwright's) under $PLAYWRIGHT_BROWSERS_PATH …" + "$TOOLS/systemone-venv/bin/playwright" install chromium \ + || { rm -rf "$TOOLS/systemone-venv"; return 1; } + chmod -R a+rX "$PLAYWRIGHT_BROWSERS_PATH" 2>/dev/null || true + fi + chromium_libs || { rm -rf "$TOOLS/systemone-venv"; return 1; } + # Prove the loop imports and the version is the pin, the way every other installer here proves + # its executable: an install that cannot import is a base that dies on its first turn. + "$TOOLS/systemone-venv/bin/python" -c ' +import sys +import systemone_harness, systemone_harness.envs.mcp, systemone_harness.envs.browser # noqa: F401 - the loop, the MCP adapter, the browser +import browser_use # noqa: F401 - the browser extra +have = systemone_harness.__version__ +if have != sys.argv[1]: + print(f"systemone-harness {have} installed, {sys.argv[1]} pinned", file=sys.stderr); sys.exit(1) +' "$SYSTEMONE_PIN" || { rm -rf "$TOOLS/systemone-venv"; return 1; } +} + install_openhands() { oh_py="${HR_OPENHANDS_BASE_PYTHON:-python3}" "$oh_py" -m venv "$TOOLS/openhands-venv" || return 1 @@ -592,6 +642,19 @@ install_backends() { try_install "OpenHands" install_openhands || true fi + s1_have="" + if [ -x "$(backend_bin systemone)" ]; then + s1_have="$("$(backend_bin systemone)" -c 'import systemone_harness as s; print(s.__version__)' 2>/dev/null || true)" + fi + if wanted systemone && [ "$s1_have" != "$SYSTEMONE_PIN" ]; then + rm -rf "$TOOLS/systemone-venv" + echo "[harnessrouter] installing System One Harness $SYSTEMONE_PIN (Apache-2.0)…" + try_install "System One Harness" install_systemone || true + fi + if wanted systemone && [ -x "$TOOLS/systemone-venv/bin/playwright" ]; then + chromium_libs || echo "[harnessrouter] WARNING: Chromium's system libraries could not be installed; the System One base's browser environments will not start until they are (retried on the next start)" + fi + if wanted kimi && [ "$("$(backend_bin kimi)" --version 2>/dev/null | head -n 1)" != "$KIMI_PIN" ]; then echo "[harnessrouter] installing Kimi Code CLI $KIMI_PIN (MIT, version-pinned)…" try_install "Kimi Code CLI" install_kimi || true diff --git a/docs/harness-verification.md b/docs/harness-verification.md index 86fc1d98..426e1523 100644 --- a/docs/harness-verification.md +++ b/docs/harness-verification.md @@ -20,6 +20,11 @@ For every harness and every model its menu offers, one session runs five scenari | Artifact | a file the task must produce exists in the turn record and is shown to the reader | | Recycle | the sandbox is let go on purpose, then a follow-up still recalls the first message | +One base is not a coding agent. `systemone` runs a decision loop over a typed action space rather +than a CLI over a workspace, so the artifact scenario's prompt does not apply to it; it is verified by +its own loop (its package's tests, its UHP conformance and its benchmark) and by the first-turn, +follow-up, switch and recycle scenarios here. See docs/support-matrix-notes.md, "systemone". + ## The rules that decide a row A scenario that "completed" is not a pass on its own. Four rules turn a run into a verdict, and each diff --git a/docs/self-hosting-guide.md b/docs/self-hosting-guide.md index 0ddfeee8..7b650c22 100644 --- a/docs/self-hosting-guide.md +++ b/docs/self-hosting-guide.md @@ -650,7 +650,7 @@ Backends are installed into your data volume rather than baked into the image, s want is a run-time setting: ```bash -docker run -e HR_BACKENDS=claude,codex,hermes,pi,dsh,opencode,qwen,gemini,cline,omp,goose,kimi,aider,openhands ... # the default +docker run -e HR_BACKENDS=claude,codex,hermes,pi,dsh,opencode,qwen,gemini,cline,omp,goose,kimi,aider,openhands,systemone ... # the default docker run -e HR_BACKENDS=opencode ... # lean ``` diff --git a/docs/support-matrix-notes.md b/docs/support-matrix-notes.md index 2f1ba3a4..721f0aed 100644 --- a/docs/support-matrix-notes.md +++ b/docs/support-matrix-notes.md @@ -1266,3 +1266,51 @@ fourteen backends: the OpenHands venv installs in about a minute beside aider's. of the turn survives the sweep. The agent doc reaches the model as context: asked, with no tool allowed, for a secret word in the harness's instructions and the installed skills, it answered both from the doc. + +## systemone: the System One Harness as the fifteenth base (2026-09-19) + +Not a coding CLI. The base runs the open-source System One Harness +(github.com/HarnessRouter/SystemOneHarness, Apache-2.0, pinned at v0.3.1 in `docker/entrypoint.sh`) +over TypeSafe's Jev, a decision model: it answers typed questions (a choice, a yes/no probability, +a score) with probabilities in one pass and writes no text. Every step is one request carrying +the next action as a choice over what the environment offers right now, every parameter of every +offered action, and a goal check; the harness gates the answer by the action's risk and executes. +The turn process is `runner/systemone_driver.py`; its events are claude's stream-json, so the +normaliser is the passthrough. + +### Measured, 2026-09-19 + +- **Through the runner's own relay.** `_build_systemone` registers the route at the provider's API + ROOT (`https://openrouter.ai/api`), because OpenRouter serves decisions at `/api/alpha/decisions` + and a POST to `/api/v1/alpha/decisions` is a 404. The driver posts to `/v1/alpha/decisions`. + One live turn on the built-in order desk: six actions, `success`, 1.44 s wall; the relay's taps + read the served model `typesafe/jev-1.13-20260917` and usage 6304 in / 1410 out off the answer, + and the driver's own result event said the same. Rule 2 of harness-verification.md holds the + way it holds for cline and qwen: the base rides the relay. +- **The environment is the harness's MCP server** (the first one configured), its tools compiled to + actions at the start of each turn; a tool that needs free text is named in the trace as not + offered. With no server, the built-in order desk, so the base answers before anything is + configured. `disabledTools` withholds an action from the question itself (hard, by omission). +- **`incomplete` is a runner status now.** A loop that stops because the model asked for help or a + destructive action never cleared its confidence bar is neither a failure nor a step cap. The + driver's result carries `subtype: incomplete` and a `reason`; `_status_from_result` returns + `incomplete`, the poll body carries `reason`, and the gateway reports it as the task's + `incomplete_details.reason` without retrying another connection. Pinned by + `runner/tests/test_systemone_backend.py` and `gateway/tests/test_systemone_catalog.py`. +- **Continuation.** The desk's state and the loop's steps persist under + `.harness/systemone//` in the workspace; a resumed turn carries on from where the last + one stopped and the model sees the earlier steps as history (a two-step first turn followed by a + continuation redid none of its picks). +- **What the five matrix scenarios mean here.** First turn, follow-up, switch and recycle apply as + written; the artifact scenario's "create a file" prompt does not, because the desk produces one + artifact of its own (`manifest.json`, on ship). The base is verified by its own loop rather than + the coding prompts: the harness package's 35 tests, its UHP core conformance (40 of 40) and its + benchmark (15 of 15 goals on the live model) are the record, in that repository. + +### The models + +`jev-1.13` and `jev-latest`, OpenRouter only (`typesafe/jev-1.13`, `~typesafe/jev-latest`), added to +OpenRouter's vendor table after the shared copy so no other aggregator inherits an id it cannot +serve. No chat model is listed on this base: the loop asks typed questions a text model cannot +answer. TypeSafe's direct endpoint (`/v1/systemone`) is wired in the runner and not yet offered by +the gateway. diff --git a/gateway/app.py b/gateway/app.py index b9c4b098..12ae3145 100644 --- a/gateway/app.py +++ b/gateway/app.py @@ -1087,6 +1087,9 @@ def _conn_public(conn: dict) -> dict: ("anthropic", "openhands"): "anthropic", ("openai", "openhands"): "openai", ("azure-foundry", "openhands"): "azure", ("openrouter", "openhands"): "openai-api", + # systemone speaks the provider's decisions endpoint through the loopback relay; OpenRouter is + # the one aggregator serving TypeSafe's Jev (beta since 2026-09-18), so it is the one wiring. + ("openrouter", "systemone"): "openrouter", ("tokenrouter", "openhands"): "tokenrouter", ("vercel", "openhands"): "tokenrouter", ("llmtr", "openhands"): "tokenrouter", ("custom", "openhands"): "openai-api", @@ -2485,13 +2488,15 @@ def _emit(ev): await _vertex_upsert(sid, {"cli_session_id": s["session_id"]}) if s.get("done"): st = s.get("status") - terminal = ("completed" if st == "done" else "incomplete" if st == "max_turns" + terminal = ("completed" if st == "done" else "incomplete" if st in ("max_turns", "incomplete") else "cancelled" if st == "cancelled" else "incomplete" if st == "timeout" else "failed") if st == "max_turns": translator.incomplete_reason = "max_steps" elif st == "timeout": translator.incomplete_reason = "timeout" + elif st == "incomplete": + translator.incomplete_reason = str(s.get("reason") or "incomplete") break if terminal is None: return False # still running / adoption interrupted — later sweep retries @@ -2933,8 +2938,9 @@ async def _harness_plugins(harness_id: str, org: str, hdr_vals: dict[str, str] | # of the deployment rather than of a saved configuration. Returning nothing here is why # the default harnesses, which is what most people actually use, had no image generation # and no document skills while custom ones did. - return [], _builtin_default_skills(), [], [], [] + return [], (_builtin_default_skills() if _base_takes_skills(harness_id) else []), [], [], [] v = await _mcp_migrate(org, harness_id, v) + takes_skills = _base_takes_skills(str(v.get("base") or harness_id)) def _arr(prop): try: @@ -3088,6 +3094,8 @@ def _arr(prop): # Built-ins the harness never mentions: on when the image says so. Implicit, so the set follows # the image rather than whatever was true when the Harness was created. skills_out += _builtin_default_skills(seen) + if not takes_skills: + skills_out = [] # nothing on this base can read a skill; see _BASE_CATALOG["systemone"] disabled_tools = [t for t in _arr("disabled_tools") if isinstance(t, str)] return mcp_out, skills_out, suppressed, disabled_tools, plugins_out @@ -4629,6 +4637,15 @@ def _provider_backends(provider: str) -> list[str]: } +def _custom_can_drive(backend: str) -> bool: + """Whether ANY custom-endpoint format can run this backend. A custom integration's models are + shown greyed on a backend its format cannot drive, so the reader learns why a model they + configured is not pickable there; on a backend no custom format drives at all (systemone runs + System One models only, and a custom endpoint speaks OpenAI or Anthropic text shapes) the row + is not an explanation but a chat model offered on a harness that cannot use one.""" + return any(backend in bs for bs in _CUSTOM_FORMAT_BACKENDS.values()) + + def _integration_serves_backend(integ: dict, backend: str) -> bool: """Can this integration actually run a turn on `backend`? For a custom provider this is gated by its api_format (see _CUSTOM_FORMAT_BACKENDS); every other provider just needs a @@ -5777,6 +5794,11 @@ async def _hosted_models_refresh(api_key: str = "", force: bool = False, models_ _OPENROUTER_NO_CHANNEL: set[str] = set() _VENDOR_MODELS["openrouter"] = {c: _OPENROUTER_RESLUG.get(c, v) for c, v in _SHARED_SLUGS.items() if c not in _OPENROUTER_NO_CHANNEL} +# System One models. TypeSafe's Jev is a decision model (typed answers with probabilities, no text) +# served by OpenRouter on its decisions endpoint and by no other aggregator here, so it is added +# AFTER the shared copy: TokenRouter and Vercel must not inherit an id they cannot serve. Only the +# systemone base lists these ids in its catalog, so no chat backend's picker ever shows them. +_VENDOR_MODELS["openrouter"].update({"jev-1.13": "typesafe/jev-1.13", "jev-latest": "~typesafe/jev-latest"}) # Vercel's AI Gateway carries the same catalogue under nearly the same slugs, so it starts from # OpenRouter's table too. Only the vendor prefix differs on four of them, and it differs because @@ -6237,6 +6259,10 @@ def _valid_default() -> str | None: "deepseek-v4.1-flash", "qwen3.8-flash", "qwen3.8-27b", "qwen3.7-plus", "hunyuan-4-preview", "nemotron-3.5-lightning", "nemotron-3-super", "grok-4.6", "grok-4.5", "grok-4.3", "grok-4.20", "grok-build-0.1", "muse-spark-1.3", "muse-spark-1.2", "muse-spark-1.1", "muse-glimmer-30b", "llama-4-maverick", "llama-3.3-70b"]}, } _MODEL_CATALOG["omp"]["models"] = list(_MODEL_CATALOG["pi"]["models"]) # pi's reach, see the omp entry +# systemone: the two ids OpenRouter serves for Jev, measured live 2026-09-19 (jev-1.13 resolves to +# typesafe/jev-1.13-20260917; jev-latest is OpenRouter's rolling alias of the same). A chat model is +# not offered here: this base asks typed questions and a text model cannot answer them. +_MODEL_CATALOG["systemone"] = {"default": "jev-1.13", "models": ["jev-1.13", "jev-latest"]} # A pair the matrix failed twice on the one aggregator that serves the id is not offered on that # harness (2026-09-13, five scenarios each): llama-4-maverick on OpenRouter under qwen writes the # tool call as prose; llama-3.3-70b on OpenRouter fails the recall under goose, the artifact under @@ -6515,7 +6541,7 @@ async def _harness_models_view(hv: dict | None, backend: str, servable: set[str] if canonical in seen: continue integ = integrations.get(iname) - if integ and str(integ.get("provider") or "").lower() == "custom": + if integ and str(integ.get("provider") or "").lower() == "custom" and _custom_can_drive(backend): models.append({"id": canonical, "label": canonical, "backend": backend, "available": False, "default": canonical == default}) seen.add(canonical) @@ -7030,6 +7056,13 @@ async def _resp_execute(translator: _RespTranslator, *, org: str, member: str, s elif st == "max_turns": terminal = "incomplete" translator.incomplete_reason = "max_steps" + elif st == "incomplete": + # The loop stopped for a reason of its own (systemone: the model asked for help, + # or a destructive action never cleared its confidence bar). Neither a failure + # nor a budget: the reason is the task's incomplete_details, and nothing is + # retried on another connection, because nothing went wrong with this one. + terminal = "incomplete" + translator.incomplete_reason = str(s.get("reason") or "incomplete") elif st in ("cancelled", "timeout"): # user cancel / wall-clock cap end the SESSION's turn — never retry it # on the next connection in the chain (that would re-run the whole task) @@ -7475,6 +7508,16 @@ async def create_response(body: CreateResponseBody, request: Request): # harness id: request metadata, else the X-Harness-Id header (set by a front proxy that maps # {harness_id}/v1/* -> /v1/*, or by the native public-shape route above). harness_id = str(meta.get("harness_id") or request.headers.get("x-harness-id") or "") + if not harness_id and body.previous_response_id: + # A continuation belongs to its session's harness. A client that does not repeat harness_id + # on a follow-up (the protocol asks only for previous_response_id) used to be routed by the + # inherited MODEL NAME, and for a base whose models no chat backend serves that fell to the + # default backend: a systemone follow-up asked claude for jev-1.13 (measured 2026-09-19). + # The session vertex records the harness the conversation started on; that is the harness. + _pr = await _resp_get(body.previous_response_id) + _psid = str(((_pr or {}).get("metadata") or {}).get("session_id") or "") + _pv = await _vertex_get(_psid) if _psid else None + harness_id = str((_pv or {}).get("harness_id") or "") harness_name = str(meta.get("harness_name") or "") hv = await _harness_vertex(harness_id) if harness_id else None # A deleted harness cannot run new turns (same 404 as the read endpoints). Cross-org runs are @@ -13254,6 +13297,12 @@ def _builtin_skills() -> dict: return out +def _base_takes_skills(base_id: str) -> bool: + """Whether skills, built-in or added, mean anything on this base. A base that declares + `"skills": False` in the catalog (systemone) gets none mounted and none offered.""" + return bool((_BASE_CATALOG.get(str(base_id or "")) or {}).get("skills", True)) + + def _builtin_default_skills(seen: set[str] | None = None) -> list[dict]: """Built-ins that are on by default, minus any the harness has its own entry for. @@ -13474,6 +13523,30 @@ def _builtin_default_skills(seen: set[str] | None = None) -> list[dict]: # of every backend here that has a shell. "tool_enforcement": "hard", }, + "systemone": { + "label": "System One", "backend": "systemone", "status": "ready", + # Delivered as the loop's task instructions on every step, beside the environment's own. + "system_prompt": ("You act inside a finite set of actions the environment offers each step. " + "Choose the action that moves the goal forward, finish when the goal is " + "reached, and escalate when nothing offered fits."), + # Nothing is built into this base: its actions are its environment's, the tools of the MCP + # server the harness configures (a kit's plugin, a server added by hand), compiled at the + # start of every turn. Listing the order desk's six actions here showed them on the Super + # Mario harness as "built into System One", enabled, while that harness never offers them. + # Without a server the loop runs the order desk as a demonstration, which is the base's + # description's business, not a tool list's. A tool disabled on the harness is withheld by + # OMISSION from the question the model answers, whatever environment offers it (pinned by + # runner/tests/test_systemone_backend.py). + "tools": [], + "tool_enforcement": "hard", + # No skills, built-in or added. A skill is prose an agent reads and scripts it runs from a + # shell, and each of the built-ins needs free text (an image prompt, a document's content, + # HTML for a PDF); a System One model chooses among offered actions and writes nothing, so + # a skill mounted for it is a capability the console would show enabled that can never + # act. Guidance reaches this base as instructions; a skill's scripts reach it as actions of + # an MCP server. Pinned by gateway/tests/test_systemone_catalog.py. + "skills": False, + }, "qwen": { "label": "Qwen Code", "backend": "qwen", "status": "ready", "system_prompt": ("You are Qwen Code, an autonomous coding agent. You work on a real git " @@ -14747,7 +14820,11 @@ async def launch_kit(kit_id: str, request: Request, body_in: KitLaunchBody | Non skills=[] if kit_plugin else _kit_skills(kit), plugins=[kit_plugin] if kit_plugin else [], mcp_servers=spec.get("mcp_servers") or [], - disabled_tools=spec.get("disabled_tools") or []) + disabled_tools=spec.get("disabled_tools") or [], + # A kit knows how long its turns run: a game played several decisions a + # second needs more steps than a document does. Absent, the base's default. + max_step=spec.get("max_step"), + timeout_seconds=spec.get("timeout_seconds")) body.mcp_servers = _mcp_servers_prepare(body.mcp_servers) body.skills = await _skills_prepare(body.skills) # The hosted entries launch attaches below (a kit's database, the media server) are names a @@ -15248,10 +15325,12 @@ async def list_bases(request: Request) -> dict: # skills of its own at run time that nothing outside a turn can enumerate, so # `builtinSkillsEnumerable` stays False: the console must say "and it brings its own" # rather than presenting this list as everything the agent has. - "builtinSkills": [{"name": n, "title": b2["title"], "description": b2["description"], - "defaultEnabled": b2["default_enabled"], "origin": b2["origin"]} - for n, b2 in sorted(_builtin_skills().items())], + "builtinSkills": ([{"name": n, "title": b2["title"], "description": b2["description"], + "defaultEnabled": b2["default_enabled"], "origin": b2["origin"]} + for n, b2 in sorted(_builtin_skills().items())] + if b.get("skills", True) else []), "builtinSkillsEnumerable": False, + "takesSkills": bool(b.get("skills", True)), }) # The limits a turn gets when neither the request nor the harness sets one, so the console can # show the number that will apply rather than a placeholder of its own. @@ -15293,7 +15372,7 @@ async def list_models(request: Request) -> dict: if _integration_serves_backend(integ, b): models.append({"id": canonical, "label": canonical, "backend": b, "available": True, "default": False}) - elif str(integ.get("provider") or "").lower() == "custom": + elif _custom_can_drive(b): models.append({"id": canonical, "label": canonical, "backend": b, "available": False, "default": False}) seen.add(canonical) diff --git a/gateway/tests/test_systemone_catalog.py b/gateway/tests/test_systemone_catalog.py new file mode 100644 index 00000000..10caacb0 --- /dev/null +++ b/gateway/tests/test_systemone_catalog.py @@ -0,0 +1,93 @@ +"""The systemone base: what the catalog promises is what the router can run. + +Jev is a decision model served by OpenRouter alone, so its ids live in OpenRouter's vendor table +and no other aggregator's, and only the systemone base lists them. A turn on the base is served +through the one wiring, an OpenRouter integration, and a run that stops for a reason of its own +reports that reason rather than a failure or a step cap. +""" +import os +import pathlib +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) +os.environ.setdefault("HR_BACKING", "local") +import app as gw # noqa: E402 + +JEV = {"jev-1.13": "typesafe/jev-1.13", "jev-latest": "~typesafe/jev-latest"} + + +def test_jev_is_on_openrouter_and_nowhere_else(): + for canonical, slug in JEV.items(): + assert gw._VENDOR_MODELS["openrouter"][canonical] == slug + for vendor in ("tokenrouter", "vercel", "llmtr", "openai", "anthropic", "google"): + table = gw._VENDOR_MODELS.get(vendor) or {} + assert not (set(JEV) & set(table)), f"{vendor} lists a Jev id it cannot serve" + + +def test_only_the_systemone_base_offers_jev_and_it_offers_nothing_else(): + assert gw._MODEL_CATALOG["systemone"] == {"default": "jev-1.13", "models": ["jev-1.13", "jev-latest"]} + for backend, cat in gw._MODEL_CATALOG.items(): + if backend == "systemone": + continue + assert not (set(JEV) & set(cat.get("models", []))), f"{backend} offers a Jev id it cannot answer" + + +def test_the_base_is_in_the_catalog_with_hard_enforcement_and_no_built_in_tools(): + b = gw._BASE_CATALOG["systemone"] + assert b["backend"] == "systemone" and b["status"] == "ready" and b["label"] == "System One" + assert b["tool_enforcement"] == "hard" + assert b["tools"] == [] # the actions are the environment's; nothing is built into the base + + +def test_an_openrouter_integration_drives_the_base_and_no_other_provider_claims_to(): + assert gw._INTEGRATION_WIRING[("openrouter", "systemone")] == "openrouter" + others = [k for k in gw._INTEGRATION_WIRING if k[1] == "systemone" and k[0] != "openrouter"] + assert others == [] + assert gw._integration_serves_backend({"provider": "openrouter"}, "systemone") + assert not gw._integration_serves_backend({"provider": "tokenrouter"}, "systemone") + + +def test_an_incomplete_turn_status_is_a_response_status_of_its_own(): + assert gw._RESP_STATUS_MAP["incomplete"] == "incomplete" + + +def test_a_custom_endpoints_models_never_appear_on_the_systemone_list(monkeypatch): + """Measured on hr-test 2026-09-19: a System One harness's picker showed claude-sonnet-4.6, + gpt-5.5 and four more as "(no provider)". They were the rows of custom-endpoint integrations, + appended to every backend as unavailable. No custom format can drive systemone, so nothing of + theirs belongs on its list; a chat backend keeps the greyed row as its explanation.""" + import asyncio + integ = [{"name": "my-openai", "provider": "custom", "config": {"api_format": "openai", "base_url": "https://x/v1"}}] + + async def _integrations(): + return integ + + async def _map(): + return {"gpt-5.5": "my-openai", "jev-1.13": "openrouter"} + + monkeypatch.setattr(gw, "_integrations_doc", _integrations) + monkeypatch.setattr(gw, "_effective_model_map", _map) + assert not gw._custom_can_drive("systemone") and gw._custom_can_drive("claude") and gw._custom_can_drive("hermes") + view = asyncio.run(gw._harness_models_view(None, "systemone", {"jev-1.13", "jev-latest"})) + assert [m["id"] for m in view["models"]] == ["jev-1.13", "jev-latest"] + assert all(m["available"] for m in view["models"]) + # a chat backend the custom format cannot drive still shows the row, greyed, as the explanation + view = asyncio.run(gw._harness_models_view(None, "claude", set())) + row = next(m for m in view["models"] if m["id"] == "gpt-5.5") + assert row["available"] is False + + +def test_the_base_takes_no_skills_and_none_are_mounted_for_a_turn(monkeypatch): + """The built-in skills are prose an agent reads and scripts it runs from a shell, each needing + free text; a System One model chooses among offered actions and writes nothing. So the base + declares it takes no skills, the bases endpoint offers none, and a turn mounts none, for the + built-in harness and for a harness forked from it alike.""" + import asyncio + assert gw._BASE_CATALOG["systemone"]["skills"] is False + assert not gw._base_takes_skills("systemone") and gw._base_takes_skills("codex") and gw._base_takes_skills("") + monkeypatch.setattr(gw, "_builtin_skills", lambda: {"pdf": {"title": "PDF", "description": "", "default_enabled": True, + "origin": "image", "files": [{"path": "SKILL.md", "content": "x"}]}}) + _, skills, _, _, _ = asyncio.run(gw._harness_plugins("systemone", "local", None, hv=None)) + assert skills == [] + _, skills, _, _, _ = asyncio.run(gw._harness_plugins("codex", "local", None, hv=None)) + assert [s["name"] for s in skills] == ["pdf"] diff --git a/runner/server.py b/runner/server.py index 167ffbd7..b18f50cb 100644 --- a/runner/server.py +++ b/runner/server.py @@ -1247,6 +1247,8 @@ def _write_agent_doc(cwd: str, backend: str, agent_doc: str | None, skills_meta: workspace PARENT, the user saw an empty turn, and three turns went to copying files into view. An instruction is the right mechanism here: writes cannot be walled in a sandbox whose point is real bash, and widening collection would ship every scratch file as a deliverable.""" + if backend == "systemone": + return # reads no workspace; its instructions travel in the job (see _build_systemone) p = _agent_doc_path(cwd, backend) base = (agent_doc or "").strip() lines = [_AGENTS_BEGIN, "## Workspace", "", @@ -1581,6 +1583,12 @@ def _status_from_result(result_ev: dict | None, exit_code: int) -> str: return "done" if sub == "error_max_turns": return "max_turns" + if sub == "incomplete" and not result_ev.get("is_error"): + # The loop stopped for a reason of its own that is neither a failure nor a budget: the + # model asked for help, or would not clear the confidence a destructive action needs + # (systemone). The reason rides the result event and the poll body, and the gateway + # reports it as the task's incomplete_details. + return "incomplete" return "failed" return "done" if exit_code == 0 else "failed" @@ -5671,6 +5679,73 @@ def _aider_eof(state: dict, rc: int) -> list[dict]: "/data/agent-tools/openhands-venv/bin/python") OPENHANDS_DRIVER = os.path.join(os.path.dirname(os.path.abspath(__file__)), "openhands_driver.py") +# ── systemone: the System One Harness (github.com/HarnessRouter/SystemOneHarness, Apache-2.0) ── +# A loop over a decision model rather than a coding CLI: TypeSafe's Jev answers typed questions with +# probabilities and writes no text, so the harness compiles the environment's actions into one +# request per step and gates the answer by the action's risk. The turn process is +# runner/systemone_driver.py; the model id reaches the provider verbatim (`typesafe/jev-1.13`). +SYSTEMONE_DEFAULT_MODEL = os.environ.get("SYSTEMONE_DEFAULT_MODEL", "typesafe/jev-1.13") +SYSTEMONE_PROVIDERS = {"openrouter", "typesafe"} +SYSTEMONE_PYTHON = os.environ.get("HR_SYSTEMONE_PYTHON", + "/data/agent-tools/systemone-venv/bin/python") +SYSTEMONE_DRIVER = os.path.join(os.path.dirname(os.path.abspath(__file__)), "systemone_driver.py") + + +def _systemone_relay_route(provider: str, base_url: str, api_key: str) -> tuple[str, str]: + """Register one systemone turn's upstream; -> (relay base for the driver, placeholder bearer). + + OpenRouter serves decisions at /api/alpha/decisions and NOWHERE under /api/v1 (404 on the + versioned path, measured 2026-09-19), so the route's base is the connection's API root with its + version segment removed: the driver posts to /v1/alpha/decisions, the relay drops the + /v1 it joins on and reaches https://openrouter.ai/api/alpha/decisions. TypeSafe's own endpoint + is /v1/systemone, under the version, so that base keeps it. The relay reads the served model + and the usage off the answer as it does for every backend: Jev's body names `model` and + carries `usage.input_tokens` / `output_tokens`, both shapes the relay already parses.""" + base = (base_url or "").rstrip("/") + if provider == "openrouter": + base = re.sub(r"/v\d+[a-z]*$", "", base) + else: + base = _relay_base_with_version(base) + with _HERMES_RELAY["lock"]: + if _HERMES_RELAY["server"] is None: + srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _HermesRelayHandler) + threading.Thread(target=srv.serve_forever, daemon=True).start() + _HERMES_RELAY["server"], _HERMES_RELAY["port"] = srv, srv.server_address[1] + tok = "hr-relay-" + uuid.uuid4().hex + _HERMES_RELAY["routes"][tok] = (base, api_key, {}) + return f"http://127.0.0.1:{_HERMES_RELAY['port']}/v1", tok + + +def _build_systemone(provider: str, auth: Auth, model: str, prompt: str, cwd: str, env: dict, + resume_session_id: str | None = None, mcp_servers: list[dict] | None = None, + tools_disabled: list[str] | None = None, max_turns: int | None = None, + timeout_seconds: int | None = None, agent_doc: str = "") -> list[str]: + """The System One Harness, one turn: the job rides argv as JSON, the driver emits claude-shaped + stream-json. The harness's MCP servers ARE the environment (the first one; the driver says so + in the trace when there are more); with none, the built-in order desk. `tools_disabled` are + actions withheld by omission from what the model is offered, the hardest enforcement there is. + `agent_doc` is the harness's instructions and travels in the job, not as a file: this backend + reads no workspace. The key rides the relay, like every backend; the driver holds a placeholder, + and it is in the environment as well as the job so the served-model and usage taps find the + route.""" + pr = provider or "openrouter" + if pr not in SYSTEMONE_PROVIDERS: + raise HTTPException(400, f"unknown systemone provider '{pr}' (one of {sorted(SYSTEMONE_PROVIDERS)})") + if not auth.base_url: + raise HTTPException(400, "systemone needs a base_url (none configured)") + if auth.api_key: + relay_base, relay_tok = _systemone_relay_route(pr, auth.base_url, auth.api_key) + auth = auth.model_copy(update={"base_url": relay_base, "api_key": relay_tok}) + env["SYSTEMONE_API_KEY"] = auth.api_key or "" + job = {"cwd": cwd, "model": model, "prompt": prompt, "provider": pr, + "base_url": auth.base_url, "api_key": auth.api_key, + "resume_session_id": resume_session_id, + "mcp_servers": [s for s in (mcp_servers or []) if (s or {}).get("url") or (s or {}).get("command")], + "tools_disabled": list(tools_disabled or []), + "max_turns": max_turns, "timeout_seconds": timeout_seconds, + "agent_doc": agent_doc or ""} + return [SYSTEMONE_PYTHON, SYSTEMONE_DRIVER, json.dumps(job)] + def _openhands_mcp_config(mcp_servers: list[dict] | None) -> dict: """The declared servers as the SDK's `Agent.mcp_config`: {name: MCPServer}. @@ -6018,6 +6093,12 @@ def _openhands_eof(state: dict, rc: int) -> list[dict]: "openhands": {"providers": sorted(OPENHANDS_PROVIDERS), "default_model": OPENHANDS_DEFAULT_MODEL, "normalize": _openhands_to_claude}, + # The System One driver emits claude's stream-json itself (system/init, assistant tool_use and + # thinking blocks, user tool_result, result with usage and a `reason`), so its normaliser is the + # passthrough, as qwen's is. + "systemone": {"providers": sorted(SYSTEMONE_PROVIDERS), + "default_model": SYSTEMONE_DEFAULT_MODEL, + "normalize": _claude_passthrough}, } @@ -6237,6 +6318,7 @@ def _run_turn_bg(turn_id: str, cmd: list[str], env: dict, cwd: str, normalize, m rec["status"] = ("cancelled" if rec.get("cancelled") else "timeout" if rec.get("capped") else _status_from_result(result_ev, rc)) + rec["reason"] = str((result_ev or {}).get("reason") or "") # Never leave a failure opaque: surface the captured CLI stderr (and result-event error) so the # gateway/trace shows WHY it failed (throttling, model error, etc.) instead of an empty string. if rec["status"] in ("failed", "error", "timeout"): @@ -7304,6 +7386,12 @@ def turn(req: TurnReq, identifier: str = "") -> dict: resume_session_id=req.resume_session_id, mcp_servers=req.mcp_servers, tools_disabled=req.tools_disabled, max_turns=req.max_turns) + elif backend == "systemone": + model = model or SYSTEMONE_DEFAULT_MODEL + cmd = _build_systemone(req.provider, auth, model, req.prompt, cwd, env, + resume_session_id=req.resume_session_id, mcp_servers=req.mcp_servers, + tools_disabled=req.tools_disabled, max_turns=req.max_turns, + timeout_seconds=req.timeout_seconds, agent_doc=agent_doc) elif backend == "gemini": model = model or GEMINI_DEFAULT_MODEL cmd = _build_gemini(req.provider, auth, model, req.prompt, cwd, env, @@ -7444,5 +7532,5 @@ def get_turn(turn_id: str, since: int = 0) -> dict: return {"turn_id": turn_id, "status": rec["status"], "done": rec["done"], "result": rec.get("result", ""), "exit_code": rec.get("exit_code"), "error": rec.get("error"), "backend": rec["backend"], "model": rec["model"], - "session_id": rec.get("session_id"), + "session_id": rec.get("session_id"), "reason": rec.get("reason") or "", "events": evs, "n_total": n, "elapsed": round(time.time() - rec["started"], 1)} diff --git a/runner/systemone_driver.py b/runner/systemone_driver.py new file mode 100644 index 00000000..9ead49e2 --- /dev/null +++ b/runner/systemone_driver.py @@ -0,0 +1,209 @@ +"""One System One turn, as a runner subprocess. + +Spawned per turn by server.py with the job as its one argument, the same one-process-per-turn +contract every other backend keeps: the runner reads NDJSON off stdout and cancel is a process-group +kill. Inside, the open-source System One Harness (github.com/HarnessRouter/SystemOneHarness) runs +its loop: observe the environment, compile the actions it offers right now into one request, ask +the model (TypeSafe's Jev, a decision function: typed answers with probabilities, no text), gate the +answer by the action's risk, execute, repeat, until a terminal state the loop can name. + +The events are Claude Code's stream-json, so the runner's passthrough normaliser reads them as it +reads qwen's: one `tool_use` and one `tool_result` per executed action, a `thinking` block for a +step the gate refused or the model ended, one assistant text for the loop's closing sentence, and a +`result` whose `subtype` says how the loop ended. `incomplete` with a `reason` is this backend's +addition to the contract: a loop that stops because the model asked for help, or would not clear +the confidence a destructive action needs, has neither failed nor run out of steps, and the +task record must say which it was. + +THE ENVIRONMENT IS THE HARNESS'S MCP SERVER. Its tools are compiled to actions once per turn (the +state definition convention: an `observe` tool, a `reset` tool, every other tool an action whose +parameters are enumerable; a tool that needs free text is listed in the trace as not offered). A +harness with no MCP server runs the harness's built-in order desk, one order to pick, pack and +ship, so the base works before anything is configured. A URL server keeps its own state between +turns; a stdio server is a new process each turn and must persist its own, and the order desk is +persisted here, under the workspace, keyed by the session. + +THE MODEL IS REACHED THROUGH THE LOOPBACK RELAY like every other backend: the driver holds a +placeholder bearer, the relay holds the key, and the relay reads the served model and the usage +off the provider's answer. The route's base is the provider's API root rather than its /v1, because +OpenRouter serves decisions at /api/alpha/decisions and nowhere under /api/v1 (measured 2026-09-19, +404 on the versioned path). +""" +from __future__ import annotations + +import dataclasses +import json +import pathlib +import sys +import uuid + +_KEEP_ON_STEP = ("index", "started_at", "action", "params", "action_confidence", "weakest", "threshold", + "verdict", "result", "latency_ms", "served_model", "request_id", "note") + + +def _emit(ev: dict) -> None: + print(json.dumps(ev, default=str), flush=True) + + +def _server_entry(s: dict) -> dict | None: + """A harness MCP server entry (name, url|command, transport?, auth?, headers?) as the adapter takes it.""" + s = s or {} + url = str(s.get("url") or "").strip() + headers = {str(k): str(v) for k, v in (s.get("headers") or {}).items() if k and v is not None} + auth = s.get("auth") + if auth: + headers["Authorization"] = auth if str(auth).lower().startswith("bearer ") else f"Bearer {auth}" + if url: + e: dict = {"url": url} + if headers: + e["headers"] = headers + if str(s.get("transport") or "").lower() == "sse": + e["transport"] = "sse" + return e + if s.get("command"): + return {"command": str(s["command"]), "args": [str(a) for a in (s.get("args") or [])], + "env": s.get("env") or None, "cwd": s.get("cwd")} + return None + + +def _provider_path(provider: str) -> str: + """Where the decisions endpoint sits under the route's base. OpenRouter: /api + /alpha/decisions. + TypeSafe direct: host + /v1/systemone.""" + return "/systemone" if provider == "typesafe" else "/alpha/decisions" + + +def run_turn(job: dict, provider=None, emit=_emit) -> dict: + """Run one turn and emit its events; returns the result event. `provider` is injectable for tests.""" + from systemone_harness import Controller, DecisionProvider + from systemone_harness.envs import McpEnvironment, OrderWorkflow + from systemone_harness.trace import Step, reasoning_text + + cwd = pathlib.Path(job.get("cwd") or ".") + model = str(job.get("model") or "") + sid = str(job.get("resume_session_id") or "") or "s1_" + uuid.uuid4().hex[:16] + resumed = bool(job.get("resume_session_id")) + emit({"type": "system", "subtype": "init", "session_id": sid, "model": model}) + + notes: list[str] = [] + servers = [e for e in (_server_entry(s) for s in (job.get("mcp_servers") or [])) if e] + env = None + try: + if servers: + if len(servers) > 1: + notes.append(f"{len(servers)} MCP servers are configured; the first is the environment, " + "the others are not used by this base.") + env = McpEnvironment(servers[0]) + cat = env.catalogue() + space = cat.space + for name, why in cat.unsupported.items(): + notes.append(f"Tool not offered, {name}: {why}.") + else: + env = OrderWorkflow("ship_cheapest") + space = OrderWorkflow.action_space() + notes.append("No MCP server is configured; the built-in order desk is the environment.") + + state_dir = cwd / ".harness" / "systemone" / sid + prior: list[Step] = [] + if resumed and (state_dir / "steps.json").exists(): + try: + prior = [Step(**{**_blank_step(), **d}) for d in json.loads((state_dir / "steps.json").read_text())] + except (ValueError, TypeError): + prior = [] + if resumed and isinstance(env, OrderWorkflow) and (state_dir / "environment.json").exists(): + try: + env.restore(json.loads((state_dir / "environment.json").read_text())) + except (ValueError, KeyError): + pass + + if provider is None: + provider = DecisionProvider(base_url=str(job["base_url"]).rstrip("/"), + path=_provider_path(str(job.get("provider") or "openrouter")), + api_key=str(job.get("api_key") or ""), model=model, + headers={"X-Title": "HarnessRouter"}) + first = {"done": False} + + def on_step(step) -> None: + n = step.index + if not first["done"] and notes: + emit({"type": "assistant", "message": {"content": [ + {"type": "thinking", "thinking": " ".join(notes)}]}}) + first["done"] = True + if step.verdict == "run": + emit({"type": "assistant", "message": {"content": [ + {"type": "tool_use", "id": f"call_{sid[-8:]}_{n}", "name": step.action, + "input": step.params or {}}]}}) + res = step.result or {} + emit({"type": "user", "message": {"content": [ + {"type": "tool_result", "tool_use_id": f"call_{sid[-8:]}_{n}", + "is_error": res.get("ok") is False, "content": str(res.get("text") or "")}]}}) + else: + emit({"type": "assistant", "message": {"content": [ + {"type": "thinking", "thinking": reasoning_text(step)}]}}) + + ctl = Controller(space, env, provider, max_steps=int(job.get("max_turns") or 100), + timeout_seconds=job.get("timeout_seconds") or None, + disabled=set(job.get("tools_disabled") or []), on_step=on_step) + run = ctl.run(str(job.get("prompt") or ""), reset=not resumed, + task_instructions=str(job.get("agent_doc") or ""), prior=prior) + if not first["done"] and notes: + emit({"type": "assistant", "message": {"content": [{"type": "thinking", "thinking": " ".join(notes)}]}}) + + state_dir.mkdir(parents=True, exist_ok=True) + kept = [{k: v for k, v in dataclasses.asdict(s).items() if k in _KEEP_ON_STEP} for s in prior + run.steps] + (state_dir / "steps.json").write_text(json.dumps(kept, default=str)) + if isinstance(env, OrderWorkflow): + (state_dir / "environment.json").write_text(json.dumps(env.snapshot())) + for art in run.artifacts: + name, sep, content = str(art).partition(": ") + if sep and name and "/" not in name and not name.startswith("."): + (cwd / name).write_text(content) + + text = run.summary() + emit({"type": "assistant", "message": {"content": [{"type": "text", "text": text}]}}) + if run.status == "completed": + subtype, is_error = "success", False + elif run.status == "incomplete" and run.reason in ("max_steps", "timeout"): + subtype, is_error = "error_max_turns", False + elif run.status == "incomplete": + subtype, is_error = "incomplete", False + else: + subtype, is_error = "error", True + result = {"type": "result", "subtype": subtype, "is_error": is_error, + "result": text if not is_error else (run.error or text), "reason": run.reason, + "session_id": sid, "model": run.served_model or model, + "usage": {"input_tokens": int(run.usage.get("input_tokens", 0)), + "output_tokens": int(run.usage.get("output_tokens", 0))} if run.steps else {}} + emit(result) + return result + finally: + if env is not None: + try: + env.close() + except Exception: # noqa: BLE001 - closing is best effort + pass + + +def _blank_step() -> dict: + return {"index": 0, "started_at": 0.0, "state": {}, "state_tokens": 0, "questions": {}, "answers": {}, + "action": "", "params": {}, "action_confidence": 0.0, "weakest": 0.0, "threshold": 0.0, + "verdict": "run", "result": None, "latency_ms": 0, "usage": {}, "served_model": "", + "request_id": "", "note": ""} + + +def main(argv: list[str] | None = None) -> int: + argv = sys.argv[1:] if argv is None else argv + if len(argv) != 1: + print("usage: systemone_driver.py ''", file=sys.stderr) + return 2 + job = json.loads(argv[0]) + try: + result = run_turn(job) + except Exception as exc: # noqa: BLE001 - a crash is a failed turn with its reason, never a silent exit + _emit({"type": "result", "subtype": "error", "is_error": True, + "result": f"{type(exc).__name__}: {exc}", "reason": "harness_error", "usage": {}}) + return 1 + return 0 if not result.get("is_error") else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/runner/tests/test_systemone_backend.py b/runner/tests/test_systemone_backend.py new file mode 100644 index 00000000..2827ef75 --- /dev/null +++ b/runner/tests/test_systemone_backend.py @@ -0,0 +1,187 @@ +"""The systemone backend: the System One Harness as a runner turn. + +What these pin, each measured before it was written down: + + the relay route's base is the provider's API ROOT for OpenRouter, because its decisions endpoint + lives at /api/alpha/decisions and a POST to /api/v1/alpha/decisions is a 404 (2026-09-19) + the driver's events are claude's stream-json: init, one tool_use + tool_result per executed + action, a thinking block for a step that executed nothing, a text, a result with usage + a loop that stops because no action cleared its confidence bar ends `incomplete` with the + reason, which the runner reports as its own status rather than a failure or a step cap + a resumed turn continues the order desk from where the last turn left it, from the workspace + disabling an action withholds it from the question the model answers +""" +import json +import pathlib +import sys +import tempfile + +import pytest + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) +import server # noqa: E402 +import systemone_driver as drv # noqa: E402 + +s1 = pytest.importorskip("systemone_harness", reason="the System One Harness is installed on the runner's venv, not here") +from systemone_harness import RecordedProvider # noqa: E402 + + +# ── the relay route ── +def test_openrouter_route_is_rooted_at_the_api_not_its_v1(): + base, tok = server._systemone_relay_route("openrouter", "https://openrouter.ai/api/v1", "sk-or-real") + assert base.startswith("http://127.0.0.1:") and base.endswith("/v1") and tok.startswith("hr-relay-") + upstream, key, _flags = server._HERMES_RELAY["routes"][tok] + assert upstream == "https://openrouter.ai/api" and key == "sk-or-real" + + +def test_typesafe_route_keeps_its_version_segment(): + base, tok = server._systemone_relay_route("typesafe", "https://api.typesafe.ai", "ts-real") + upstream, _key, _flags = server._HERMES_RELAY["routes"][tok] + assert upstream == "https://api.typesafe.ai/v1" + + +def test_the_builder_hands_the_driver_a_placeholder_never_the_key(): + env: dict = {} + cmd = server._build_systemone("openrouter", server.Auth(api_key="sk-or-real", base_url="https://openrouter.ai/api/v1"), + "typesafe/jev-1.13", "ship it", "/tmp/ws", env, + mcp_servers=[{"name": "desk", "url": "https://x/mcp"}, {"name": "bad"}], + tools_disabled=["ship"], max_turns=7, timeout_seconds=30, agent_doc="be quick") + assert cmd[0] == server.SYSTEMONE_PYTHON and cmd[1].endswith("systemone_driver.py") + job = json.loads(cmd[2]) + assert job["api_key"].startswith("hr-relay-") and "sk-or-real" not in cmd[2] + assert job["base_url"].endswith("/v1") and job["provider"] == "openrouter" + assert env["SYSTEMONE_API_KEY"] == job["api_key"] # the taps find the route by it + assert job["mcp_servers"] == [{"name": "desk", "url": "https://x/mcp"}] # an entry with no target is dropped + assert job["tools_disabled"] == ["ship"] and job["max_turns"] == 7 and job["timeout_seconds"] == 30 + assert job["agent_doc"] == "be quick" and job["model"] == "typesafe/jev-1.13" + + +def test_an_unknown_provider_and_a_missing_base_url_are_refused(): + with pytest.raises(Exception, match="unknown systemone provider"): + server._build_systemone("bedrock", server.Auth(api_key="k", base_url="https://x"), "m", "p", "/tmp", {}) + with pytest.raises(Exception, match="base_url"): + server._build_systemone("openrouter", server.Auth(api_key="k"), "m", "p", "/tmp", {}) + + +# ── the driver's events, on a scripted model ── +def _choice(questions, name, pick, p=0.95): + probs = {k: 0.0 for k in questions[name]["criteria"]} + probs[pick] = p + return {"type": "choice", "choice": pick, "probabilities": probs, "confidence": p} + + +def _fill(questions, answers): + for k, q in questions.items(): + if k in answers: + continue + if q["type"] == "choice": + answers[k] = _choice(questions, k, next(iter(q["criteria"])), 0.5) + elif q["type"] == "noul": + answers[k] = {"type": "noul", "noul": 0.5} + else: + answers[k] = {"type": "score", "score": 0, "probabilities": {}, "confidence": 0.5} + return answers + + +def _perfect(state, questions): + obs = state["observation"] + a = {"goal_reached": {"type": "noul", "noul": 0.02}} + if obs.get("unpicked"): + a["next_action"] = _choice(questions, "next_action", "pick_item") + a["pick_item__item"] = _choice(questions, "pick_item__item", obs["unpicked"][0]) + elif not obs.get("packed"): + a["next_action"] = _choice(questions, "next_action", "pack") + elif not obs.get("carrier"): + a["next_action"] = _choice(questions, "next_action", "choose_carrier") + a["choose_carrier__carrier"] = _choice(questions, "choose_carrier__carrier", "post") + else: + a["next_action"] = _choice(questions, "next_action", "ship") + return _fill(questions, a) + + +def _job(cwd, **over): + return {"cwd": cwd, "model": "typesafe/jev-1.13", "prompt": "Ship order A-104 with the cheapest carrier.", + "provider": "openrouter", "base_url": "http://127.0.0.1:1/v1", "api_key": "hr-relay-x", + "mcp_servers": [], "tools_disabled": [], "max_turns": 20, "timeout_seconds": None, "agent_doc": "", **over} + + +def test_a_turn_is_init_then_a_call_and_result_per_action_then_the_sentence_and_a_result(): + with tempfile.TemporaryDirectory() as cwd: + events = [] + result = drv.run_turn(_job(cwd), provider=RecordedProvider(_perfect), emit=events.append) + assert events[0]["type"] == "system" and events[0]["subtype"] == "init" and events[0]["session_id"].startswith("s1_") + thinking = [e for e in events if e["type"] == "assistant" and e["message"]["content"][0]["type"] == "thinking"] + assert "built-in order desk" in thinking[0]["message"]["content"][0]["thinking"] + calls = [e["message"]["content"][0] for e in events if e["type"] == "assistant" and e["message"]["content"][0]["type"] == "tool_use"] + outs = [e["message"]["content"][0] for e in events if e["type"] == "user"] + assert [c["name"] for c in calls] == ["pick_item", "pick_item", "pick_item", "pack", "choose_carrier", "ship"] + assert calls[0]["input"] == {"item": "blue mug"} and [c["id"] for c in calls] == [o["tool_use_id"] for o in outs] + assert outs[-1]["content"] == "Shipped by post." and outs[-1]["is_error"] is False + text = [e for e in events if e["type"] == "assistant" and e["message"]["content"][0]["type"] == "text"] + assert text[-1]["message"]["content"][0]["text"] == "The environment reached a terminal state after 6 actions." + assert events[-1] is result and result["subtype"] == "success" and result["is_error"] is False + assert result["reason"] == "environment_terminal" and result["usage"]["input_tokens"] > 0 + assert result["model"] == "recorded/jev" and result["session_id"] == events[0]["session_id"] + assert server._status_from_result(result, 0) == "done" + # the manifest is a file in the workspace, where the artifact cards read from + assert json.loads((pathlib.Path(cwd) / "manifest.json").read_text())["carrier"] == "post" + # and the session's state is under the workspace, for the next turn + sid = result["session_id"] + assert (pathlib.Path(cwd) / ".harness" / "systemone" / sid / "environment.json").exists() + + +def test_a_resumed_turn_continues_the_desk_from_where_it_stopped(): + with tempfile.TemporaryDirectory() as cwd: + first = [] + r1 = drv.run_turn(_job(cwd, max_turns=2), provider=RecordedProvider(_perfect), emit=first.append) + assert r1["subtype"] == "error_max_turns" and server._status_from_result(r1, 0) == "max_turns" + sid = r1["session_id"] + second = [] + seen_history = [] + prov = RecordedProvider(lambda s, q: seen_history.append(s.get("history")) or _perfect(s, q)) + r2 = drv.run_turn(_job(cwd, prompt="Carry on.", resume_session_id=sid), provider=prov, emit=second.append) + assert second[0]["session_id"] == sid and r2["subtype"] == "success" + names = [e["message"]["content"][0]["name"] for e in second + if e["type"] == "assistant" and e["message"]["content"][0]["type"] == "tool_use"] + assert names == ["pick_item", "pack", "choose_carrier", "ship"] # the two picks of turn one are not redone + assert seen_history[0] and seen_history[0][0].startswith("pick_item(item='blue mug')") + + +def test_a_refusal_streak_is_incomplete_with_its_reason_not_a_failure(): + shaky = lambda s, q: _fill(q, {"next_action": _choice(q, "next_action", "ship", 0.6)}) + with tempfile.TemporaryDirectory() as cwd: + events = [] + # picked, packed and carrier chosen already: only ship remains, and the model is 60% sure + from systemone_harness.envs import OrderWorkflow + desk = OrderWorkflow("ship_cheapest") + for item in ("blue mug", "tea towel", "kettle"): + desk.execute("pick_item", {"item": item}) + desk.execute("pack", {}) + desk.execute("choose_carrier", {"carrier": "post"}) + sid = "s1_resume01" + d = pathlib.Path(cwd) / ".harness" / "systemone" / sid + d.mkdir(parents=True) + (d / "environment.json").write_text(json.dumps(desk.snapshot())) + result = drv.run_turn(_job(cwd, resume_session_id=sid), provider=RecordedProvider(shaky), emit=events.append) + assert result["subtype"] == "incomplete" and result["is_error"] is False + assert result["reason"] == "no_confident_action" + assert server._status_from_result(result, 0) == "incomplete" + thinking = [e["message"]["content"][0]["thinking"] for e in events + if e["type"] == "assistant" and e["message"]["content"][0]["type"] == "thinking"] + assert any("Refused ship" in t and "0.60" in t for t in thinking) + assert not any(e["type"] == "user" for e in events) # nothing executed + + +def test_a_disabled_action_is_never_offered(): + seen = [] + prov = RecordedProvider(lambda s, q: seen.append(list(q["next_action"]["criteria"])) or _perfect(s, q)) + with tempfile.TemporaryDirectory() as cwd: + drv.run_turn(_job(cwd, tools_disabled=["ship"], max_turns=6), provider=prov, emit=lambda e: None) + assert all("ship" not in offered for offered in seen) and seen + + +def test_an_mcp_entry_carries_its_auth_as_a_header_and_a_targetless_one_is_dropped(): + e = drv._server_entry({"name": "x", "url": "https://h/mcp", "auth": "tok", "headers": {"A": "b"}, "transport": "sse"}) + assert e == {"url": "https://h/mcp", "headers": {"A": "b", "Authorization": "Bearer tok"}, "transport": "sse"} + assert drv._server_entry({"name": "plugin", "command": "./run.sh", "args": [1]})["args"] == ["1"] + assert drv._server_entry({"name": "nothing"}) is None diff --git a/ui/public/logos/systemone.png b/ui/public/logos/systemone.png new file mode 100644 index 00000000..d92aa006 Binary files /dev/null and b/ui/public/logos/systemone.png differ diff --git a/ui/src/components/HarnessLogo.tsx b/ui/src/components/HarnessLogo.tsx index 8aa10406..1c3257fb 100644 --- a/ui/src/components/HarnessLogo.tsx +++ b/ui/src/components/HarnessLogo.tsx @@ -19,6 +19,7 @@ const LOGO: Record = { kimi: '/logos/kimi.png', // the official Kimi mark (the K app icon from kimi.ai/code), supplied by Richard 2026-09-17 aider: '/logos/aider.png', // aider's own app icon (aider.chat/assets/icons/apple-touch-icon.png, from the Apache-2.0 repo's website assets) openhands: '/logos/openhands.png', // OpenHands' own app icon (public/apple-touch-icon.png in the MIT OpenHands/OpenHands repository) + systemone: '/logos/systemone.png', // the System One Harness's own mark (the project has no vendor logo; Jev is TypeSafe's model, not the harness) }; export function HarnessLogo({ id, size = 26 }: { id: string; size?: number }) { diff --git a/ui/src/components/HarnessSettings.tsx b/ui/src/components/HarnessSettings.tsx index db30a3af..ca751f65 100644 --- a/ui/src/components/HarnessSettings.tsx +++ b/ui/src/components/HarnessSettings.tsx @@ -194,7 +194,8 @@ export function HarnessSettings({ id, embedded = false, onNavigate }: { const ownSkills = skills.map((s, idx) => ({ s, idx })).filter(({ s }) => isOwnSkill(s)); // Built-ins the harness has not replaced with one of its own. A built-in is implicit: the // harness stores an entry only when its answer differs from the image's default. - const baseSkills = (srvBase?.builtinSkills || []).filter((b) => !ownSkills.some(({ s }) => s.name === b.name)); + const takesSkills = srvBase?.takesSkills !== false; + const baseSkills = (takesSkills ? (srvBase?.builtinSkills || []) : []).filter((b) => !ownSkills.some(({ s }) => s.name === b.name)); const disabledTools = new Set(draft?.disabledTools || []); const stats = statsFor(cards); @@ -273,7 +274,7 @@ export function HarnessSettings({ id, embedded = false, onNavigate }: {

Agent instructions

Persistent role, conventions, constraints, and output contract loaded on every Task.

-
+