From 88a0f2b4d412040cdf4da8654635843950b1c542 Mon Sep 17 00:00:00 2001 From: Huai-Ming Yu Date: Mon, 27 Jul 2026 23:07:23 +0800 Subject: [PATCH] feat: add ForgeCode agent support --- AGENTS.md | 36 +- adapters/__init__.py | 3 +- adapters/forgecode.py | 414 +++++++++++++++++++++ conf.toml | 7 + images/framework/Dockerfile | 22 ++ scripts/build_challenge_quantum_image.sh | 4 +- scripts/run_harbor_challenge.py | 99 ++++- scripts/smoke_docker.sh | 1 + scripts/tests/test_forgecode_adapter.py | 75 ++++ scripts/tests/test_run_harbor_challenge.py | 142 +++++++ 10 files changed, 787 insertions(+), 16 deletions(-) create mode 100644 adapters/forgecode.py create mode 100644 scripts/tests/test_forgecode_adapter.py create mode 100644 scripts/tests/test_run_harbor_challenge.py diff --git a/AGENTS.md b/AGENTS.md index dd1b8f9..bbe8e09 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,11 +36,11 @@ The default TensorCircuit tag is: challenge-benchmark-quantum-tensorcircuit:py311 ``` -Verify that the image contains Codex CLI and Claude Code: +Verify that the image contains Codex CLI, Claude Code, and ForgeCode: ```bash docker run --rm challenge-benchmark-quantum-tensorcircuit:py311 \ - sh -lc 'codex --version && claude --version' + sh -lc 'codex --version && claude --version && forge --version' ``` Generate framework prompt files and canonical tasks only when templates or upstream problem files change: @@ -115,7 +115,7 @@ AUDIT_MODEL_NAME=YOUR_AUDIT_MODEL_NAME `MODEL_NAME` is consumed by Harbor's `-m` flag for the solver agent. `AUDIT_MODEL_NAME` is passed through `--verifier-kwarg "audit_model=$AUDIT_MODEL_NAME"` and becomes `CODEX_AUDIT_MODEL` inside the verifier container. -## Run A Challenge With Codex Or Claude Code +## Run A Challenge With Codex, Claude Code, Or ForgeCode Use the wrapper so `tasks/challenge-*` stay fixed while the framework is selected from command-line arguments: @@ -190,6 +190,26 @@ Harbor runs through `adapters.claude_para:ClaudePara` already set these Claude-specific env vars, so this workaround is mainly for direct `docker run` checks and one-off image smoke tests. +To solve with ForgeCode while keeping the verifier Codex-based, select +`--solver-agent forgecode` and pass a `provider/model` name: + +```bash +python3 scripts/run_harbor_challenge.py \ + --challenge 01 \ + --framework tensorcircuit \ + --solver-agent forgecode \ + --model codex/gpt-5.6-sol \ + --solver-reasoning-effort high \ + --codex-force-auth-json +``` + +`adapters.forgecode:ForgeCode` uses ForgeCode's one-shot `forge -p` mode with +the file-modifying `forge` agent. The `codex` provider can temporarily convert +the file-backed Codex login at `~/.codex/auth.json`. Other providers can use +`--forgecode-credentials-path ~/.forge/.credentials.json` or their standard API +key environment variable. The adapter removes provider credentials before +Harbor collects artifacts. + If Harbor reports Docker is not running but `docker ps` works, this is usually Codex sandbox permission around Docker preflight/socket access. Re-run the same Harbor command with escalated Docker access. In sandboxed Codex sessions, treat Docker and Harbor access failures conservatively: @@ -255,7 +275,9 @@ Within a given framework, the agent and separate verifier use the same prebuilt This avoids rebuilding heavy quantum dependencies per run and avoids storing repeated environment files under each task. -Solver agents can use Harbor's built-in Codex, the local CodexPara adapter, or the local Claude adapter, while the verifier remains Codex-based: +Solver agents can use Harbor's built-in Codex, the local CodexPara adapter, the +local Claude adapter, or the local ForgeCode adapter, while the verifier remains +Codex-based: ```bash --agent-import-path harbor.agents.installed.codex:Codex @@ -263,6 +285,8 @@ Solver agents can use Harbor's built-in Codex, the local CodexPara adapter, or t --agent-import-path adapters.codex_para:CodexPara # or --agent-import-path adapters.claude_para:ClaudePara +# or +--agent-import-path adapters.forgecode:ForgeCode --verifier-import-path adapters.codex_para_verifier:CodexParaVerifier ``` @@ -275,6 +299,7 @@ python:3.11-slim nodejs/npm/ripgrep @openai/codex @anthropic-ai/claude-code +ForgeCode `forge` build-essential, pkg-config, rustc, cargo, procps ``` @@ -302,6 +327,7 @@ scripts/generate_framework_prompts.py prompts/frameworks/ adapters/codex_para.py adapters/claude_para.py +adapters/forgecode.py adapters/codex_para_verifier.py adapters/framework_docker.py scripts/inspect_harbor_job.py @@ -414,5 +440,7 @@ public solver agent: Harbor built-in `codex` from `conf.toml` local private override: `conf.local.toml` may set `codex-para`, `para`, and custom model names Codex reasoning effort: high Claude Code reasoning effort: max when `--solver-agent claude-code` unless overridden by `--solver-reasoning-effort` / `SOLVER_REASONING_EFFORT` +ForgeCode model: `codex/gpt-5.6-sol` from `[forgecode]` in `conf.toml` unless overridden +ForgeCode reasoning effort: high from `[forgecode]` unless overridden by `--solver-reasoning-effort` / `FORGECODE_REASONING_EFFORT` verifier audit model: `gpt-5` from `conf.toml` unless overridden ``` diff --git a/adapters/__init__.py b/adapters/__init__.py index e103993..76c06df 100644 --- a/adapters/__init__.py +++ b/adapters/__init__.py @@ -2,5 +2,6 @@ from .codex_para import CodexPara from .claude_para import ClaudePara +from .forgecode import ForgeCode -__all__ = ["ClaudePara", "CodexPara"] +__all__ = ["ClaudePara", "CodexPara", "ForgeCode"] diff --git a/adapters/forgecode.py b/adapters/forgecode.py new file mode 100644 index 0000000..20a7f66 --- /dev/null +++ b/adapters/forgecode.py @@ -0,0 +1,414 @@ +from __future__ import annotations + +import base64 +import json +import re +import shlex +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, override + +from harbor.agents.installed.base import ( + AgentAuthenticationError, + BaseInstalledAgent, + EnvVar, + ErrorPattern, + ModelNotFoundError, + with_prompt_template, +) +from harbor.environments.base import BaseEnvironment +from harbor.models.agent.context import AgentContext +from harbor.models.trial.paths import EnvironmentPaths + +_FORGECODE_AGENT_RE = re.compile(r"^[A-Za-z0-9_.-]+$") +_SOLUTION_RE = re.compile(r"\bsolution_(\d+)\.py\b") +_VERSION_RE = re.compile(r"(\d+\.\d+\.\d+)") + +_PROVIDER_ENV_VARS: dict[str, tuple[str, ...]] = { + "anthropic": ("ANTHROPIC_API_KEY", "ANTHROPIC_URL"), + "cerebras": ("CEREBRAS_API_KEY",), + "codex": ("CODEX_API_KEY",), + "deepseek": ("DEEPSEEK_API_KEY",), + "forge": ("FORGE_API_KEY",), + "github_copilot": ("GITHUB_COPILOT_API_KEY",), + "google_ai_studio": ("GOOGLE_AI_STUDIO_API_KEY", "GOOGLE_API_KEY"), + "groq": ("GROQ_API_KEY",), + "kimi_coding": ("KIMI_API_KEY",), + "minimax": ("MINIMAX_API_KEY",), + "moonshot": ("MOONSHOT_API_KEY",), + "open_router": ("OPENROUTER_API_KEY",), + "openai": ("OPENAI_API_KEY",), + "openai_compatible": ("OPENAI_API_KEY", "OPENAI_URL"), + "openai_responses_compatible": ("OPENAI_API_KEY", "OPENAI_URL"), + "requesty": ("REQUESTY_API_KEY",), + "xai": ("XAI_API_KEY",), + "zai": ("ZAI_API_KEY",), + "zai_coding": ("ZAI_CODING_API_KEY",), +} + +_CODEX_OAUTH_CONFIG = { + "auth_url": "https://auth.openai.com/oauth/authorize", + "token_url": "https://auth.openai.com/oauth/token", + "client_id": "app_EMoamEEZ73f0CkXaXp7hrann", + "scopes": ["openid", "profile", "email", "offline_access"], + "redirect_uri": "http://localhost:1455/auth/callback", + "use_pkce": True, + "custom_headers": {"originator": "forge"}, + "extra_auth_params": { + "id_token_add_organizations": "true", + "codex_cli_simplified_flow": "true", + }, +} + + +class ForgeCode(BaseInstalledAgent): + """Harbor adapter for ForgeCode's non-interactive one-shot CLI.""" + + _OUTPUT_FILENAME = "forgecode.txt" + _REMOTE_CONFIG_DIR = Path("/tmp/harbor-forgecode") + + ENV_VARS = [ + EnvVar( + "reasoning_effort", + env="FORGE_REASONING__EFFORT", + type="enum", + choices=["none", "minimal", "low", "medium", "high", "xhigh", "max"], + env_fallback="FORGECODE_REASONING_EFFORT", + ), + ] + ERROR_PATTERNS = [ + *BaseInstalledAgent.ERROR_PATTERNS, + ErrorPattern( + r"provider .+ (is not configured|requires credentials)|" + r"provider .+ is not available|no configured provider|" + r"authentication required", + AgentAuthenticationError, + ), + ErrorPattern( + r"model .+ (not found|is not available)|undefined model", + ModelNotFoundError, + ), + ] + + def __init__( + self, + *args: Any, + credentials_path: str | None = None, + codex_auth_json_path: str | None = None, + agent_id: str = "forge", + **kwargs: Any, + ) -> None: + super().__init__(*args, **kwargs) + + if not agent_id or not _FORGECODE_AGENT_RE.fullmatch(agent_id): + raise ValueError( + "ForgeCode agent_id must contain only letters, digits, " + "underscore, dot, or dash" + ) + self.agent_id = agent_id + self.credentials_path = ( + Path(credentials_path).expanduser() if credentials_path else None + ) + self.codex_auth_json_path = ( + Path(codex_auth_json_path).expanduser() if codex_auth_json_path else None + ) + + @staticmethod + @override + def name() -> str: + return "forgecode" + + @override + def get_version_command(self) -> str | None: + return "forge --version" + + @override + def parse_version(self, stdout: str) -> str: + match = _VERSION_RE.search(stdout) + return match.group(1) if match else stdout.strip() + + async def _installed_forge_satisfies_version( + self, environment: BaseEnvironment + ) -> bool: + result = await environment.exec(command="forge --version") + if result.return_code != 0: + return False + if self._version is None: + return True + return self.parse_version(result.stdout or "") == self._version.lstrip("v") + + @override + async def install(self, environment: BaseEnvironment) -> None: + if await self._installed_forge_satisfies_version(environment): + self.logger.debug("ForgeCode is already available") + return + + requested = self._version or "latest" + if not re.fullmatch(r"(?:latest|v?[0-9A-Za-z][0-9A-Za-z._-]*)", requested): + raise ValueError(f"Invalid ForgeCode version: {requested!r}") + tag = ( + requested + if requested == "latest" or requested.startswith("v") + else f"v{requested}" + ) + release_path = ( + "latest/download" if tag == "latest" else f"download/{shlex.quote(tag)}" + ) + + await self.exec_as_root( + environment, + command=( + "set -euo pipefail; " + "if command -v apt-get >/dev/null 2>&1; then " + "apt-get update && apt-get install -y --no-install-recommends " + "ca-certificates curl; " + "elif command -v apk >/dev/null 2>&1; then " + "apk add --no-cache ca-certificates curl; " + "elif command -v yum >/dev/null 2>&1; then " + "yum install -y ca-certificates curl; " + "fi; " + 'case "$(uname -m)" in ' + "x86_64|amd64) forge_arch=x86_64 ;; " + "aarch64|arm64) forge_arch=aarch64 ;; " + '*) echo "Unsupported ForgeCode architecture: ' + '$(uname -m)" >&2; exit 1 ;; ' + "esac; " + "curl -fsSL --retry 3 " + f'"https://github.com/tailcallhq/forgecode/releases/{release_path}/' + 'forge-${forge_arch}-unknown-linux-gnu" ' + "-o /usr/local/bin/forge; " + "chmod 0755 /usr/local/bin/forge; " + "forge --version" + ), + env={"DEBIAN_FRONTEND": "noninteractive"}, + ) + + def _split_model(self) -> tuple[str, str]: + if not self.model_name or "/" not in self.model_name: + raise ValueError( + "ForgeCode model must be in provider/model format, " + "for example codex/gpt-5.6-sol or anthropic/claude-sonnet-4-6" + ) + provider, model = self.model_name.split("/", 1) + provider = provider.strip().lower().replace("-", "_") + model = model.strip() + if not provider or not model: + raise ValueError("ForgeCode provider and model must both be non-empty") + return provider, model + + @staticmethod + def _jwt_claims(token: str) -> dict[str, Any]: + parts = token.split(".") + if len(parts) != 3: + return {} + try: + payload = parts[1] + "=" * (-len(parts[1]) % 4) + decoded = base64.urlsafe_b64decode(payload) + claims = json.loads(decoded) + except (ValueError, json.JSONDecodeError): + return {} + return claims if isinstance(claims, dict) else {} + + @classmethod + def _forge_credentials_from_codex_auth(cls, path: Path) -> list[dict[str, Any]]: + if not path.is_file(): + raise ValueError(f"Codex auth file was not found: {path}") + + data = json.loads(path.read_text(encoding="utf-8")) + tokens = data.get("tokens") + if not isinstance(tokens, dict): + raise ValueError(f"Codex auth file has no tokens object: {path}") + + access_token = tokens.get("access_token") + if not isinstance(access_token, str) or not access_token: + raise ValueError(f"Codex auth file has no access token: {path}") + + claims = cls._jwt_claims(access_token) + expires_at_raw = claims.get("exp") + if not isinstance(expires_at_raw, (int, float)): + raise ValueError("Codex access token has no usable expiry timestamp") + expires_at = ( + datetime.fromtimestamp(expires_at_raw, tz=timezone.utc) + .isoformat() + .replace("+00:00", "Z") + ) + + refresh_token = tokens.get("refresh_token") + if not isinstance(refresh_token, str) or not refresh_token: + refresh_token = None + + account_id = tokens.get("account_id") + if not isinstance(account_id, str) or not account_id: + nested = claims.get("https://api.openai.com/auth") + account_id = ( + nested.get("chatgpt_account_id") if isinstance(nested, dict) else None + ) + + credential: dict[str, Any] = { + "id": "codex", + "auth_details": { + "o_auth": { + "tokens": { + "access_token": access_token, + "refresh_token": refresh_token, + "expires_at": expires_at, + }, + "config": dict(_CODEX_OAUTH_CONFIG), + } + }, + "url_params": {}, + } + if isinstance(account_id, str) and account_id: + credential["url_params"]["chatgpt_account_id"] = account_id + return [credential] + + def _resolve_credentials_file( + self, provider: str + ) -> tuple[Path | None, Path | None]: + if self.credentials_path is not None: + path = self.credentials_path.resolve() + if not path.is_file(): + raise ValueError(f"ForgeCode credentials file was not found: {path}") + return path, None + + if provider != "codex": + return None, None + + path = self.codex_auth_json_path + if path is None: + raw_path = self._get_env("CODEX_AUTH_JSON_PATH") + path = Path(raw_path).expanduser() if raw_path else None + if path is None: + return None, None + + credentials = self._forge_credentials_from_codex_auth(path.resolve()) + handle = tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + prefix="harbor-forgecode-", + suffix=".json", + delete=False, + ) + temp_path = Path(handle.name) + try: + json.dump(credentials, handle) + handle.write("\n") + finally: + handle.close() + temp_path.chmod(0o600) + return temp_path, temp_path + + def _provider_environment(self, provider: str) -> dict[str, str]: + env: dict[str, str] = {} + for key in _PROVIDER_ENV_VARS.get(provider, ()): + value = self._get_env(key) + if value: + env[key] = value + return env + + def _expected_solution_path(self, instruction: str) -> str | None: + matches = _SOLUTION_RE.findall(instruction) + if not matches: + return None + return f"/root/solution_{matches[-1]}.py" + + @with_prompt_template + @override + async def run( + self, + instruction: str, + environment: BaseEnvironment, + context: AgentContext, + ) -> None: + provider, model = self._split_model() + remote_config = self._REMOTE_CONFIG_DIR.as_posix() + remote_credentials = (self._REMOTE_CONFIG_DIR / ".credentials.json").as_posix() + output_path = (EnvironmentPaths.agent_dir / self._OUTPUT_FILENAME).as_posix() + preserved_path = (EnvironmentPaths.agent_dir / "forgecode").as_posix() + expected_solution = self._expected_solution_path(instruction) + + env = self._provider_environment(provider) + env.update(self.resolve_env_vars()) + env.update( + { + "CI": "true", + "FORGE_AUTO_INSTALL_VSCODE_EXTENSION": "false", + "FORGE_CONFIG": remote_config, + "FORGE_RESTRICTED": "false", + "FORGE_SESSION__MODEL_ID": model, + "FORGE_SESSION__PROVIDER_ID": provider, + "FORGE_TOOL_SUPPORTED": "true", + "FORGE_UPDATES__AUTO_UPDATE": "false", + "NO_COLOR": "1", + "SHELL": "/bin/bash", + "TERM": "dumb", + } + ) + + local_credentials: Path | None = None + temp_credentials: Path | None = None + try: + local_credentials, temp_credentials = self._resolve_credentials_file( + provider + ) + await self.exec_as_agent( + environment, + command=( + f"mkdir -p {shlex.quote(remote_config)} " + f"{shlex.quote(EnvironmentPaths.agent_dir.as_posix())}" + ), + env=env, + ) + if local_credentials is not None: + await environment.upload_file(local_credentials, remote_credentials) + await self.exec_as_agent( + environment, + command=f"chmod 600 {shlex.quote(remote_credentials)}", + env=env, + ) + + if self.skills_dir: + await self.exec_as_agent( + environment, + command=( + f"mkdir -p {shlex.quote(remote_config)}/skills && " + f"cp -R {shlex.quote(self.skills_dir)}/* " + f"{shlex.quote(remote_config)}/skills/ 2>/dev/null || true" + ), + env=env, + ) + + command = ( + "forge " + f"--agent {shlex.quote(self.agent_id)} " + "-C /root " + f"-p {shlex.quote(instruction)} " + f"2>&1 &2; exit 1 ;; \ + esac; \ + if [ "${FORGECODE_VERSION}" = "latest" ]; then \ + forge_release_path="latest/download"; \ + else \ + case "${FORGECODE_VERSION}" in \ + v*) forge_tag="${FORGECODE_VERSION}" ;; \ + *) forge_tag="v${FORGECODE_VERSION}" ;; \ + esac; \ + forge_release_path="download/${forge_tag}"; \ + fi; \ + curl -fsSL --retry 3 \ + "https://github.com/tailcallhq/forgecode/releases/${forge_release_path}/forge-${forge_arch}-unknown-linux-gnu" \ + -o /usr/local/bin/forge; \ + chmod 0755 /usr/local/bin/forge; \ + forge --version + WORKDIR /root RUN mkdir -p /root/results /logs/verifier diff --git a/scripts/build_challenge_quantum_image.sh b/scripts/build_challenge_quantum_image.sh index 132df77..5b0aeca 100755 --- a/scripts/build_challenge_quantum_image.sh +++ b/scripts/build_challenge_quantum_image.sh @@ -11,6 +11,7 @@ FRAMEWORK_SLUG="$( IMAGE="${IMAGE:-challenge-benchmark-quantum-${FRAMEWORK_SLUG}:py311}" CODEX_VERSION="${CODEX_VERSION:-latest}" CLAUDE_CODE_VERSION="${CLAUDE_CODE_VERSION:-latest}" +FORGECODE_VERSION="${FORGECODE_VERSION:-latest}" DOCKERFILE="${DOCKERFILE:-${ROOT}/images/framework/Dockerfile}" BUILD_CONTEXT="${BUILD_CONTEXT:-${ROOT}}" REQUIREMENTS_FILE="${REQUIREMENTS_FILE:-${ROOT}/frameworks/${FRAMEWORK_SLUG}/requirements.txt}" @@ -38,10 +39,11 @@ docker build \ -f "${DOCKERFILE}" \ --build-arg "CODEX_VERSION=${CODEX_VERSION}" \ --build-arg "CLAUDE_CODE_VERSION=${CLAUDE_CODE_VERSION}" \ + --build-arg "FORGECODE_VERSION=${FORGECODE_VERSION}" \ --build-arg "FRAMEWORK_REQUIREMENTS=${FRAMEWORK_REQUIREMENTS}" \ -t "${IMAGE}" \ "${BUILD_CONTEXT}" docker image inspect "${IMAGE}" >/dev/null -docker run --rm "${IMAGE}" sh -lc 'codex --version && claude --version' +docker run --rm "${IMAGE}" sh -lc 'codex --version && claude --version && forge --version' echo "Built ${IMAGE}" diff --git a/scripts/run_harbor_challenge.py b/scripts/run_harbor_challenge.py index 1aa1f72..183f0f0 100644 --- a/scripts/run_harbor_challenge.py +++ b/scripts/run_harbor_challenge.py @@ -18,6 +18,7 @@ "codex": "harbor.agents.installed.codex:Codex", "codex-para": "adapters.codex_para:CodexPara", "claude-code": "adapters.claude_para:ClaudePara", + "forgecode": "adapters.forgecode:ForgeCode", } @@ -230,6 +231,17 @@ def parse_args() -> argparse.Namespace: default=None, help="Upload ~/.codex/auth.json instead of relying on OPENAI_API_KEY.", ) + parser.add_argument( + "--forgecode-credentials-path", + type=Path, + default=None, + help="Optional ForgeCode .credentials.json file.", + ) + parser.add_argument( + "--forgecode-agent", + default=None, + help="ForgeCode agent ID. Defaults to the file-modifying 'forge' agent.", + ) parser.add_argument( "--yes", action=argparse.BooleanOptionalAction, @@ -248,6 +260,11 @@ def challenge_name(raw: str) -> str: def resolve_solver_model(solver_agent: str, explicit_model: str | None) -> str: if explicit_model: + if solver_agent == "forgecode" and "/" not in explicit_model: + raise ValueError( + "ForgeCode model must be in provider/model format, " + "for example codex/gpt-5.6-sol" + ) return explicit_model if solver_agent == "claude-code": model = os.environ.get("ANTHROPIC_MODEL") or os.environ.get( @@ -259,6 +276,11 @@ def resolve_solver_model(solver_agent: str, explicit_model: str | None) -> str: "MODEL_NAME, MODEL, ANTHROPIC_MODEL, or ANTHROPIC_DEFAULT_OPUS_MODEL." ) return model + if solver_agent == "forgecode": + raise ValueError( + "ForgeCode solver model is required in provider/model format. " + "Pass --model, export FORGECODE_MODEL, or set [forgecode].model." + ) return "gpt-5" @@ -318,11 +340,18 @@ def main() -> int: f"{', '.join(sorted(SOLVER_AGENT_IMPORTS))}" ) - configured_model = first_value( - args.model, - env_value("MODEL_NAME", "MODEL"), - config_get(config, "codex", "model"), - ) + if solver_agent == "forgecode": + configured_model = first_value( + args.model, + env_value("FORGECODE_MODEL", "MODEL_NAME", "MODEL"), + config_get(config, "forgecode", "model"), + ) + else: + configured_model = first_value( + args.model, + env_value("MODEL_NAME", "MODEL"), + config_get(config, "codex", "model"), + ) solver_model = resolve_solver_model( solver_agent, str(configured_model) if configured_model is not None else None ) @@ -342,11 +371,18 @@ def main() -> int: config_get(config, "codex", "audit_model"), solver_model, ) - solver_reasoning_effort = first_value( - args.solver_reasoning_effort, - env_value("SOLVER_REASONING_EFFORT", "CLAUDE_CODE_EFFORT_LEVEL"), - config_get(config, "run", "solver_reasoning_effort"), - ) + if solver_agent == "forgecode": + solver_reasoning_effort = first_value( + args.solver_reasoning_effort, + env_value("FORGECODE_REASONING_EFFORT", "SOLVER_REASONING_EFFORT"), + config_get(config, "forgecode", "reasoning_effort"), + ) + else: + solver_reasoning_effort = first_value( + args.solver_reasoning_effort, + env_value("SOLVER_REASONING_EFFORT", "CLAUDE_CODE_EFFORT_LEVEL"), + config_get(config, "run", "solver_reasoning_effort"), + ) if solver_agent == "claude-code" and not solver_reasoning_effort: solver_reasoning_effort = "max" @@ -415,6 +451,24 @@ def main() -> int: ) ) env["CODEX_FORCE_AUTH_JSON"] = "true" if codex_force_auth_json else "false" + forgecode_credentials_path = resolve_path( + first_value( + args.forgecode_credentials_path, + env_value("FORGECODE_CREDENTIALS_PATH"), + config_get(config, "forgecode", "credentials_path"), + ) + ) + forgecode_agent = str( + first_value( + args.forgecode_agent, + env_value("FORGECODE_AGENT"), + config_get(config, "forgecode", "agent"), + "forge", + ) + ).strip() + forgecode_codex_auth_path = resolve_path(env_value("CODEX_AUTH_JSON_PATH")) + if codex_force_auth_json and forgecode_codex_auth_path is None: + forgecode_codex_auth_path = Path.home() / ".codex" / "auth.json" cmd = [ str(harbor_bin), @@ -464,6 +518,21 @@ def main() -> int: "model_catalog_path", codex_model_catalog_path, ) + elif solver_agent == "forgecode": + add_kwarg( + cmd, + "--agent-kwarg", + "credentials_path", + forgecode_credentials_path, + ) + if codex_force_auth_json: + add_kwarg( + cmd, + "--agent-kwarg", + "codex_auth_json_path", + forgecode_codex_auth_path, + ) + add_kwarg(cmd, "--agent-kwarg", "agent_id", forgecode_agent) add_kwarg(cmd, "--verifier-kwarg", "profile", codex_profile) add_kwarg( @@ -492,6 +561,16 @@ def main() -> int: print(f"Solver reasoning effort: {solver_reasoning_effort or '(agent default)'}") print(f"Audit model: {audit_model}") print(f"Codex profile: {codex_profile or '(default)'}") + if solver_agent == "forgecode": + credential_source = ( + display_path(forgecode_credentials_path) + if forgecode_credentials_path + else "Codex auth.json" + if codex_force_auth_json + else "provider environment" + ) + print(f"ForgeCode agent: {forgecode_agent}") + print(f"ForgeCode credentials: {credential_source}") sys.stdout.flush() return subprocess.run(cmd, env=env, check=False).returncode diff --git a/scripts/smoke_docker.sh b/scripts/smoke_docker.sh index f80b803..44dfd57 100755 --- a/scripts/smoke_docker.sh +++ b/scripts/smoke_docker.sh @@ -20,6 +20,7 @@ docker run --rm \ bash -lc ' set -euo pipefail codex --version + forge --version python - <<'"'"'PY'"'"' import importlib import os diff --git a/scripts/tests/test_forgecode_adapter.py b/scripts/tests/test_forgecode_adapter.py new file mode 100644 index 0000000..3c76e27 --- /dev/null +++ b/scripts/tests/test_forgecode_adapter.py @@ -0,0 +1,75 @@ +import base64 +import json +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +from adapters.forgecode import ForgeCode + + +def jwt_with_claims(claims: dict) -> str: + payload = base64.urlsafe_b64encode(json.dumps(claims).encode()).decode() + return f"header.{payload.rstrip('=')}.signature" + + +def test_codex_auth_is_converted_to_forgecode_credentials(tmp_path: Path): + expires_at = 1_900_000_000 + access_token = jwt_with_claims( + { + "exp": expires_at, + "https://api.openai.com/auth": {"chatgpt_account_id": "account-from-claim"}, + } + ) + auth_json = tmp_path / "auth.json" + auth_json.write_text( + json.dumps( + { + "auth_mode": "chatgpt", + "tokens": { + "access_token": access_token, + "refresh_token": "refresh-token", + }, + } + ), + encoding="utf-8", + ) + + credentials = ForgeCode._forge_credentials_from_codex_auth(auth_json) + + assert len(credentials) == 1 + credential = credentials[0] + assert credential["id"] == "codex" + assert credential["url_params"]["chatgpt_account_id"] == "account-from-claim" + oauth = credential["auth_details"]["o_auth"] + assert oauth["tokens"]["access_token"] == access_token + assert oauth["tokens"]["refresh_token"] == "refresh-token" + assert oauth["tokens"]["expires_at"] == datetime.fromtimestamp( + expires_at, timezone.utc + ).isoformat().replace("+00:00", "Z") + assert oauth["config"]["client_id"].startswith("app_") + + +def test_forgecode_requires_provider_model(): + adapter = ForgeCode(logs_dir=Path("/tmp/logs"), model_name="gpt-5.6-sol") + + with pytest.raises(ValueError, match="provider/model"): + adapter._split_model() + + +def test_forgecode_normalizes_provider_slug(): + adapter = ForgeCode( + logs_dir=Path("/tmp/logs"), + model_name="open-router/model-name", + ) + + assert adapter._split_model() == ("open_router", "model-name") + + +def test_forgecode_rejects_unsafe_agent_id(): + with pytest.raises(ValueError, match="agent_id"): + ForgeCode( + logs_dir=Path("/tmp/logs"), + model_name="codex/gpt-5.6-sol", + agent_id="forge; touch /tmp/unsafe", + ) diff --git a/scripts/tests/test_run_harbor_challenge.py b/scripts/tests/test_run_harbor_challenge.py new file mode 100644 index 0000000..c1031cd --- /dev/null +++ b/scripts/tests/test_run_harbor_challenge.py @@ -0,0 +1,142 @@ +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from scripts import run_harbor_challenge as runner + + +RUNNER_ENV_NAMES = ( + "AUDIT_MODEL", + "AUDIT_MODEL_NAME", + "CHALLENGE_ID", + "CODEX_AUTH_JSON_PATH", + "CODEX_FORCE_AUTH_JSON", + "CODEX_MODEL_CATALOG_PATH", + "CODEX_PROFILE", + "CODEX_PROFILE_CONFIG_PATH", + "FORGECODE_AGENT", + "FORGECODE_CREDENTIALS_PATH", + "FORGECODE_MODEL", + "FORGECODE_REASONING_EFFORT", + "FRAMEWORK", + "HARBOR_BIN", + "HARBOR_YES", + "JOB_NAME", + "MODEL", + "MODEL_NAME", + "SOLVER_AGENT", + "SOLVER_REASONING_EFFORT", +) + + +@pytest.fixture +def clean_runner_env(monkeypatch: pytest.MonkeyPatch): + for name in RUNNER_ENV_NAMES: + monkeypatch.delenv(name, raising=False) + + +def run_and_capture( + monkeypatch: pytest.MonkeyPatch, + *extra_args: str, +) -> tuple[list[str], dict[str, str]]: + captured: dict[str, object] = {} + + def fake_run(cmd, *, env, check): + captured["cmd"] = cmd + captured["env"] = env + captured["check"] = check + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(runner.subprocess, "run", fake_run) + monkeypatch.setattr( + runner.sys, + "argv", + [ + "run_harbor_challenge.py", + "--challenge", + "01", + "--framework", + "tensorcircuit", + "--harbor-bin", + "harbor", + "--no-yes", + *extra_args, + ], + ) + + assert runner.main() == 0 + assert captured["check"] is False + return captured["cmd"], captured["env"] + + +def test_forgecode_uses_provider_model_and_codex_login( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + clean_runner_env, +): + auth_json = tmp_path / "auth.json" + auth_json.write_text("{}", encoding="utf-8") + monkeypatch.setenv("CODEX_AUTH_JSON_PATH", str(auth_json)) + + cmd, _ = run_and_capture( + monkeypatch, + "--solver-agent", + "forgecode", + "--model", + "codex/gpt-5.6-sol", + "--solver-reasoning-effort", + "xhigh", + "--codex-force-auth-json", + ) + + assert "adapters.forgecode:ForgeCode" in cmd + assert "reasoning_effort=xhigh" in cmd + assert f"codex_auth_json_path={auth_json}" in cmd + assert "agent_id=forge" in cmd + + +def test_forgecode_accepts_its_credentials_file( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + clean_runner_env, +): + credentials = tmp_path / ".credentials.json" + credentials.write_text("[]", encoding="utf-8") + + cmd, _ = run_and_capture( + monkeypatch, + "--solver-agent", + "forgecode", + "--model", + "anthropic/claude-sonnet-4-6", + "--forgecode-credentials-path", + str(credentials), + "--forgecode-agent", + "forge", + ) + + assert f"credentials_path={credentials}" in cmd + assert "agent_id=forge" in cmd + + +def test_forgecode_rejects_model_without_provider( + monkeypatch: pytest.MonkeyPatch, + clean_runner_env, +): + monkeypatch.setattr( + runner.sys, + "argv", + [ + "run_harbor_challenge.py", + "--challenge", + "01", + "--solver-agent", + "forgecode", + "--model", + "gpt-5.6-sol", + ], + ) + + with pytest.raises(ValueError, match="provider/model"): + runner.main()