diff --git a/benchmark/edgebench/README.md b/benchmark/edgebench/README.md index 55caac8dea..01179b263e 100644 --- a/benchmark/edgebench/README.md +++ b/benchmark/edgebench/README.md @@ -97,6 +97,33 @@ proxy when the pinned source installer needs to build frontend assets. It is passed only to setup subprocesses and cleared before solver execution and native network isolation. It is never placed in the container's persistent environment. +### Retry transient API rate limits + +For API-key trials, an explicit `--provider-rate-limit-retry-seconds 900` +lets the SForge host credential gateway retry HTTP 429 before forwarding any +response bytes. Both `native-goal` and `official` workers use this transport. +This requires a SForge revision with `agent_rate_limit_retry_seconds`; an older +source or an OAuth-only route fails before launch when the option is requested. +The default is zero, preserving historical pass-through behavior. + +The gateway honors `Retry-After` seconds and HTTP dates. Without a usable header, +it backs off from about 5 seconds to at most 60 seconds, with jitter, within the +configured window. A provider delay beyond the remaining window returns the +original 429 instead of retrying early. Cancellation or trial cleanup stops +further retries. Other status codes and already-started streams are passed +through without gateway replay. Upstream requests already in flight retain the +existing request timeout. + +This is distinct from Codex `stream_max_retries`: a directly returned HTTP 429 +can terminate a native Goal without entering the stream retry loop. Increasing +that count alone does not qualify HTTP 429 recovery. The gateway preserves the +request, Codex session, Goal, workspace and original trial deadline; retry wait +consumes that deadline. It does not continue a completed Goal or grant more time. +The run receipt records the configured retry window and source revision; proxy +logs record retry counts and delays without prompts or credentials. Apply the +same option to comparison arms and keep older attempts under their original +settings. Sustained limits beyond the window can still fail the trial. + ### Continue after native account exhaustion Keep a quota interruption distinct from a task result. Codex's message-only diff --git a/benchmark/edgebench/run.py b/benchmark/edgebench/run.py index 92c0803873..2da4d2e56e 100644 --- a/benchmark/edgebench/run.py +++ b/benchmark/edgebench/run.py @@ -115,6 +115,19 @@ def _task_default(task_id: str, key: str, explicit: int | None, fallback: int) - return value +def _configure_provider_retries(config, seconds): + if type(seconds) is not int or seconds < 0: + raise ValueError("--provider-rate-limit-retry-seconds must be nonnegative") + if seconds and (not config.agent_api_base_url or not config.agent_api_key): + raise ValueError("--provider-rate-limit-retry-seconds requires an API key and base URL") + if not hasattr(config, "agent_rate_limit_retry_seconds"): + if seconds: + raise ValueError("the selected SForge source does not support HTTP 429 retry configuration") + return config + # Explicit runner settings override ambient SForge defaults, including zero. + return replace(config, agent_rate_limit_retry_seconds=seconds) + + def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--task", required=True) @@ -133,6 +146,9 @@ def main(argv=None): "continuation Goals until the budget edge (native-goal worker only)") parser.add_argument("--model", required=True) parser.add_argument("--effort", choices=REASONING_EFFORTS, required=True) + parser.add_argument("--provider-rate-limit-retry-seconds", type=int, default=0, + help="Host API gateway HTTP 429 retry window for new trials; " + "0 keeps pass-through behavior, e.g. 900 tolerates transient limits") parser.add_argument("--timeout", type=int, help="Total trial seconds; task-defaults.json overrides the 18h fallback") parser.add_argument("--task-entry", choices=TASK_ENTRIES, @@ -301,6 +317,10 @@ def main(argv=None): judge_cpu_limit=None if args.restore_checkpoint else 4, judge_mem_limit=None if args.restore_checkpoint else "8g", ) + try: + config = _configure_provider_retries(config, args.provider_rate_limit_retry_seconds) + except ValueError as exc: + parser.error(str(exc)) if args.api_proxy_url: config.agent_extra_env = { "HTTPS_PROXY": args.api_proxy_url, "HTTP_PROXY": args.api_proxy_url, @@ -400,6 +420,7 @@ def main(argv=None): **({"planning_timeout_seconds": None} if agent.task_entry == "loopx-planned" else {}), "model": args.model, "effort": args.effort, "timeout_seconds": args.timeout, + "provider_rate_limit_retry_seconds": args.provider_rate_limit_retry_seconds, "loopx_commit": pins[0], "runner_commit": pins[1], "sforge_source": active_source_receipt(), **turn_envelope_receipt(explicit=args.turn_envelope), diff --git a/benchmark/tests/test_provider_retry_settings.py b/benchmark/tests/test_provider_retry_settings.py new file mode 100644 index 0000000000..f5c90b1a89 --- /dev/null +++ b/benchmark/tests/test_provider_retry_settings.py @@ -0,0 +1,56 @@ +"""Runner retry settings must bind the actual host gateway or fail closed.""" +from dataclasses import dataclass + +import pytest + +pytest.importorskip('sforge') +pytest.importorskip('harbor') + +from benchmark.edgebench.run import _configure_provider_retries +from sforge.harness.config import SForgeConfig +from sforge.harness.run_agent import _prepare_credential_proxy +from sforge.harness.agent.codex import CodexAgent + + +def test_runner_setting_reaches_real_gateway(): + if not hasattr(SForgeConfig, 'agent_rate_limit_retry_seconds'): + pytest.skip('real gateway qualification requires the companion SForge retry revision') + config = SForgeConfig(agent_api_base_url='http://127.0.0.1:12345', agent_api_key='synthetic') + configured = _configure_provider_retries(config, 900) + backend = type('Backend', (), {'backend_name': 'docker'})() + proxy, route = _prepare_credential_proxy(CodexAgent(configured), configured, backend) + try: + assert proxy.rate_limit_retry_seconds == 900 + assert route == f'http://host.docker.internal:{proxy.port}' + assert config.agent_rate_limit_retry_seconds == 0 + finally: + proxy.stop() + + +def test_zero_overrides_ambient_retry_setting(): + @dataclass + class Config(SForgeConfig): + agent_rate_limit_retry_seconds: int = 900 + assert _configure_provider_retries(Config(), 0).agent_rate_limit_retry_seconds == 0 + + +@pytest.mark.parametrize('seconds', [-1, True, 1.5, '900']) +def test_invalid_setting_is_rejected(seconds): + with pytest.raises(ValueError, match='nonnegative'): + _configure_provider_retries(SForgeConfig(), seconds) + + +def test_oauth_does_not_silently_accept_inapplicable_setting(): + with pytest.raises(ValueError, match='API key'): + _configure_provider_retries(SForgeConfig(), 900) + + +def test_old_sforge_keeps_default_but_rejects_unavailable_retry(): + @dataclass + class Legacy: + agent_api_base_url: str = 'http://127.0.0.1:12345' + agent_api_key: str = 'synthetic' + old = Legacy() + assert _configure_provider_retries(old, 0) is old + with pytest.raises(ValueError, match='does not support'): + _configure_provider_retries(old, 900)