From 3398874f7330bf37c46dbf2cf193e06cc32e6a20 Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 07:07:47 +1000
Subject: [PATCH 01/34] Avoid speculative type requirements in reconstruction
prompts
---
src/re_agent/agents/prompts/checker_system.md | 2 +-
src/re_agent/agents/prompts/reverser_task.md | 4 ++--
tests/test_agents/test_prompt_contract.py | 18 ++++++++++++++++++
3 files changed, 21 insertions(+), 3 deletions(-)
create mode 100644 tests/test_agents/test_prompt_contract.py
diff --git a/src/re_agent/agents/prompts/checker_system.md b/src/re_agent/agents/prompts/checker_system.md
index 8936d453..3287894c 100644
--- a/src/re_agent/agents/prompts/checker_system.md
+++ b/src/re_agent/agents/prompts/checker_system.md
@@ -2,7 +2,7 @@ You are a reverse engineering quality checker. Your job is to verify that revers
Verification standards:
- Every line of Ghidra logic must have corresponding source code
-- Every struct offset must map to a named member
+- Use supplied type evidence to verify named members. When no layout is available, accept explicit byte offsets that preserve access widths and semantics; do not fail code solely for lacking member names or demand invented structs.
- Every function call must be identified and matched
- Expression order must match exactly (floating point is order-sensitive)
- No missing branches, conditions, or edge cases
diff --git a/src/re_agent/agents/prompts/reverser_task.md b/src/re_agent/agents/prompts/reverser_task.md
index 891fcb86..c0e714d1 100644
--- a/src/re_agent/agents/prompts/reverser_task.md
+++ b/src/re_agent/agents/prompts/reverser_task.md
@@ -24,8 +24,8 @@ ${project_rules}
Requirements:
1. Match every branch and call from the decompile
-2. Map all offsets (e.g. param_1 + 0x88) to real member names
+2. Use member names only when supported by supplied type evidence. Otherwise preserve the exact byte offset and access width, and document the unresolved layout; do not invent structs or member names.
3. Preserve exact expression/operand order
4. Use existing project patterns and naming conventions
-5. Output the complete function implementation in a ```cpp block
+5. Output exactly one complete function implementation in a ```cpp block. Do not wrap it in namespace/class definitions or add helper type/function definitions; report missing declarations outside the code block.
6. End with: REVERSED_FUNCTION: ${class_name}::${function_name} (${address})
diff --git a/tests/test_agents/test_prompt_contract.py b/tests/test_agents/test_prompt_contract.py
new file mode 100644
index 00000000..03c06ed3
--- /dev/null
+++ b/tests/test_agents/test_prompt_contract.py
@@ -0,0 +1,18 @@
+"""Keep reconstruction/review instructions consistent with missing type evidence."""
+from re_agent.agents.checker import PROMPTS_DIR
+from re_agent.utils.templates import render_template
+
+
+def test_missing_layout_does_not_require_invented_members():
+ reverser = render_template(PROMPTS_DIR / "reverser_task.md")
+ checker = render_template(PROMPTS_DIR / "checker_system.md")
+ assert "preserve the exact byte offset and access width" in reverser
+ assert "do not invent structs or member names" in reverser
+ assert "accept explicit byte offsets" in checker
+ assert "do not fail code solely for lacking member names" in checker
+
+
+def test_requested_candidate_matches_single_body_overlay_contract():
+ prompt = render_template(PROMPTS_DIR / "reverser_task.md")
+ assert "exactly one complete function implementation" in prompt
+ assert "report missing declarations outside the code block" in prompt
From 13abe167613542eaeb913b4051a55e578101face Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 07:07:47 +1000
Subject: [PATCH 02/34] Skip model reviews for candidates that fail local
preflight
---
src/re_agent/agents/loop.py | 23 +++++++++++++++--
tests/test_agents/test_preflight.py | 39 +++++++++++++++++++++++++++++
2 files changed, 60 insertions(+), 2 deletions(-)
create mode 100644 tests/test_agents/test_preflight.py
diff --git a/src/re_agent/agents/loop.py b/src/re_agent/agents/loop.py
index 36a028b0..fc0d4487 100644
--- a/src/re_agent/agents/loop.py
+++ b/src/re_agent/agents/loop.py
@@ -25,6 +25,7 @@
from re_agent.llm.observed import CallBudget, ObservedProvider
from re_agent.llm.protocol import LLMProvider
from re_agent.parity.source_indexer import SourceIndexer
+from re_agent.verification.candidate import extract_candidate_body
from re_agent.verification.objective import verify_candidate
@@ -128,8 +129,25 @@ def run_fix_loop(
log_path = log_dir / f"round{round_num}-{timestamp}-reverser.json"
log_path.write_text(json.dumps(log_entry, indent=2), encoding="utf-8")
- # Check
- verdict = checker.check(code, target)
+ # Avoid spending a reviewer call on output the configured candidate gate
+ # cannot consume. Keep the same repair/checkpoint path as other failures.
+ preflight_error = None
+ if candidate_gate is not None:
+ try:
+ extract_candidate_body(code)
+ except ValueError as exc:
+ preflight_error = str(exc)
+ if preflight_error:
+ checker.last_prompt = ""
+ checker.last_response = ""
+ verdict = CheckerVerdict(
+ verdict=Verdict.FAIL,
+ summary="Local candidate preflight failed; model review skipped",
+ issues=[preflight_error],
+ fix_instructions=["Return exactly one complete function definition without helper definitions."],
+ )
+ else:
+ verdict = checker.check(code, target)
last_verdict = verdict
objective_verdict: ObjectiveVerdict | None = None
@@ -148,6 +166,7 @@ def run_fix_loop(
"round": round_num,
"timestamp": timestamp,
"phase": "check",
+ "checker_skipped": preflight_error is not None,
"prompt": checker.last_prompt,
"response": checker.last_response,
"verdict": verdict.verdict.value,
diff --git a/tests/test_agents/test_preflight.py b/tests/test_agents/test_preflight.py
new file mode 100644
index 00000000..5dd0347d
--- /dev/null
+++ b/tests/test_agents/test_preflight.py
@@ -0,0 +1,39 @@
+"""Reject unusable candidate shapes before spending reviewer calls."""
+import json
+
+import pytest
+
+from re_agent.agents.loop import run_fix_loop
+from re_agent.backend.stub import StubBackend
+from re_agent.core.models import FunctionTarget
+from tests.test_agents.test_loop import MockLLM
+
+
+@pytest.mark.parametrize("invalid", [
+ "struct Layout { int field; }; int f(Layout* p) { return p->field; }",
+ "int f() { return 0;",
+ "int f() { return 0; } int g() { return 1; }",
+])
+def test_invalid_candidate_skips_reviewer_then_repairs_within_budget(tmp_path, invalid):
+ reverser = MockLLM([f"```cpp\n{invalid}\n```", "```cpp\nint f() { return 0; }\n```"])
+ checker = MockLLM(['{"verdict":"PASS","summary":"Matches","issues":[],"fix_instructions":[]}'])
+ result = run_fix_loop(FunctionTarget("100", "", "f"), StubBackend(), reverser, checker,
+ candidate_gate=lambda result: result, objective_verifier_enabled=False,
+ max_rounds=2, max_llm_calls=3, log_dir=tmp_path)
+ assert result.success
+ assert result.rounds_used == 2
+ assert reverser._idx == 2
+ assert checker._idx == 1
+ log = next(tmp_path.glob("*/round1-*-checker.json"))
+ assert json.loads(log.read_text())["checker_skipped"] is True
+
+
+def test_repeated_invalid_candidates_never_call_reviewer(tmp_path):
+ reverser = MockLLM(["```cpp\nint f() {\n```"])
+ checker = MockLLM(["VERDICT: PASS"])
+ result = run_fix_loop(FunctionTarget("100", "", "f"), StubBackend(), reverser, checker,
+ candidate_gate=lambda result: result, objective_verifier_enabled=False,
+ max_rounds=3, log_dir=tmp_path)
+ assert not result.success
+ assert checker._idx == 0
+ assert result.error and "without progress" in result.error
From 540de3e9547fcc0db39e337dc8ddfa97be27382f Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 07:32:16 +1000
Subject: [PATCH 03/34] Add decoupled live progress monitor with tested worker
controls
---
README.md | 2 +
docs/live-monitor.md | 44 ++++
pyproject.toml | 5 +-
src/re_agent/cli/cmd_monitor.py | 22 ++
src/re_agent/cli/main.py | 14 ++
src/re_agent/monitor/__init__.py | 1 +
src/re_agent/monitor/index.html | 393 +++++++++++++++++++++++++++++++
src/re_agent/monitor/server.py | 220 +++++++++++++++++
tests/test_monitor.py | 186 +++++++++++++++
9 files changed, 885 insertions(+), 2 deletions(-)
create mode 100644 docs/live-monitor.md
create mode 100644 src/re_agent/cli/cmd_monitor.py
create mode 100644 src/re_agent/monitor/__init__.py
create mode 100644 src/re_agent/monitor/index.html
create mode 100644 src/re_agent/monitor/server.py
create mode 100644 tests/test_monitor.py
diff --git a/README.md b/README.md
index e30e268d..a320bf63 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,7 @@
# ReAgent
+For a decoupled local progress window with optional worker controls, see the [live monitor guide](docs/live-monitor.md).
+
[](https://pypi.org/project/auto-re-agent/)
[](https://pypi.org/project/auto-re-agent/)
[](https://github.com/Dryxio/reagent/actions/workflows/ci.yml)
diff --git a/docs/live-monitor.md b/docs/live-monitor.md
new file mode 100644
index 00000000..75ca2cf5
--- /dev/null
+++ b/docs/live-monitor.md
@@ -0,0 +1,44 @@
+# Live progress monitor
+
+`re-agent monitor` serves a local dashboard independently of the reversal worker. It reads existing session JSON files and logs; it does not import model providers or start work automatically.
+
+```sh
+re-agent monitor --work-dir /path/to/project --session-glob re-agent-progress.json --total 500
+```
+
+Open the printed `http://127.0.0.1:8765` address. The page refreshes every two seconds and shows saved function results, review passes, failures, rounds, and recent log output. Omit `--total` when the planned count is unknown. A review pass is not a proof of compilation or binary equivalence.
+
+Session patterns are relative to `--work-dir` and can be repeated. For a batch runner:
+
+```sh
+re-agent monitor --work-dir /path/to/project --session-glob 'reports/batch-*/progress.json' --log-glob 'reports/batch-*/stderr.log' --total 500
+```
+
+The monitor deduplicates function addresses across matching files using their latest result timestamps. Select sessions for the same binary/version; unrelated projects can reuse addresses. Unchanged files are cached, partial JSON writes retain the last readable snapshot, and log output is bounded to the last 7,000 bytes. Counters describe saved final results, not in-flight rounds.
+
+## Optional worker controls
+
+Install the optional process-management dependency:
+
+```sh
+pip install 'auto-re-agent[monitor]'
+```
+
+Supply an explicit argument array after `--worker`, which must be the last monitor option:
+
+```sh
+re-agent monitor --work-dir /path/to/project --session-glob re-agent-progress.json --total 500 --worker re-agent --config re-agent.yaml reverse --manifest targets.json --max-functions 500
+```
+
+On Windows, quote paths containing spaces and use an executable such as `python.exe` or `re-agent.exe`, rather than a shell script. Worker arguments are passed directly, without shell interpretation.
+
+- **Start / resume** launches the configured command. Resuming completed work depends on that command's checkpoint behavior. ReAgent uses its configured session and attempt limits; the monitor does not reset them.
+- **Stop run** terminates the owned worker process tree. Saved session files remain intact. An interrupted model call may need to be repeated.
+- Closing the browser or monitor host leaves the worker running. Restart the monitor with the same work directory, state directory and worker command to reconnect.
+- Duplicate starts sharing a state directory are protected by a process lock. Adoption checks the command, working directory record, PID and process creation time, so stale PID records do not attach to unrelated processes.
+
+Control records and worker stdout/stderr live in `reports/monitor` by default; override with `--state-dir`. Treat these as local run artifacts, not source files. `--port` changes the port; zero selects an available port.
+
+## Local access
+
+The server binds only to IPv4 loopback. Host-header checks reject DNS-rebinding requests. Control endpoints require a random per-host token and reject cross-origin requests. HTTP clients cannot supply a different worker command or arbitrary file path. No external scripts, fonts, telemetry or services are used by the dashboard. This is a local tool, not a multi-user network service.
diff --git a/pyproject.toml b/pyproject.toml
index cd534874..d0a83f42 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -43,10 +43,11 @@ dependencies = [
]
[project.optional-dependencies]
+monitor = ["psutil>=7.0"]
ghidra-bridge = ["ghidra-ai-bridge>=0.2.0"]
headless = ["ghidra-ai-bridge[headless]>=0.2.0"]
benchmark = ["pefile>=2024.8.26", "unicorn>=2.1"]
-dev = ["pytest>=8.0", "pytest-cov>=5.0", "ruff>=0.8", "mypy>=1.13"]
+dev = ["pytest>=8.0", "pytest-cov>=5.0", "ruff>=0.8", "mypy>=1.13", "psutil>=7.0"]
[project.scripts]
re-agent = "re_agent.cli.main:main"
@@ -74,7 +75,7 @@ warn_return_any = true
warn_unused_configs = true
[[tool.mypy.overrides]]
-module = ["anthropic.*", "openai.*"]
+module = ["anthropic.*", "openai.*", "psutil.*"]
ignore_missing_imports = true
[tool.pytest.ini_options]
diff --git a/src/re_agent/cli/cmd_monitor.py b/src/re_agent/cli/cmd_monitor.py
new file mode 100644
index 00000000..67fea363
--- /dev/null
+++ b/src/re_agent/cli/cmd_monitor.py
@@ -0,0 +1,22 @@
+"""Serve a local progress dashboard without importing model providers."""
+from __future__ import annotations
+
+import argparse
+import contextlib
+from pathlib import Path
+
+
+def cmd_monitor(args: argparse.Namespace) -> int:
+ from re_agent.monitor.server import Monitor, make_server
+
+ if not 0 <= args.port <= 65535:
+ raise ValueError("Port must be between 0 and 65535")
+ root = Path(args.work_dir).resolve()
+ monitor = Monitor(root, root / args.state_dir, args.session_glob or ["re-agent-progress.json"],
+ args.log_glob, args.total, args.worker)
+ with make_server(monitor, args.port) as server:
+ print(f"Live monitor: http://127.0.0.1:{server.server_port}", flush=True)
+ print("Closing the monitor does not stop the worker. Use Stop run to terminate it.", flush=True)
+ with contextlib.suppress(KeyboardInterrupt):
+ server.serve_forever()
+ return 0
diff --git a/src/re_agent/cli/main.py b/src/re_agent/cli/main.py
index d89a2a3b..5282bc8b 100644
--- a/src/re_agent/cli/main.py
+++ b/src/re_agent/cli/main.py
@@ -15,6 +15,15 @@ def build_parser() -> argparse.ArgumentParser:
sub = parser.add_subparsers(dest="command", help="Available commands")
+ monitor_p = sub.add_parser("monitor", help="Host a local live progress dashboard")
+ monitor_p.add_argument("--work-dir", default=".", help="Directory containing run sessions")
+ monitor_p.add_argument("--state-dir", default="reports/monitor", help="Monitor control state and worker logs")
+ monitor_p.add_argument("--session-glob", action="append", help="Relative session pattern (repeatable)")
+ monitor_p.add_argument("--log-glob", help="Relative activity-log pattern; shows most recently modified log")
+ monitor_p.add_argument("--total", type=int, default=0, help="Planned function count; 0 means unknown")
+ monitor_p.add_argument("--port", type=int, default=8765)
+ monitor_p.add_argument("--worker", nargs=argparse.REMAINDER, help="Optional worker argv; must be the last option")
+
doctor_p = sub.add_parser("doctor", help="Check configuration and exported evidence without LLM calls")
doctor_p.add_argument("--address", help="Check evidence for one function")
benchmark_p = sub.add_parser("benchmark", help="Run a differential harness manifest")
@@ -78,6 +87,11 @@ def _main(argv: list[str] | None = None) -> int:
parser.print_help()
return 0
+ if args.command == "monitor":
+ from re_agent.cli.cmd_monitor import cmd_monitor
+
+ return cmd_monitor(args)
+
if args.command == "evidence":
from re_agent.cli.cmd_evidence import cmd_evidence
diff --git a/src/re_agent/monitor/__init__.py b/src/re_agent/monitor/__init__.py
new file mode 100644
index 00000000..c32bcb9a
--- /dev/null
+++ b/src/re_agent/monitor/__init__.py
@@ -0,0 +1 @@
+"""Optional local progress dashboard; independent of the reversal worker."""
diff --git a/src/re_agent/monitor/index.html b/src/re_agent/monitor/index.html
new file mode 100644
index 00000000..0ad65da4
--- /dev/null
+++ b/src/re_agent/monitor/index.html
@@ -0,0 +1,393 @@
+
+
+
+
+
+ Reconstruction · Live
+
+
+
+
+
+
+
ReAgent / local workspace
+
Reconstruction, live.
+
Session progress · generation and review
+
+
+
+
+
+
+
+
+
+ Connecting…
+
Waiting for worker state
+
+
+ 0.00%
+
No results yet
+
+
+
+
Loading live state…
+
+
+
+
FUNCTIONS ATTEMPTED
+
0
+
Saved final results
+
+
+
REVIEW PASSED
+
0
+
Not a proof of equivalence
+
+
+
NEEDS ATTENTION
+
0
+
Review or processing failed
+
+
+
REVIEW ROUNDS
+
0
+
Across saved results
+
+
+
+ Draft pass — not a verified build.
+
+ Review passes do not establish compilation or binary equivalence.
+ Saved source remains separate from accepted implementations.
+
+
+
+
+
Recent results
+
+
+
+
Address
+
Outcome
+
Rounds
+
+
+
+
+
Results appear as functions finish.
+
+
+
+
+
+
+
Worker activity
+ Live
+
+
Waiting for activity…
+
+
+
+
+
+
+
diff --git a/src/re_agent/monitor/server.py b/src/re_agent/monitor/server.py
new file mode 100644
index 00000000..086767a9
--- /dev/null
+++ b/src/re_agent/monitor/server.py
@@ -0,0 +1,220 @@
+"""Loopback-only progress reporting and explicitly configured worker control."""
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import secrets
+import subprocess
+import threading
+import time
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+from typing import Any
+
+from re_agent.utils.storage import atomic_json, file_lock
+
+
+def read_json(path: Path) -> dict[str, Any]:
+ try:
+ value = json.loads(path.read_text(encoding="utf-8"))
+ return value if isinstance(value, dict) else {}
+ except (OSError, ValueError):
+ return {}
+
+
+class Monitor:
+ """Read sessions without changing them; control only a configured child worker."""
+
+ def __init__(self, work_dir: Path, state_dir: Path, session_globs: list[str],
+ log_glob: str | None = None, total: int = 0, worker: list[str] | None = None) -> None:
+ if total < 0:
+ raise ValueError("Total function count must be nonnegative")
+ for pattern in [*session_globs, *([log_glob] if log_glob else [])]:
+ if Path(pattern).anchor or ".." in Path(pattern).parts:
+ raise ValueError("Monitor patterns must be relative to the working directory")
+ self.work_dir = work_dir.resolve()
+ self.state_dir = state_dir.resolve()
+ self.session_globs = session_globs
+ self.log_glob = log_glob
+ self.total = total
+ self.worker = worker or []
+ self.record = self.state_dir / "worker.json"
+ self.process: Any = None
+ self.psutil: Any = None
+ if self.worker:
+ try:
+ import psutil
+ except ImportError as exc:
+ raise RuntimeError("Worker controls require: pip install 'auto-re-agent[monitor]'") from exc
+ self.psutil = psutil
+ self.lock = threading.RLock()
+ self.cache: dict[Path, tuple[int, list[dict[str, Any]]]] = {}
+ self._adopt()
+
+ def _adopt(self) -> None:
+ if not self.worker:
+ return
+ saved = read_json(self.record)
+ try:
+ if saved.get("command") != self.worker or saved.get("cwd") != str(self.work_dir):
+ return
+ process = self.psutil.Process(saved["pid"])
+ if (process.create_time() == saved["created"]
+ and process.cmdline() == saved.get("process_command", self.worker)):
+ self.process = process
+ except (self.psutil.Error, KeyError):
+ pass
+
+ def active(self) -> bool:
+ if self.process is None:
+ return False
+ try:
+ if isinstance(self.process, self.psutil.Popen) and self.process.poll() is not None:
+ return False
+ return bool(self.process.is_running() and self.process.status() != self.psutil.STATUS_ZOMBIE)
+ except self.psutil.Error:
+ return False
+
+ def start(self) -> dict[str, Any]:
+ if not self.worker:
+ raise ValueError("Read-only monitor: no worker command configured")
+ with self.lock, file_lock(self.record):
+ self._adopt()
+ if self.active():
+ return {"message": "Already running", "pid": self.process.pid}
+ self.state_dir.mkdir(parents=True, exist_ok=True)
+ with (self.state_dir / "worker.stdout.log").open("ab") as out, \
+ (self.state_dir / "worker.stderr.log").open("ab") as err:
+ self.process = self.psutil.Popen(
+ self.worker, cwd=self.work_dir, stdout=out, stderr=err,
+ creationflags=int(getattr(subprocess, "CREATE_NO_WINDOW", 0)) if os.name == "nt" else 0,
+ start_new_session=os.name != "nt",
+ )
+ atomic_json(self.record, {"pid": self.process.pid, "created": self.process.create_time(),
+ "command": self.worker, "process_command": self.process.cmdline(),
+ "cwd": str(self.work_dir)})
+ return {"message": "Worker started", "pid": self.process.pid}
+
+ def stop(self) -> dict[str, str]:
+ if not self.worker:
+ raise ValueError("Read-only monitor: no worker command configured")
+ with self.lock, file_lock(self.record):
+ self._adopt()
+ if self.active():
+ if os.name == "nt":
+ subprocess.run(["taskkill", "/PID", str(self.process.pid), "/T", "/F"],
+ capture_output=True, timeout=20, check=True)
+ else:
+ import signal
+
+ os.killpg(self.process.pid, signal.SIGTERM)
+ try:
+ self.process.wait(timeout=5)
+ except self.psutil.TimeoutExpired:
+ os.killpg(self.process.pid, signal.SIGKILL)
+ # Descendants can ignore SIGTERM even when the parent exits.
+ with contextlib.suppress(ProcessLookupError):
+ os.killpg(self.process.pid, signal.SIGKILL)
+ self.process.wait(timeout=20)
+ return {"message": "Worker stopped; saved results retained"}
+
+ def snapshot(self) -> dict[str, Any]:
+ with self.lock:
+ paths = {p for pattern in self.session_globs for p in self.work_dir.glob(pattern) if p.is_file()}
+ for stale in self.cache.keys() - paths:
+ del self.cache[stale]
+ functions: dict[str, dict[str, Any]] = {}
+ for path in sorted(paths):
+ try:
+ stamp = path.stat().st_mtime_ns
+ except OSError:
+ continue
+ if path not in self.cache or self.cache[path][0] != stamp:
+ data = read_json(path).get("functions")
+ if isinstance(data, dict):
+ rows = [{key: row.get(key) for key in
+ ("address", "function_name", "success", "rounds_used", "timestamp")}
+ for row in data.values()
+ if isinstance(row, dict) and isinstance(row.get("address"), str)]
+ self.cache[path] = (stamp, rows)
+ for row in self.cache.get(path, (0, []))[1]:
+ address = row["address"].lower().removeprefix("0x").lstrip("0") or "0"
+ prior = functions.get(address)
+ if prior is None or str(row.get("timestamp") or "") >= str(prior.get("timestamp") or ""):
+ functions[address] = row
+ recent = sorted(functions.values(), key=lambda r: str(r.get("timestamp") or ""), reverse=True)
+ passed = sum(row["success"] is True for row in recent)
+ rounds = sum(row["rounds_used"] for row in recent if type(row.get("rounds_used")) is int)
+ tail = ""
+ try:
+ logs = (list(self.work_dir.glob(self.log_glob)) if self.log_glob
+ else [self.state_dir / "worker.stderr.log"])
+ log = max((p for p in logs if p.is_file()), key=lambda p: p.stat().st_mtime_ns, default=None)
+ if log:
+ with log.open("rb") as stream:
+ stream.seek(max(0, log.stat().st_size - 7000))
+ tail = stream.read().decode("utf-8", errors="replace")
+ except OSError:
+ pass
+ return {"active": self.active(), "controls": bool(self.worker),
+ "phase": "running" if self.active() else "idle" if self.worker else "read-only",
+ "targets": self.total, "completed": len(recent), "passed": passed,
+ "failed": len(recent) - passed, "rounds": rounds, "recent": recent[:18], "log": tail,
+ "updated": time.strftime("%H:%M:%S"), "output": str(self.work_dir)}
+
+
+def make_server(monitor: Monitor, port: int = 8765) -> ThreadingHTTPServer:
+ """Bind only loopback; never accept worker commands from HTTP clients."""
+ token = secrets.token_urlsafe(32)
+
+ class Handler(BaseHTTPRequestHandler):
+ server: ThreadingHTTPServer
+
+ def reply(self, code: int, body: str, content_type: str = "application/json") -> None:
+ payload = body.encode("utf-8")
+ self.send_response(code)
+ self.send_header("Content-Type", content_type + "; charset=utf-8")
+ self.send_header("Cache-Control", "no-store")
+ self.send_header("X-Content-Type-Options", "nosniff")
+ self.send_header("Content-Security-Policy", "frame-ancestors 'none'")
+ self.send_header("Content-Length", str(len(payload)))
+ self.end_headers()
+ self.wfile.write(payload)
+
+ def valid_host(self) -> bool:
+ return self.headers.get("Host") == f"127.0.0.1:{self.server.server_port}"
+
+ def do_GET(self) -> None:
+ if not self.valid_host():
+ self.reply(403, '{"error":"Invalid host"}')
+ elif self.path == "/":
+ html = Path(__file__).with_name("index.html").read_text(encoding="utf-8")
+ self.reply(200, html.replace("__TOKEN__", token), "text/html")
+ elif self.path == "/api/status":
+ self.reply(200, json.dumps(monitor.snapshot()))
+ else:
+ self.reply(404, '{"error":"Not found"}')
+
+ def do_POST(self) -> None:
+ origin = f"http://127.0.0.1:{self.server.server_port}"
+ if (not self.valid_host() or self.headers.get("Origin") not in (None, origin)
+ or self.headers.get("X-Control-Token") != token):
+ self.reply(403, '{"error":"Control request rejected"}')
+ return
+ try:
+ if self.path == "/api/start":
+ result = monitor.start()
+ elif self.path == "/api/stop":
+ result = monitor.stop()
+ else:
+ self.reply(404, '{"error":"Not found"}')
+ return
+ self.reply(200, json.dumps(result))
+ except (OSError, ValueError, RuntimeError, subprocess.SubprocessError) as exc:
+ self.reply(400, json.dumps({"error": str(exc)}))
+
+ def log_message(self, format: str, *args: Any) -> None:
+ pass
+
+ return ThreadingHTTPServer(("127.0.0.1", port), Handler)
diff --git a/tests/test_monitor.py b/tests/test_monitor.py
new file mode 100644
index 00000000..5e28c9af
--- /dev/null
+++ b/tests/test_monitor.py
@@ -0,0 +1,186 @@
+"""Exercise monitor accounting, HTTP boundaries and real disposable worker trees."""
+from __future__ import annotations
+
+import json
+import re
+import sys
+import threading
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+import psutil
+import pytest
+
+from re_agent.cli.main import build_parser
+from re_agent.monitor.server import Monitor, make_server
+
+
+def save(path, functions):
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(json.dumps({"functions": functions}), encoding="utf-8")
+
+
+def test_sessions_count_latest_result_once_and_tolerate_partial_writes(tmp_path):
+ a, b = tmp_path / "a.json", tmp_path / "b.json"
+ save(a, {"f": {"address": "0x00100", "success": False, "rounds_used": 3, "timestamp": "1"}})
+ save(b, {"f": {"address": "100", "success": True, "rounds_used": 1, "timestamp": "2"},
+ "g": {"address": "200", "success": False, "rounds_used": 2}, "malformed": []})
+ monitor = Monitor(tmp_path, tmp_path / "state", ["*.json", "a.json"])
+ state = monitor.snapshot()
+ assert (state["completed"], state["passed"], state["failed"], state["rounds"]) == (2, 1, 1, 3)
+ assert state["phase"] == "read-only"
+ assert not state["controls"]
+ b.write_text('{"functions":', encoding="utf-8")
+ assert monitor.snapshot()["completed"] == 2
+ b.unlink()
+ assert monitor.snapshot()["passed"] == 0
+ assert monitor.snapshot()["completed"] == 1
+
+
+def test_log_tail_and_recent_results_are_bounded(tmp_path):
+ (tmp_path / "activity.log").write_bytes(b"x" * 20000 + b"END")
+ save(tmp_path / "session.json", {str(n): {"address": hex(n), "success": True} for n in range(30)})
+ state = Monitor(tmp_path, tmp_path / "state", ["session.json"], "*.log").snapshot()
+ assert len(state["log"]) == 7000
+ assert state["log"].endswith("END")
+ assert state["completed"] == 30
+ assert len(state["recent"]) == 18
+
+
+@pytest.mark.parametrize("pattern", ["../*.json", "/absolute/session.json"])
+def test_patterns_cannot_escape_work_dir(tmp_path, pattern):
+ with pytest.raises(ValueError, match="relative"):
+ Monitor(tmp_path, tmp_path / "state", [pattern])
+
+
+def test_read_only_monitor_cannot_start_or_stop(tmp_path):
+ monitor = Monitor(tmp_path, tmp_path / "state", [])
+ for action in (monitor.start, monitor.stop):
+ with pytest.raises(ValueError, match="Read-only"):
+ action()
+ assert not (tmp_path / "state").exists()
+
+
+def test_worker_tree_stop_duplicate_start_and_host_reconnect(tmp_path):
+ script = tmp_path / "worker with spaces.py"
+ script.write_text(
+ "import pathlib, subprocess, sys, time\n"
+ "child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(120)'])\n"
+ "pathlib.Path('child.pid').write_text(str(child.pid))\n"
+ "time.sleep(120)\n", encoding="utf-8")
+ command = [sys.executable, str(script)]
+ first = Monitor(tmp_path, tmp_path / "state", [], worker=command)
+ second = None
+ try:
+ first.start()
+ pid = first.process.pid
+ assert first.start()["pid"] == pid
+ second = Monitor(tmp_path, tmp_path / "state", [], worker=command)
+ assert second.active()
+ assert second.start()["pid"] == pid
+ deadline = time.monotonic() + 10
+ while not (tmp_path / "child.pid").exists() and time.monotonic() < deadline:
+ time.sleep(.02)
+ child = psutil.Process(int((tmp_path / "child.pid").read_text()))
+ second.stop()
+ assert not second.active()
+ assert not first.active()
+ deadline = time.monotonic() + 5
+ while child.is_running() and child.status() != psutil.STATUS_ZOMBIE and time.monotonic() < deadline:
+ time.sleep(.02)
+ assert not child.is_running() or child.status() == psutil.STATUS_ZOMBIE
+ first.start()
+ assert first.active()
+ first.stop()
+ finally:
+ first.stop()
+ if second:
+ second.stop()
+
+
+def test_stale_identity_and_unrelated_process_are_not_adopted(tmp_path):
+ state = tmp_path / "state"
+ state.mkdir()
+ current = psutil.Process()
+ worker = [sys.executable, "not-the-running-worker.py"]
+ record = {"pid": current.pid, "created": current.create_time(), "command": worker, "cwd": str(tmp_path)}
+ (state / "worker.json").write_text(json.dumps(record))
+ monitor = Monitor(tmp_path, state, [], worker=worker)
+ assert not monitor.active()
+ monitor.stop()
+ assert current.is_running()
+ record.update(created=0, command=current.cmdline())
+ (state / "worker.json").write_text(json.dumps(record))
+ assert not Monitor(tmp_path, state, [], worker=current.cmdline()).active()
+
+
+@pytest.fixture
+def http_monitor(tmp_path):
+ monitor = Monitor(tmp_path, tmp_path / "state", [])
+ server = make_server(monitor, 0)
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ try:
+ yield monitor, f"http://127.0.0.1:{server.server_port}"
+ finally:
+ server.shutdown()
+ server.server_close()
+ thread.join(timeout=5)
+
+
+def test_http_serves_packaged_ui_and_read_only_status(http_monitor):
+ _, url = http_monitor
+ with urllib.request.urlopen(url) as response:
+ html = response.read().decode()
+ assert "Reconstruction, live." in html
+ assert "__TOKEN__" not in html
+ assert response.headers["Cache-Control"] == "no-store"
+ with urllib.request.urlopen(url + "/api/status") as response:
+ state = json.load(response)
+ assert not state["controls"]
+
+
+@pytest.mark.parametrize("headers", [{}, {"X-Control-Token": "wrong"}, {"Host": "untrusted.example"}])
+def test_http_rejects_unauthorized_control(http_monitor, headers):
+ _, url = http_monitor
+ req = urllib.request.Request(url + "/api/start", method="POST", headers=headers)
+ with pytest.raises(urllib.error.HTTPError) as error:
+ urllib.request.urlopen(req)
+ assert error.value.code == 403
+
+
+def test_http_origin_rebinding_and_no_arbitrary_command(http_monitor, monkeypatch):
+ monitor, url = http_monitor
+ with urllib.request.urlopen(url) as response:
+ token = re.search(r"const token\s*=\s*[\"']([^\"']+)[\"']", response.read().decode()).group(1)
+ for path, headers in [("/", {"Host": "other.example"}),
+ ("/api/start", {"X-Control-Token": token, "Origin": "https://other.example"})]:
+ req = urllib.request.Request(url + path, method="GET" if path == "/" else "POST", headers=headers)
+ with pytest.raises(urllib.error.HTTPError) as error:
+ urllib.request.urlopen(req)
+ assert error.value.code == 403
+ calls = []
+ monkeypatch.setattr(monitor, "start", lambda: calls.append("configured worker") or {"message": "started"})
+ req = urllib.request.Request(url + "/api/start", data=b'{"command":["untrusted"]}',
+ headers={"X-Control-Token": token, "Origin": url}, method="POST")
+ with urllib.request.urlopen(req) as response:
+ assert response.status == 200
+ assert calls == ["configured worker"]
+
+
+def test_cli_parses_worker_argv_without_shell_interpretation():
+ args = build_parser().parse_args(["monitor", "--total", "10", "--session-glob", "batches/*/progress.json",
+ "--worker", "python", "script with spaces.py", "--flag", "a&b"])
+ assert args.command == "monitor"
+ assert args.worker == ["python", "script with spaces.py", "--flag", "a&b"]
+ assert args.total == 10
+
+
+def test_packaged_html_uses_text_nodes_for_untrusted_results():
+ import re_agent.monitor.server as server
+
+ html = Path(server.__file__).with_name("index.html").read_text(encoding="utf-8")
+ assert "textContent" in html
+ assert "innerHTML" not in html
From ce06b7cb73469eddfeafb34116046ef426109ae6 Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 07:47:08 +1000
Subject: [PATCH 04/34] Reject unresolved decompiler targets and include
disassembly evidence
---
src/re_agent/agents/loop.py | 9 +++--
src/re_agent/agents/reverser.py | 5 ++-
src/re_agent/verification/candidate.py | 11 ++++++
tests/test_agents/test_unresolved.py | 46 ++++++++++++++++++++++++++
4 files changed, 68 insertions(+), 3 deletions(-)
create mode 100644 tests/test_agents/test_unresolved.py
diff --git a/src/re_agent/agents/loop.py b/src/re_agent/agents/loop.py
index fc0d4487..dd8acb4f 100644
--- a/src/re_agent/agents/loop.py
+++ b/src/re_agent/agents/loop.py
@@ -25,7 +25,7 @@
from re_agent.llm.observed import CallBudget, ObservedProvider
from re_agent.llm.protocol import LLMProvider
from re_agent.parity.source_indexer import SourceIndexer
-from re_agent.verification.candidate import extract_candidate_body
+from re_agent.verification.candidate import extract_candidate_body, unresolved_placeholders
from re_agent.verification.objective import verify_candidate
@@ -137,6 +137,9 @@ def run_fix_loop(
extract_candidate_body(code)
except ValueError as exc:
preflight_error = str(exc)
+ placeholders = unresolved_placeholders(code)
+ if placeholders:
+ preflight_error = "Unresolved decompiler placeholders: " + ", ".join(placeholders)
if preflight_error:
checker.last_prompt = ""
checker.last_response = ""
@@ -144,7 +147,9 @@ def run_fix_loop(
verdict=Verdict.FAIL,
summary="Local candidate preflight failed; model review skipped",
issues=[preflight_error],
- fix_instructions=["Return exactly one complete function definition without helper definitions."],
+ fix_instructions=[
+ "Resolve the reported issue using binary evidence; do not invent declarations for unknown targets."
+ ],
)
else:
verdict = checker.check(code, target)
diff --git a/src/re_agent/agents/reverser.py b/src/re_agent/agents/reverser.py
index 68927d1b..5c19a52d 100644
--- a/src/re_agent/agents/reverser.py
+++ b/src/re_agent/agents/reverser.py
@@ -152,7 +152,7 @@ def add(label: str, getter: object, argument: str) -> None:
return
if artifact is None:
return
- content = getattr(artifact, "content", "")
+ content = getattr(artifact, "content", "") or getattr(artifact, "instructions", "")
if content:
if label == "Function evidence bundle" and self._knowledge_graph is not None:
self._knowledge_graph.ingest_context(str(content))
@@ -161,6 +161,9 @@ def add(label: str, getter: object, argument: str) -> None:
caps = self.backend.capabilities
if getattr(caps, "has_context", False):
add("Function evidence bundle", getattr(self.backend, "get_context", None), target.address)
+ if getattr(caps, "has_asm", False):
+ add("Disassembly (resolve decompiler jump/call ambiguities)", getattr(self.backend, "get_asm", None),
+ target.address)
if getattr(caps, "has_vtables", False) and target.class_name:
add("Vtable", getattr(self.backend, "get_vtable", None), target.class_name)
if getattr(caps, "has_cfg", False):
diff --git a/src/re_agent/verification/candidate.py b/src/re_agent/verification/candidate.py
index 32f42f17..b1674491 100644
--- a/src/re_agent/verification/candidate.py
+++ b/src/re_agent/verification/candidate.py
@@ -14,6 +14,17 @@
from re_agent.core.models import FunctionTarget, SourceMatch, ValidationVerdict, Verdict
from re_agent.utils.process import run_process
+_NON_CODE = re.compile(
+ r'//[^\n]*|/\*[\s\S]*?\*/|R"(?P[^ ()\\\t\r\n]{0,16})\([\s\S]*?\)(?P=delimiter)"'
+ r'|"(?:\\.|[^"\\])*"|\'(?:\\.|[^\'\\])*\''
+)
+
+
+def unresolved_placeholders(code: str) -> list[str]:
+ """Find executable unrecovered-jump placeholders, ignoring comments/literals."""
+ tokens = _NON_CODE.sub(" ", code)
+ return sorted(set(re.findall(r"\bUNRECOVERED_JUMPTABLE(?:_\w+)?\b", tokens)))
+
def extract_candidate_body(code: str) -> str:
"""Extract the outer C++ body from generated code."""
diff --git a/tests/test_agents/test_unresolved.py b/tests/test_agents/test_unresolved.py
new file mode 100644
index 00000000..36f13982
--- /dev/null
+++ b/tests/test_agents/test_unresolved.py
@@ -0,0 +1,46 @@
+"""Do not accept unresolved decompiler targets as reconstructed control flow."""
+import pytest
+
+from re_agent.agents.loop import run_fix_loop
+from re_agent.agents.reverser import ReverserAgent
+from re_agent.backend.protocol import BackendCapabilities
+from re_agent.backend.stub import StubBackend
+from re_agent.core.models import AsmResult, FunctionTarget
+from re_agent.verification.candidate import unresolved_placeholders
+from tests.test_agents.test_loop import MockLLM
+
+
+@pytest.mark.parametrize("code", [
+ 'void f() { UNRECOVERED_JUMPTABLE(); }',
+ 'extern void UNRECOVERED_JUMPTABLE(); void f() { UNRECOVERED_JUMPTABLE(); }',
+ 'void f() { ((void(*)())UNRECOVERED_JUMPTABLE_1)(); }',
+])
+def test_placeholder_cannot_pass_even_without_validation_gate(code):
+ checker = MockLLM(['{"verdict":"PASS","summary":"ok"}'])
+ result = run_fix_loop(FunctionTarget("100", "", "f"), StubBackend(),
+ MockLLM([f"```cpp\n{code}\n```"]), checker, max_rounds=1)
+ assert not result.success
+ assert checker._idx == 0
+ assert "Unresolved" in result.checker_verdict.issues[0]
+
+
+def test_comments_and_literals_do_not_trigger_placeholder_detection():
+ code = '''// UNRECOVERED_JUMPTABLE
+ void f() { /* UNRECOVERED_JUMPTABLE */
+ const char* a = "UNRECOVERED_JUMPTABLE";
+ const char* b = R"tag(escaped " UNRECOVERED_JUMPTABLE)tag";
+ }'''
+ assert unresolved_placeholders(code) == []
+
+
+def test_disassembly_is_available_with_small_evidence_budget():
+ class Backend(StubBackend):
+ @property
+ def capabilities(self):
+ return BackendCapabilities(has_asm=True)
+
+ def get_asm(self, address):
+ return AsmResult(address, "100 JMP RAX", 1, 0, False)
+
+ agent = ReverserAgent(MockLLM([]), Backend(), max_investigations=1)
+ assert "100 JMP RAX" in agent._build_investigation_context(FunctionTarget("100", "", "f"))
From f7a1b0b4ad9887f30a857d4b867bacf767526653 Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 07:49:53 +1000
Subject: [PATCH 05/34] Run local candidate validation before model review
---
src/re_agent/agents/loop.py | 11 ++++
src/re_agent/orchestrator/single.py | 13 ++++
tests/test_agents/test_compile_preflight.py | 69 +++++++++++++++++++++
3 files changed, 93 insertions(+)
create mode 100644 tests/test_agents/test_compile_preflight.py
diff --git a/src/re_agent/agents/loop.py b/src/re_agent/agents/loop.py
index dd8acb4f..e5c5c02c 100644
--- a/src/re_agent/agents/loop.py
+++ b/src/re_agent/agents/loop.py
@@ -48,6 +48,7 @@ def run_fix_loop(
max_investigations: int = 8,
candidate_gate: Callable[[ReversalResult], ReversalResult] | None = None,
max_llm_calls: int = 80,
+ candidate_preflight: Callable[[ReversalResult], ReversalResult] | None = None,
) -> ReversalResult:
"""Run the reverser->checker->fix loop up to max_rounds.
@@ -140,6 +141,16 @@ def run_fix_loop(
placeholders = unresolved_placeholders(code)
if placeholders:
preflight_error = "Unresolved decompiler placeholders: " + ", ".join(placeholders)
+ if not preflight_error and candidate_preflight is not None:
+ checked = candidate_preflight(ReversalResult(
+ target=target, code=code, success=True, rounds_used=round_num, run_id=run_id,
+ ))
+ if not checked.success:
+ details = []
+ if checked.validation_verdict:
+ details = [checked.validation_verdict.summary, *checked.validation_verdict.findings]
+ details.extend(f.reason for f in checked.parity_findings)
+ preflight_error = "Local validation preflight failed: " + "; ".join(details)
if preflight_error:
checker.last_prompt = ""
checker.last_response = ""
diff --git a/src/re_agent/orchestrator/single.py b/src/re_agent/orchestrator/single.py
index e3cb5e3a..71e20e6f 100644
--- a/src/re_agent/orchestrator/single.py
+++ b/src/re_agent/orchestrator/single.py
@@ -45,8 +45,20 @@ def reverse_single(
it here to avoid re-scanning the entire source tree each time.
"""
log_dir = Path(config.output.log_dir) if config.output.log_dir else None
+ checked: ReversalResult | None = None
+
+ def preflight(result: ReversalResult) -> ReversalResult:
+ nonlocal checked
+ checked = validate_result(result, config, backend, indexer)
+ return checked
def gate(result: ReversalResult) -> ReversalResult:
+ if checked is not None and checked.code == result.code:
+ result.validation_verdict = checked.validation_verdict
+ result.parity_status = checked.parity_status
+ result.parity_findings = checked.parity_findings
+ result.success = result.success and checked.success
+ return result
return validate_result(result, config, backend, indexer)
try:
@@ -68,6 +80,7 @@ def gate(result: ReversalResult) -> ReversalResult:
investigation_enabled=config.orchestrator.investigation_enabled,
max_investigations=config.orchestrator.max_investigations,
candidate_gate=gate,
+ candidate_preflight=preflight if config.validation.enabled else None,
max_llm_calls=config.orchestrator.max_llm_calls_per_function,
)
except (RuntimeError, OSError, ValueError) as exc:
diff --git a/tests/test_agents/test_compile_preflight.py b/tests/test_agents/test_compile_preflight.py
new file mode 100644
index 00000000..ecad7735
--- /dev/null
+++ b/tests/test_agents/test_compile_preflight.py
@@ -0,0 +1,69 @@
+"""Real compiler failures feed repair before spending reviewer calls."""
+import shutil
+
+import pytest
+
+from re_agent.backend.stub import StubBackend
+from re_agent.config.schema import ReAgentConfig
+from re_agent.core.models import FunctionTarget, Verdict
+from re_agent.orchestrator.single import reverse_single
+from tests.test_agents.test_loop import MockLLM
+
+
+def test_compile_failure_repairs_without_reviewing_or_rebuilding_bad_output(tmp_path, monkeypatch):
+ compiler = shutil.which("clang++") or shutil.which("g++")
+ if not compiler:
+ pytest.skip("C++ compiler unavailable")
+ config = ReAgentConfig()
+ source_root = tmp_path / "src"
+ source_root.mkdir()
+ config.project_profile.source_root = str(source_root)
+ config.output.report_dir = str(tmp_path / "reports")
+ config.output.log_dir = str(tmp_path / "logs")
+ config.parity.enabled = False
+ config.orchestrator.objective_verifier_enabled = False
+ config.orchestrator.max_review_rounds = 2
+ config.orchestrator.max_llm_calls_per_function = 3
+ config.validation.build_commands = [[compiler, "-std=c++20", "-fsyntax-only", "{candidate_file}"]]
+ config.validation.require_build = True
+ config.validation.trust_configured_commands = True
+ config.validation.working_directory = str(tmp_path)
+ bad = 'std::uint32_t f() { return 1; }'
+ good = '#include \nstd::uint32_t f() { return 1; }'
+ reverser = MockLLM([f"```cpp\n{bad}\n```", f"```cpp\n{good}\n```"])
+ checker = MockLLM(['{"verdict":"PASS","summary":"Matches"}'])
+ from re_agent.verification import candidate
+
+ original = candidate.run_process
+ calls = []
+
+ def observed(*args, **kwargs):
+ calls.append(args)
+ return original(*args, **kwargs)
+
+ monkeypatch.setattr(candidate, "run_process", observed)
+ result = reverse_single(FunctionTarget("100", "", "f"), config, StubBackend(), reverser, checker_llm=checker)
+ assert result.success
+ assert result.rounds_used == 2
+ assert checker._idx == 1
+ assert reverser._idx == 2
+ assert len(calls) == 2 # Exactly once per candidate, never again after review.
+ assert result.validation_verdict.verdict == Verdict.PASS
+
+
+def test_successful_local_gate_does_not_override_failed_model_review(tmp_path):
+ config = ReAgentConfig()
+ source_root = tmp_path / "src"
+ source_root.mkdir()
+ config.project_profile.source_root = str(source_root)
+ config.output.report_dir = str(tmp_path / "reports")
+ config.output.log_dir = str(tmp_path / "logs")
+ config.parity.enabled = False
+ config.orchestrator.max_review_rounds = 1
+ config.validation.require_verified = False
+ result = reverse_single(FunctionTarget("100", "", "f"), config, StubBackend(),
+ MockLLM(['```cpp\nint f() { return 1; }\n```']),
+ checker_llm=MockLLM(['{"verdict":"FAIL","summary":"Wrong return"}']))
+ assert not result.success
+ assert result.checker_verdict.verdict == Verdict.FAIL
+
From 0588217c564aad44f62e9c6b054d3b7cc9d23eee Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 07:50:36 +1000
Subject: [PATCH 06/34] Persist monitor stop state across host reconnects
---
src/re_agent/monitor/server.py | 18 +++++++++++++++---
tests/test_monitor.py | 14 ++++++++++++++
2 files changed, 29 insertions(+), 3 deletions(-)
diff --git a/src/re_agent/monitor/server.py b/src/re_agent/monitor/server.py
index 086767a9..ed44a981 100644
--- a/src/re_agent/monitor/server.py
+++ b/src/re_agent/monitor/server.py
@@ -93,7 +93,7 @@ def start(self) -> dict[str, Any]:
)
atomic_json(self.record, {"pid": self.process.pid, "created": self.process.create_time(),
"command": self.worker, "process_command": self.process.cmdline(),
- "cwd": str(self.work_dir)})
+ "cwd": str(self.work_dir), "phase": "running"})
return {"message": "Worker started", "pid": self.process.pid}
def stop(self) -> dict[str, str]:
@@ -117,6 +117,9 @@ def stop(self) -> dict[str, str]:
with contextlib.suppress(ProcessLookupError):
os.killpg(self.process.pid, signal.SIGKILL)
self.process.wait(timeout=20)
+ saved = read_json(self.record)
+ saved.update(phase="stopped", stopped_at=time.time())
+ atomic_json(self.record, saved)
return {"message": "Worker stopped; saved results retained"}
def snapshot(self) -> dict[str, Any]:
@@ -157,8 +160,17 @@ def snapshot(self) -> dict[str, Any]:
tail = stream.read().decode("utf-8", errors="replace")
except OSError:
pass
- return {"active": self.active(), "controls": bool(self.worker),
- "phase": "running" if self.active() else "idle" if self.worker else "read-only",
+ self._adopt()
+ active = self.active()
+ saved = read_json(self.record)
+ matching = saved.get("command") == self.worker and saved.get("cwd") == str(self.work_dir)
+ phase = "idle" if self.worker else "read-only"
+ if active:
+ phase = "running"
+ elif self.worker and matching:
+ phase = "stopped" if saved.get("phase") == "stopped" else "exited"
+ return {"active": active, "controls": bool(self.worker),
+ "phase": phase,
"targets": self.total, "completed": len(recent), "passed": passed,
"failed": len(recent) - passed, "rounds": rounds, "recent": recent[:18], "log": tail,
"updated": time.strftime("%H:%M:%S"), "output": str(self.work_dir)}
diff --git a/tests/test_monitor.py b/tests/test_monitor.py
index 5e28c9af..7285c285 100644
--- a/tests/test_monitor.py
+++ b/tests/test_monitor.py
@@ -87,12 +87,15 @@ def test_worker_tree_stop_duplicate_start_and_host_reconnect(tmp_path):
second.stop()
assert not second.active()
assert not first.active()
+ assert first.snapshot()["phase"] == "stopped"
+ assert Monitor(tmp_path, tmp_path / "state", [], worker=command).snapshot()["phase"] == "stopped"
deadline = time.monotonic() + 5
while child.is_running() and child.status() != psutil.STATUS_ZOMBIE and time.monotonic() < deadline:
time.sleep(.02)
assert not child.is_running() or child.status() == psutil.STATUS_ZOMBIE
first.start()
assert first.active()
+ assert second.snapshot()["phase"] == "running"
first.stop()
finally:
first.stop()
@@ -116,6 +119,17 @@ def test_stale_identity_and_unrelated_process_are_not_adopted(tmp_path):
assert not Monitor(tmp_path, state, [], worker=current.cmdline()).active()
+def test_natural_exit_is_distinct_from_explicit_stop(tmp_path):
+ command = [sys.executable, "-c", "import time; time.sleep(.2)"]
+ monitor = Monitor(tmp_path, tmp_path / "state", [], worker=command)
+ assert monitor.snapshot()["phase"] == "idle"
+ monitor.start()
+ monitor.process.wait(timeout=10)
+ assert monitor.snapshot()["phase"] == "exited"
+ monitor.stop()
+ assert Monitor(tmp_path, tmp_path / "state", [], worker=command).snapshot()["phase"] == "exited"
+
+
@pytest.fixture
def http_monitor(tmp_path):
monitor = Monitor(tmp_path, tmp_path / "state", [])
From 6b6354a87a19308544631204a067338b47c57704 Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 07:56:43 +1000
Subject: [PATCH 07/34] Use stable launch identity when reconnecting to workers
---
src/re_agent/monitor/server.py | 12 +++++++++---
tests/test_monitor.py | 17 +++++++++++++++++
2 files changed, 26 insertions(+), 3 deletions(-)
diff --git a/src/re_agent/monitor/server.py b/src/re_agent/monitor/server.py
index ed44a981..dc23ac7b 100644
--- a/src/re_agent/monitor/server.py
+++ b/src/re_agent/monitor/server.py
@@ -60,8 +60,12 @@ def _adopt(self) -> None:
if saved.get("command") != self.worker or saved.get("cwd") != str(self.work_dir):
return
process = self.psutil.Process(saved["pid"])
- if (process.create_time() == saved["created"]
- and process.cmdline() == saved.get("process_command", self.worker)):
+ # Launchers may exec another interpreter and rewrite argv on startup.
+ # A per-launch marker survives exec without relaxing PID reuse checks.
+ marker = saved.get("launch_marker")
+ identity_matches = (process.environ().get("RE_AGENT_MONITOR_WORKER") == marker if marker
+ else process.cmdline() == saved.get("process_command", self.worker))
+ if process.create_time() == saved["created"] and identity_matches:
self.process = process
except (self.psutil.Error, KeyError):
pass
@@ -84,15 +88,17 @@ def start(self) -> dict[str, Any]:
if self.active():
return {"message": "Already running", "pid": self.process.pid}
self.state_dir.mkdir(parents=True, exist_ok=True)
+ marker = secrets.token_urlsafe(32)
with (self.state_dir / "worker.stdout.log").open("ab") as out, \
(self.state_dir / "worker.stderr.log").open("ab") as err:
self.process = self.psutil.Popen(
self.worker, cwd=self.work_dir, stdout=out, stderr=err,
+ env={**os.environ, "RE_AGENT_MONITOR_WORKER": marker},
creationflags=int(getattr(subprocess, "CREATE_NO_WINDOW", 0)) if os.name == "nt" else 0,
start_new_session=os.name != "nt",
)
atomic_json(self.record, {"pid": self.process.pid, "created": self.process.create_time(),
- "command": self.worker, "process_command": self.process.cmdline(),
+ "command": self.worker, "launch_marker": marker,
"cwd": str(self.work_dir), "phase": "running"})
return {"message": "Worker started", "pid": self.process.pid}
diff --git a/tests/test_monitor.py b/tests/test_monitor.py
index 7285c285..836677a5 100644
--- a/tests/test_monitor.py
+++ b/tests/test_monitor.py
@@ -130,6 +130,23 @@ def test_natural_exit_is_distinct_from_explicit_stop(tmp_path):
assert Monitor(tmp_path, tmp_path / "state", [], worker=command).snapshot()["phase"] == "exited"
+def test_adoption_uses_launch_marker_even_if_launcher_changes_argv(tmp_path):
+ command = [sys.executable, "-c", "import time; time.sleep(120)"]
+ first = Monitor(tmp_path, tmp_path / "state", [], worker=command)
+ try:
+ first.start()
+ saved = json.loads(first.record.read_text())
+ saved["process_command"] = ["a-transient-launcher", "different-argv"]
+ first.record.write_text(json.dumps(saved))
+ adopted = Monitor(tmp_path, tmp_path / "state", [], worker=command)
+ assert adopted.active()
+ saved["launch_marker"] = "unrelated-launch"
+ first.record.write_text(json.dumps(saved))
+ assert not Monitor(tmp_path, tmp_path / "state", [], worker=command).active()
+ finally:
+ first.stop()
+
+
@pytest.fixture
def http_monitor(tmp_path):
monitor = Monitor(tmp_path, tmp_path / "state", [])
From ab9feea983e67b346d8c31f0802cf5f402283bcb Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 08:16:00 +1000
Subject: [PATCH 08/34] Test compiler preflight repair for atomic candidates
---
tests/test_agents/test_compile_preflight.py | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/tests/test_agents/test_compile_preflight.py b/tests/test_agents/test_compile_preflight.py
index ecad7735..e2591bcc 100644
--- a/tests/test_agents/test_compile_preflight.py
+++ b/tests/test_agents/test_compile_preflight.py
@@ -10,7 +10,14 @@
from tests.test_agents.test_loop import MockLLM
-def test_compile_failure_repairs_without_reviewing_or_rebuilding_bad_output(tmp_path, monkeypatch):
+@pytest.mark.parametrize(("bad", "good"), [
+ ('std::uint32_t f() { return 1; }',
+ '#include \nstd::uint32_t f() { return 1; }'),
+ ('unsigned f(unsigned* p, unsigned v) { std::atomic_ref(*p).exchange(v); return v; }',
+ '#include \nunsigned f(unsigned* p, unsigned v) { '
+ 'std::atomic_ref(*p).exchange(v); return v; }'),
+], ids=["fixed-width-header", "atomic-header"])
+def test_compile_failure_repairs_without_reviewing_or_rebuilding_bad_output(tmp_path, monkeypatch, bad, good):
compiler = shutil.which("clang++") or shutil.which("g++")
if not compiler:
pytest.skip("C++ compiler unavailable")
@@ -28,8 +35,6 @@ def test_compile_failure_repairs_without_reviewing_or_rebuilding_bad_output(tmp_
config.validation.require_build = True
config.validation.trust_configured_commands = True
config.validation.working_directory = str(tmp_path)
- bad = 'std::uint32_t f() { return 1; }'
- good = '#include \nstd::uint32_t f() { return 1; }'
reverser = MockLLM([f"```cpp\n{bad}\n```", f"```cpp\n{good}\n```"])
checker = MockLLM(['{"verdict":"PASS","summary":"Matches"}'])
from re_agent.verification import candidate
From b88fbfd865abfffb3d544b0ecc9d3dd26dfed0c5 Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 08:17:42 +1000
Subject: [PATCH 09/34] Preserve first compiler errors in bounded repair
diagnostics
---
src/re_agent/verification/candidate.py | 9 ++++-
.../test_diagnostic_excerpt.py | 40 +++++++++++++++++++
2 files changed, 47 insertions(+), 2 deletions(-)
create mode 100644 tests/test_verification/test_diagnostic_excerpt.py
diff --git a/src/re_agent/verification/candidate.py b/src/re_agent/verification/candidate.py
index b1674491..2e630a0d 100644
--- a/src/re_agent/verification/candidate.py
+++ b/src/re_agent/verification/candidate.py
@@ -181,8 +181,13 @@ def validate_candidate(
except subprocess.TimeoutExpired:
checks.append({"kind": kind, "verdict": "FAIL", "detail": "timed out"})
return _failed(f"{kind} command timed out: {command}", candidate_file, findings, checks)
- tail = "\n".join((proc.stdout + proc.stderr).splitlines()[-20:])
- findings.append(f"{kind}: {command} -> exit {proc.returncode}\n{tail}".rstrip())
+ # First errors often explain cascades of missing types/declarations.
+ # Keep them alongside the final summary, prioritizing compiler stderr.
+ lines = (proc.stderr + "\n" + proc.stdout).strip().splitlines()
+ if len(lines) > 20:
+ lines = [*lines[:10], "[intermediate output omitted]", *lines[-10:]]
+ excerpt = "\n".join(line[:500] for line in lines)
+ findings.append(f"{kind}: {command} -> exit {proc.returncode}\n{excerpt}".rstrip())
checks.append({"kind": kind, "verdict": "PASS" if proc.returncode == 0 else "FAIL",
"detail": f"exit {proc.returncode}"})
if proc.returncode != 0:
diff --git a/tests/test_verification/test_diagnostic_excerpt.py b/tests/test_verification/test_diagnostic_excerpt.py
new file mode 100644
index 00000000..afda8c79
--- /dev/null
+++ b/tests/test_verification/test_diagnostic_excerpt.py
@@ -0,0 +1,40 @@
+"""Preserve actionable first compiler errors without unbounded repair prompts."""
+import sys
+
+from re_agent.config.schema import ValidationConfig
+from re_agent.core.models import Verdict
+from re_agent.verification.candidate import validate_candidate
+
+
+def test_failed_command_preserves_first_error_and_final_summary(tmp_path):
+ candidate = tmp_path / "candidate.cpp"
+ candidate.write_text("void f() {}")
+ script = (
+ "import sys; "
+ "print('error: missing declaration', file=sys.stderr); "
+ "[print('cascade ' + str(i), file=sys.stderr) for i in range(40)]; "
+ "print('compilation failed', file=sys.stderr); sys.exit(1)"
+ )
+ config = ValidationConfig(
+ build_commands=[[sys.executable, "-c", script, "{candidate_file}"]],
+ working_directory=str(tmp_path),
+ )
+ result = validate_candidate(config, candidate, None)
+ detail = result.findings[-1]
+ assert result.verdict == Verdict.FAIL
+ assert "error: missing declaration" in detail
+ assert "compilation failed" in detail
+ assert "[intermediate output omitted]" in detail
+ assert "cascade 20" not in detail
+
+
+def test_stderr_precedes_stdout_and_long_diagnostic_lines_are_bounded(tmp_path):
+ candidate = tmp_path / "candidate.cpp"
+ candidate.write_text("void f() {}")
+ script = "import sys; print('x'*20000); print('root error', file=sys.stderr); sys.exit(1)"
+ config = ValidationConfig(build_commands=[[sys.executable, "-c", script, "{candidate_file}"]],
+ working_directory=str(tmp_path))
+ result = validate_candidate(config, candidate, None)
+ detail = result.findings[-1]
+ assert "root error" in detail
+ assert "x" * 501 not in detail
From aa7a5e6e7b83c0e8bc2f9677d120c84b3bb8887c Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 08:31:20 +1000
Subject: [PATCH 10/34] Stop candidate repair when structural evidence scopes
conflict
---
src/re_agent/agents/loop.py | 5 +-
src/re_agent/core/models.py | 1 +
src/re_agent/verification/objective.py | 20 ++++++++
.../test_verification/test_evidence_scope.py | 48 +++++++++++++++++++
4 files changed, 73 insertions(+), 1 deletion(-)
create mode 100644 tests/test_verification/test_evidence_scope.py
diff --git a/src/re_agent/agents/loop.py b/src/re_agent/agents/loop.py
index e5c5c02c..e2c4c8cf 100644
--- a/src/re_agent/agents/loop.py
+++ b/src/re_agent/agents/loop.py
@@ -209,6 +209,9 @@ def run_fix_loop(
)
if candidate_gate is not None:
result = candidate_gate(result)
+ if objective_verdict is not None and objective_verdict.evidence_conflict:
+ result.success = False
+ result.error = "Stopped: reconcile incompatible structural evidence before retrying"
gate_issues = [f"parity: {f.reason}" for f in result.parity_findings]
if result.validation_verdict and result.validation_verdict.verdict != Verdict.PASS:
gate_issues.extend([result.validation_verdict.summary, *result.validation_verdict.findings])
@@ -218,7 +221,7 @@ def run_fix_loop(
(log_dir / f"round{round_num}-result.json").write_text(results_to_json([result]), encoding="utf-8")
if session is not None:
session.record_checkpoint(result)
- if result.success:
+ if result.success or (objective_verdict is not None and objective_verdict.evidence_conflict):
return result
failure_key = hashlib.sha256(
json.dumps(
diff --git a/src/re_agent/core/models.py b/src/re_agent/core/models.py
index 95095400..81c3f9a7 100644
--- a/src/re_agent/core/models.py
+++ b/src/re_agent/core/models.py
@@ -71,6 +71,7 @@ class ObjectiveVerdict:
verdict: Verdict
summary: str
findings: list[str] = field(default_factory=list)
+ evidence_conflict: bool = False
@dataclass
diff --git a/src/re_agent/verification/objective.py b/src/re_agent/verification/objective.py
index 092ad141..7eb51a10 100644
--- a/src/re_agent/verification/objective.py
+++ b/src/re_agent/verification/objective.py
@@ -31,6 +31,7 @@ def verify_candidate(
findings: list[str] = []
checks_run = 0
+ asm = None
try:
decompile = backend.decompile(target.address)
@@ -112,6 +113,25 @@ def integer(text: str) -> int:
if isinstance(pcode, list):
pcode = [item for item in pcode if isinstance(item, dict) and item.get("opcode")]
if isinstance(pcode, list) and pcode:
+ if asm is not None:
+ listed = set(re.findall(r"(?m)^\s*(?:0x)?([0-9a-fA-F]+)\s+", asm.instructions))
+ asm_addresses = {int(address, 16) for address in listed}
+ ir_addresses = {
+ int(str(item["address"]), 16) for item in pcode
+ if re.fullmatch(r"(?:0x)?[0-9a-fA-F]+", str(item.get("address", "")))
+ }
+ outside = ir_addresses - asm_addresses
+ if asm_addresses and outside:
+ return ObjectiveVerdict(
+ verdict=Verdict.FAIL,
+ summary="Structural evidence scopes differ; reconcile evidence before repairing code",
+ findings=[
+ f"P-code contains {len(outside)} instruction addresses absent from supplied assembly "
+ f"(first: {min(outside):x}). Expanded tail targets or incomplete exports may explain this; "
+ "cross-scope branch/call counts cannot establish a candidate mismatch."
+ ],
+ evidence_conflict=True,
+ )
checks_run += 1
opcodes = [str(item.get("opcode", "")).upper() for item in pcode if isinstance(item, dict)]
ir_calls = sum(op in {"CALL", "CALLIND"} for op in opcodes)
diff --git a/tests/test_verification/test_evidence_scope.py b/tests/test_verification/test_evidence_scope.py
new file mode 100644
index 00000000..a1257e8c
--- /dev/null
+++ b/tests/test_verification/test_evidence_scope.py
@@ -0,0 +1,48 @@
+"""Expanded or incomplete IR must not drive repeated candidate repairs."""
+import json
+
+from re_agent.agents.loop import run_fix_loop
+from re_agent.backend.protocol import BackendCapabilities
+from re_agent.backend.stub import StubBackend
+from re_agent.core.models import AnalysisArtifact, AsmResult, FunctionTarget, Verdict
+from re_agent.verification.objective import verify_candidate
+from tests.test_agents.test_loop import MockLLM
+
+
+class ScopeBackend(StubBackend):
+ def __init__(self, address="200"):
+ self.ir_address = address
+
+ @property
+ def capabilities(self):
+ return BackendCapabilities(has_asm=True, has_pcode=True)
+
+ def get_asm(self, address):
+ return AsmResult(address, "100 CALL RAX\n102 JMP RBX", 2, 1, False)
+
+ def get_pcode(self, address):
+ return AnalysisArtifact("pcode", address, json.dumps({"data": [
+ {"address": self.ir_address, "opcode": "CALLIND"}]}))
+
+
+def test_out_of_scope_ir_is_reported_as_evidence_conflict():
+ result = verify_candidate("void f() {}", FunctionTarget("100", "", "f"), ScopeBackend())
+ assert result.verdict == Verdict.FAIL
+ assert result.evidence_conflict
+ assert "200" in result.findings[0]
+ assert "candidate mismatch" in result.findings[0]
+
+
+def test_matching_ir_scope_does_not_block_candidate():
+ result = verify_candidate("void f() {}", FunctionTarget("100", "", "f"), ScopeBackend("0x100"))
+ assert not result.evidence_conflict
+
+
+def test_evidence_conflict_stops_repair_loop_without_accepting_candidate():
+ reverser = MockLLM(["```cpp\nvoid f() {}\n```"])
+ checker = MockLLM(['{"verdict":"PASS","summary":"Matches assembly"}'])
+ result = run_fix_loop(FunctionTarget("100", "", "f"), ScopeBackend(), reverser, checker, max_rounds=3)
+ assert not result.success
+ assert result.error == "Stopped: reconcile incompatible structural evidence before retrying"
+ assert result.rounds_used == 1
+ assert reverser._idx == checker._idx == 1
From b7a773960bff971e3bc02adad54403e7081b868e Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 08:52:23 +1000
Subject: [PATCH 11/34] Add Grok Build CLI provider with native session support
---
README.md | 17 ++++++
docs/grok-build.md | 25 +++++++++
src/re_agent/llm/grok_cli.py | 93 ++++++++++++++++++++++++++++++++
src/re_agent/llm/registry.py | 10 +++-
tests/test_llm/test_grok_cli.py | 95 +++++++++++++++++++++++++++++++++
5 files changed, 239 insertions(+), 1 deletion(-)
create mode 100644 docs/grok-build.md
create mode 100644 src/re_agent/llm/grok_cli.py
create mode 100644 tests/test_llm/test_grok_cli.py
diff --git a/README.md b/README.md
index a320bf63..862a87e1 100644
--- a/README.md
+++ b/README.md
@@ -245,6 +245,23 @@ Claude CLI supports real session resume and reports usage/cost metadata. A
stale CLI login can still require re-authentication even when its auth-status
command reports a session.
+### Grok Build CLI
+
+Authenticate with `grok login`, then configure:
+
+```yaml
+llm:
+ provider: grok-cli
+ model: "" # Use Grok Build's configured model, or specify a model ID.
+ cli_path: grok
+ timeout_s: 600
+```
+
+Supports native session resume, JSON response validation, usage metadata,
+and tool-free requests from an isolated working directory. Large evidence
+prompts use temporary files. See [Grok Build configuration](docs/grok-build.md)
+for supported settings and limits.
+
### OpenAI-compatible APIs
```yaml
diff --git a/docs/grok-build.md b/docs/grok-build.md
new file mode 100644
index 00000000..ce92bcef
--- /dev/null
+++ b/docs/grok-build.md
@@ -0,0 +1,25 @@
+# Grok Build CLI provider
+
+Install Grok Build and authenticate using its own `grok login` workflow. ReAgent uses the local CLI login; it does not read or store Grok credentials.
+
+```yaml
+llm:
+ provider: grok-cli
+ model: "" # Use the model configured in Grok Build, or supply a model from grok models.
+ cli_path: grok # An absolute path to grok.exe also works on Windows.
+ timeout_s: 600
+```
+
+Set `model` explicitly, including an empty string to use Grok's configured model. Omitting it inherits ReAgent's general default model, which is not a Grok model.
+
+The same configuration can be used under `agents.reverser` or `agents.checker` for mixed-provider runs. Optional `effort` is forwarded to Grok's `--effort` argument.
+
+The adapter uses `--prompt-file` for large evidence inputs, native session IDs for repair conversations, and native JSON output (`text`, `sessionId`, `stopReason`). It requires a completed `end_turn` response and rejects empty, malformed, errored, or incomplete results. Usage and cost fields are retained in call metadata when supplied by Grok.
+
+Requests run in a temporary working directory with built-in tools disabled, a deny-all permission rule, web tools disabled, and subagents disabled. ReAgent supplies evidence and controls compilation/validation. The CLI still uses its own global configuration and stores its own sessions. Temporary prompt files are removed after each request; workspace cleanup does not delete Grok's persistent sessions.
+
+The shared subprocess runner terminates the child process tree on timeout. ReAgent's `max_tokens`, `temperature`, and API-key settings are not forwarded to this CLI. `max_budget_usd` is rejected because the adapter cannot enforce it through the supported CLI flags. Configure account/model limits in Grok Build.
+
+Validated with Grok Build 1.0.25 on Windows, including a live two-turn session. Unit tests run without a Grok installation or network access.
+
+Official CLI documentation: https://docs.x.ai/build/cli/headless-scripting
diff --git a/src/re_agent/llm/grok_cli.py b/src/re_agent/llm/grok_cli.py
new file mode 100644
index 00000000..855592f6
--- /dev/null
+++ b/src/re_agent/llm/grok_cli.py
@@ -0,0 +1,93 @@
+"""Grok Build headless provider using the local CLI login."""
+from __future__ import annotations
+
+import json
+import subprocess
+import tempfile
+import uuid
+from pathlib import Path
+from typing import Any
+
+from re_agent.llm.protocol import Message
+from re_agent.utils.process import run_process
+
+
+class GrokCLIProvider:
+ """Run tool-free Grok requests in an isolated workspace, retaining sessions."""
+
+ def __init__(self, model: str = "", timeout_s: int = 1800,
+ grok_bin: str = "grok", effort: str | None = None) -> None:
+ self._model = model
+ self._timeout_s = timeout_s
+ self._grok_bin = grok_bin
+ self._effort = effort
+ self._workspace = tempfile.TemporaryDirectory(prefix="re-agent-grok-")
+ self._conversations: dict[str, tuple[str, bool]] = {}
+ self.last_metadata: dict[str, Any] = {}
+
+ @property
+ def supports_conversations(self) -> bool:
+ return True
+
+ def new_conversation(self, system: str) -> str:
+ cid = str(uuid.uuid4())
+ self._conversations[cid] = (system, False)
+ return cid
+
+ def resume(self, conversation_id: str, message: str) -> str:
+ if conversation_id not in self._conversations:
+ raise KeyError(f"Unknown conversation ID: {conversation_id}")
+ system, started = self._conversations[conversation_id]
+ text = self._run(message, system=system if not started else None,
+ session_id=conversation_id if not started else None,
+ resume=conversation_id if started else None)
+ self._conversations[conversation_id] = (system, True)
+ return text
+
+ def send(self, messages: list[Message], **kwargs: Any) -> str:
+ system = "\n\n".join(m.content for m in messages if m.role == "system")
+ prompt = "\n\n".join(f"[{m.role.upper()}]\n{m.content}" for m in messages if m.role != "system")
+ return self._run(prompt, system=system or None, model=kwargs.get("model"))
+
+ def _run(self, prompt: str, *, system: str | None = None, session_id: str | None = None,
+ resume: str | None = None, model: str | None = None) -> str:
+ self.last_metadata = {}
+ path = Path(self._workspace.name) / f"prompt-{uuid.uuid4().hex}.txt"
+ path.write_text(prompt, encoding="utf-8")
+ command = [self._grok_bin, "--prompt-file", str(path), "--output-format", "json",
+ "--tools", "", "--deny", "*", "--no-subagents", "--disable-web-search",
+ "--max-turns", "1"]
+ for flag, value in [("--model", model if model is not None else self._model),
+ ("--system-prompt-override", system), ("--session-id", session_id),
+ ("--resume", resume), ("--effort", self._effort)]:
+ if value:
+ command.extend([flag, value])
+ try:
+ proc = run_process(command, cwd=self._workspace.name, timeout_s=self._timeout_s)
+ if proc.returncode != 0:
+ detail = (proc.stderr.strip() or proc.stdout.strip())[-2000:]
+ raise RuntimeError(f"Grok CLI failed with exit code {proc.returncode}: {detail}")
+ try:
+ payload = json.loads(proc.stdout)
+ except json.JSONDecodeError as exc:
+ raise RuntimeError("Grok CLI returned invalid JSON") from exc
+ if not isinstance(payload, dict):
+ raise RuntimeError("Grok CLI returned an unexpected JSON payload")
+ self.last_metadata = {key: payload[key] for key in
+ ("sessionId", "requestId", "usage", "modelUsage", "total_cost_usd", "stopReason")
+ if key in payload}
+ if payload.get("error") or payload.get("is_error") or payload.get("stopReason") != "end_turn":
+ raise RuntimeError(f"Grok CLI did not complete a text response: {payload.get('stopReason', 'error')}")
+ text = payload.get("text")
+ if not isinstance(text, str) or not text.strip():
+ raise RuntimeError("Grok CLI JSON payload has no text response")
+ expected = session_id or resume
+ if expected and payload.get("sessionId") != expected:
+ raise RuntimeError("Grok CLI returned a different session ID")
+ return text
+ except subprocess.TimeoutExpired as exc:
+ raise RuntimeError(f"Grok CLI timed out after {self._timeout_s}s") from exc
+ except FileNotFoundError as exc:
+ raise RuntimeError(f"Grok CLI not found: {self._grok_bin}") from exc
+ finally:
+ path.unlink(missing_ok=True)
diff --git a/src/re_agent/llm/registry.py b/src/re_agent/llm/registry.py
index 6c6bbb4f..e2cdcb58 100644
--- a/src/re_agent/llm/registry.py
+++ b/src/re_agent/llm/registry.py
@@ -53,6 +53,14 @@ def create_provider(config: LLMConfig) -> LLMProvider:
base_url=config.base_url,
)
+ if config.provider == "grok-cli":
+ from re_agent.llm.grok_cli import GrokCLIProvider
+
+ if config.max_budget_usd is not None:
+ raise ValueError("grok-cli does not support max_budget_usd; configure limits in Grok Build")
+ return GrokCLIProvider(model=config.model, timeout_s=config.timeout_s,
+ grok_bin=config.cli_path or "grok", effort=config.effort)
+
if config.provider == "codex":
from re_agent.llm.codex_cli import CodexCLIProvider
@@ -65,5 +73,5 @@ def create_provider(config: LLMConfig) -> LLMProvider:
raise ValueError(
f"Unknown LLM provider: {config.provider!r}. "
f"Supported providers: 'claude', 'claude-cli', 'openai', "
- f"'openai-compat', 'codex'."
+ f"'openai-compat', 'codex', 'grok-cli'."
)
diff --git a/tests/test_llm/test_grok_cli.py b/tests/test_llm/test_grok_cli.py
new file mode 100644
index 00000000..448c2b7e
--- /dev/null
+++ b/tests/test_llm/test_grok_cli.py
@@ -0,0 +1,95 @@
+"""Grok Build CLI contract, based on native headless JSON responses."""
+import json
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from re_agent.config.schema import LLMConfig
+from re_agent.llm.grok_cli import GrokCLIProvider
+from re_agent.llm.protocol import Message
+from re_agent.llm.registry import create_provider
+
+
+def test_native_sessions_prompt_files_and_tool_free_arguments(monkeypatch):
+ calls = []
+
+ def fake(command, **kwargs):
+ path = Path(command[command.index("--prompt-file") + 1])
+ calls.append((command, kwargs, path, path.read_text(encoding="utf-8")))
+ flag = "--resume" if "--resume" in command else "--session-id"
+ cid = command[command.index(flag) + 1]
+ return subprocess.CompletedProcess(command, 0, json.dumps({
+ "text": "int f() { return 1; }", "stopReason": "end_turn", "sessionId": cid,
+ "usage": {"input_tokens": 12}, "total_cost_usd": 0.01}), "")
+
+ monkeypatch.setattr("re_agent.llm.grok_cli.run_process", fake)
+ provider = create_provider(LLMConfig(provider="grok-cli", model="example-model", cli_path="grok path",
+ effort="high", timeout_s=17))
+ cid = provider.new_conversation("Preserve behavior")
+ prompt = "Evidence π $() & " * 10000
+ assert "int f()" in provider.resume(cid, prompt)
+ provider.resume(cid, "Repair")
+ first, second = calls
+ assert first[3] == prompt
+ assert prompt not in first[0]
+ assert first[0][0] == "grok path"
+ assert first[0][first[0].index("--tools") + 1] == ""
+ assert first[0][first[0].index("--deny") + 1] == "*"
+ assert "--no-subagents" in first[0] and "--disable-web-search" in first[0]
+ assert first[0][first[0].index("--max-turns") + 1] == "1"
+ assert first[0][first[0].index("--system-prompt-override") + 1] == "Preserve behavior"
+ assert "--session-id" in first[0] and "--resume" not in first[0]
+ assert "--resume" in second[0] and "--session-id" not in second[0]
+ assert first[1]["timeout_s"] == 17
+ assert first[1]["cwd"] == second[1]["cwd"] != str(Path.cwd())
+ assert all(not call[2].exists() for call in calls)
+ assert provider.last_metadata["usage"] == {"input_tokens": 12}
+
+
+@pytest.mark.parametrize("payload", [[], {}, {"text": "", "stopReason": "end_turn"},
+ {"text": "partial", "stopReason": "max_turns"},
+ {"text": "bad", "stopReason": "end_turn", "is_error": True},
+ {"text": "ok", "stopReason": "end_turn", "sessionId": "wrong-session"}])
+def test_invalid_or_incomplete_responses_do_not_start_conversation(monkeypatch, payload):
+ paths = []
+
+ def fake(command, **kwargs):
+ paths.append(Path(command[2]))
+ return subprocess.CompletedProcess(command, 0, json.dumps(payload), "")
+
+ monkeypatch.setattr("re_agent.llm.grok_cli.run_process", fake)
+ provider = GrokCLIProvider()
+ cid = provider.new_conversation("system")
+ with pytest.raises(RuntimeError):
+ provider.resume(cid, "prompt")
+ assert provider._conversations[cid] == ("system", False)
+ assert not paths[0].exists()
+
+
+@pytest.mark.parametrize("failure, message", [
+ (FileNotFoundError(), "not found"),
+ (subprocess.TimeoutExpired("grok", 2), "timed out"),
+ (subprocess.CompletedProcess([], 1, "", "login required"), "login required"),
+ (subprocess.CompletedProcess([], 0, "not JSON", ""), "invalid JSON"),
+])
+def test_process_errors_and_prompt_cleanup(monkeypatch, failure, message):
+ paths = []
+
+ def fake(command, **kwargs):
+ paths.append(Path(command[2]))
+ if isinstance(failure, Exception):
+ raise failure
+ return failure
+
+ monkeypatch.setattr("re_agent.llm.grok_cli.run_process", fake)
+ with pytest.raises(RuntimeError, match=message):
+ GrokCLIProvider().send([Message("system", "rules"), Message("user", "prompt")])
+ assert not paths[0].exists()
+
+
+def test_unknown_conversation_and_unsupported_budget():
+ with pytest.raises(KeyError):
+ GrokCLIProvider().resume("missing", "prompt")
+ with pytest.raises(ValueError, match="max_budget_usd"):
+ create_provider(LLMConfig(provider="grok-cli", max_budget_usd=1))
From 7faceefa94320333dd9095ef684cd243e5642b6d Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 09:12:08 +1000
Subject: [PATCH 12/34] Document provider-independent parallel function
execution plan
---
docs/parallel-functions-plan.md | 216 ++++++++++++++++++++++++++++++++
1 file changed, 216 insertions(+)
create mode 100644 docs/parallel-functions-plan.md
diff --git a/docs/parallel-functions-plan.md b/docs/parallel-functions-plan.md
new file mode 100644
index 00000000..c0872c97
--- /dev/null
+++ b/docs/parallel-functions-plan.md
@@ -0,0 +1,216 @@
+# Bounded parallel function processing
+
+Status: implementation plan; no parallel execution is enabled by this document.
+
+## Objective
+
+Reduce elapsed time for independent function reconstruction while preserving the
+same acceptance gates, per-function model budgets, checkpoints, and stop controls.
+Scheduling belongs to ReAgent and must work with all providers registered through
+`llm/registry.py`, including API providers and Claude, Codex, and Grok CLI providers.
+This is separate from a provider's internal subagent or workflow features.
+
+## Existing integration points
+
+- `orchestrator/class_runner.py` calls `pick_next` and `reverse_single` sequentially.
+ Successful cumulative candidates are promoted into one scratch project and the
+ source index is rebuilt before subsequent work.
+- `orchestrator/batch_runner.py` adapts manifests to the class runner. Both entry
+ points should use one scheduler rather than separate parallel implementations.
+- `orchestrator/single.py` and `agents/loop.py` own the existing function pipeline,
+ validation, repair rounds, logs, checkpoints, and final result recording.
+- Provider instances hold conversation state and mutable `last_metadata`. They
+ must not be shared by concurrently running functions.
+- `core/session.py` protects individual writes with file locks, but selecting a
+ target and claiming it are not one transaction. Locks alone do not prevent two
+ workers from selecting the same function or reading stale in-memory state.
+- `core/function_picker.py` implements selection and dependency ordering. Reuse
+ its ranking semantics; do not repeatedly traverse the graph for each free slot.
+- `monitor/server.py` reads saved results and controls an owned worker tree. It
+ currently has no per-function in-flight view.
+
+## Initial contract
+
+Proposed configuration, to be implemented and validated in stage 1:
+
+```yaml
+orchestrator:
+ max_parallel_functions: 1
+ max_parallel_validations: 1
+```
+
+Both are positive integers. Initial supported function concurrency is 1 through
+32, with a default of 1; this is a local scheduling limit, not a claim about any
+provider's entitlement or throughput. Validation concurrency cannot exceed
+function concurrency. The first parallel release permits validation concurrency
+of 1 only; higher values require stage 6's isolation checks.
+
+Keep the current sequential path and public APIs working unchanged at concurrency
+1. Existing callers that supply already-created providers may use that path.
+Parallel callers must supply factories/configuration: never clone or share an
+unknown live provider instance. Reject incompatible settings before any model call.
+
+The first release supports independent draft reconstruction against a fixed input
+snapshot. Reject parallel execution when both `validation.copy_project` and
+`orchestrator.cumulative_validation` are true. Do not silently turn off cumulative
+validation, weaken a gate, or promote speculative results. Stage 8 extends this.
+
+## Scheduler and ownership
+
+Use a bounded thread executor for orchestration: most work waits on model requests
+and subprocesses. Threads avoid requiring every backend, provider, and test double
+to be serializable. Cancellation of a future does not stop an executing thread;
+explicit request/subprocess cancellation is a separate requirement below.
+
+One coordinator owns target enumeration, normalized-address claims, attempt
+accounting, final result publication, and durable session writes. Submit at most
+the available worker slots instead of queuing the entire inventory into futures.
+Never have two active attempts for the same normalized address.
+
+Each job owns fresh reverser/checker provider instances, conversation IDs, call
+budget, configuration copy, source-index state, backend view, and artifact paths.
+Use `//` isolation, including compiler working directories
+and mutable parity caches. Audit lazy caches in indexers/backends rather than
+assuming read methods are immutable. Provider resources must be closed after a
+job on success, exception, or cancellation.
+
+Share immutable exported evidence where possible. Initially serialize calls to
+live/stateful backends through a coordinator-owned adapter. Allow independent
+backend instances only after their transport/session isolation is tested. Never
+issue concurrent operations against a shared Ghidra analysis session by default.
+
+Worker checkpoints are typed events sent to the coordinator. A worker waits for
+durable acknowledgement before claiming a checkpoint is saved. Prior feedback is
+an immutable per-target snapshot. Workers do not mutate a shared `Session` object.
+Keep the existing on-disk session result format readable by existing tools.
+
+Acquire an exclusive coordinator lease for the run/session identity. A second
+process targeting the same session must fail before dispatch. Release it on clean
+exit; recover after process death using OS locking and recorded ownership, not a
+stale PID alone. Use attempt IDs to ignore duplicate/stale completion events.
+
+Return results in deterministic planned order, while publishing progress as jobs
+finish. This stabilizes reports without blocking execution on an early slow job.
+
+## Selection and dependency semantics
+
+Prepare a normalized, deduplicated candidate inventory once. Apply completion and
+attempt filters from the bound session and retain current strategy tie-breakers.
+Refresh eligibility after each terminal attempt; never exceed the invocation's
+function-attempt limit when retries are admitted.
+
+For `dependency-order`, derive strongly connected components and schedule ready
+components callee-first. Process targets within a cycle deterministically and
+sequentially; independent ready components can run concurrently. External or
+unselected callees are evidence dependencies, not jobs to invent automatically.
+
+In fixed-snapshot draft mode, this order does not make a callee's generated source
+available to its caller. A terminal failed callee does not automatically reject a
+caller; record the outcome and let existing evidence/validation gates decide.
+Document this distinction from cumulative source promotion.
+
+## Cancellation, failures, and recovery
+
+Stop closes dispatch immediately, marks queued targets unstarted, and requests
+cancellation of active jobs. A shared cancellation token is checked before model
+calls, repairs, validation commands, and result publication. Waiting on a semaphore
+or backend lock must also be cancellable.
+
+Extend the shared process runner with explicit cancellation and an owned-process
+registry. Migrate CLI providers still using direct `subprocess.run` to this path.
+Terminate child trees and reap processes on timeout/cancellation. POSIX subprocesses
+may create their own sessions: killing only the coordinator's process group is
+insufficient. Cover detached descendants on POSIX and process trees on Windows.
+Keep the monitor's hard-stop fallback for an unresponsive coordinator.
+
+For synchronous API requests that cannot be interrupted, stop dispatch immediately
+and wait no longer than the configured request timeout. Discard late results from
+cancelled attempts. Report `stopping` until cleanup finishes; never claim an
+in-flight remote request was revoked if the provider cannot do that.
+
+Persist `queued`, `running`, `completed`, `failed`, and `interrupted` attempt states
+separately from acceptance verdicts. Interrupted jobs retain saved round feedback
+and can resume without being mistaken for successful results. Keep a record of
+calls already spent; resumed work must not reset the same attempt's call budget.
+A subsequent deliberate attempt follows the existing attempt-limit policy.
+
+One ordinary candidate rejection must not cancel independent jobs. Fatal shared
+failures (authentication/configuration failure or inaccessible evidence source)
+stop dispatch and record a run-level reason. Introduce typed failure categories;
+do not infer authentication or rate limits from arbitrary diagnostic substrings.
+Do not add an unbounded retry loop. Provider-specific rate-limit handling must
+respect existing budgets and cancellation.
+
+Separate scheduling settings from semantic acceptance identity: changing worker
+count alone should not invalidate completed results. Evidence, source, model, and
+acceptance-policy changes must continue to invalidate incompatible results.
+
+## Live reporting
+
+Add an atomically written, versioned execution-status file owned by the coordinator.
+Keep final session results authoritative for completed/accepted counts and use the
+status file for in-flight state. The monitor remains a decoupled reader.
+
+Report the run/attempt IDs, concurrency limit, active target addresses, current
+stage, elapsed times, queue count, compiler waits, model waits, stop state, and
+completed/failed/interrupted counts. Reconcile old worker-stage data with live
+process ownership so a killed worker cannot remain displayed as reconstructing.
+Show review passes and validation outcomes separately.
+
+Attach progress events and logs to job IDs; do not interleave unlabelled model
+output. Preserve the existing single-worker display for older status/session files.
+
+## Staged commits and required tests
+
+Each stage is a separate implementation commit, with its own regression tests.
+Keep intermediate stages disabled until their required infrastructure is present.
+
+1. **Define parallel execution configuration and job contracts.** Add validated
+ options, immutable job/event types, and provider/backend factory boundaries.
+ Test YAML/per-role loading, invalid limits, unsupported cumulative mode, and
+ exact compatibility of the default sequential path.
+2. **Isolate function job resources.** Reuse `reverse_single` through a worker
+ context and coordinator checkpoint sink. Test overlapping jobs with barriers:
+ distinct conversations, metadata, budgets, artifacts, caches, and compiler
+ working directories; cleanup on failures. Cover every registered provider with
+ fake transports, without paid or network requests in CI.
+3. **Add bounded scheduling and deterministic dependency selection.** Test active
+ worker caps, duplicate address normalization, diamond graphs, cycles, external
+ callees, retry/attempt limits, out-of-order completion, and stable output order.
+ Use events/barriers rather than brittle wall-clock speed assertions.
+4. **Make parallel checkpoints and resume durable.** Test simultaneous checkpoints,
+ duplicate completion events, coordinator crashes, lease recovery, competing
+ coordinators, preserved budgets, and worker-count-only configuration changes.
+5. **Implement cancellation across workers and providers.** Test stop while queued,
+ waiting for a lock, generating, compiling, and publishing. Use disposable real
+ process trees on Windows/Linux/macOS, including POSIX detached descendants.
+ Test late API replies and ensure Stop cannot promote or count cancelled results.
+6. **Control validation concurrency.** Keep one shared validation lane by default.
+ Opt-in parallel validation requires isolated writable roots and an explicit
+ concurrency-safe command contract; isolated paths alone do not make external
+ services or reference harnesses safe. Test common build filenames, shared
+ reference commands, resource limits, and unchanged acceptance verdicts.
+7. **Expose the scheduler through CLI and monitor.** Wire both manifest and class
+ entry points. Test full fake-provider runs at 1/2/4 workers, status transitions,
+ monitor reconnect, cancellation controls, and backwards-compatible sessions.
+ Enable parallel mode only once stages 1–6 are integrated and passing.
+8. **Support cumulative validation separately.** Use immutable source generations
+ for parallel proposals and deterministic serial promotion. Revalidate each
+ proposal against the latest promoted generation; reject stale candidates or
+ return them to a bounded repair step. Test two candidates editing one source
+ file, dependent promotions, validation failure, cycles, and crash recovery.
+ Do not publish a proposal's provisional PASS before promotion/revalidation.
+
+## Measurement and release criteria
+
+Benchmark the same pinned evidence and target set at 1, 2, and 4 workers, using
+fresh isolated outputs and the same models, prompts, budgets, and acceptance
+rules. Compare wall time, summed model time, calls, compiler time, memory, rate
+limits, interruptions, and acceptance results. Repeat trials to distinguish
+provider variation from scheduler effects. Keep benchmark reports out of Git.
+
+Release the independent parallel mode after cross-platform tests establish no
+duplicate jobs, lost checkpoints, metadata cross-talk, orphaned local workers,
+or acceptance changes caused by scheduling. Report measured throughput honestly;
+do not promise linear scaling or enable 32 workers by default. Grok-managed
+subagents and workflows remain a separate future design.
From 793495f380937dccba6c742888e1ead0cb34784d Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 09:18:01 +1000
Subject: [PATCH 13/34] Define bounded parallel execution and validation
settings
---
src/re_agent/config/loader.py | 8 ++++++++
src/re_agent/config/schema.py | 3 +++
src/re_agent/core/identity.py | 1 +
tests/test_config/test_parallel.py | 28 ++++++++++++++++++++++++++++
4 files changed, 40 insertions(+)
create mode 100644 tests/test_config/test_parallel.py
diff --git a/src/re_agent/config/loader.py b/src/re_agent/config/loader.py
index 3bb55cee..a619442d 100644
--- a/src/re_agent/config/loader.py
+++ b/src/re_agent/config/loader.py
@@ -240,6 +240,14 @@ def load_config(
def validate_config(config: ReAgentConfig) -> None:
+ workers = config.orchestrator.max_parallel_functions
+ validations = config.orchestrator.max_parallel_validations
+ if type(workers) is not int or not 1 <= workers <= 32:
+ raise ValueError("max_parallel_functions must be an integer from 1 to 32")
+ if type(validations) is not int or not 1 <= validations <= workers:
+ raise ValueError("max_parallel_validations must be between 1 and max_parallel_functions")
+ if validations > 1 and not (config.validation.parallel_safe and config.validation.copy_project):
+ raise ValueError("Parallel validation requires copy_project and an explicit parallel_safe command contract")
for name in (
"max_review_rounds",
"max_functions_per_class",
diff --git a/src/re_agent/config/schema.py b/src/re_agent/config/schema.py
index 301a3b9c..5e7c18b2 100644
--- a/src/re_agent/config/schema.py
+++ b/src/re_agent/config/schema.py
@@ -117,6 +117,8 @@ class OrchestratorConfig:
max_investigations: int = 8
selection_strategy: str = "dependency-order"
max_attempts_per_function: int = 3
+ max_parallel_functions: int = 1
+ max_parallel_validations: int = 1
@dataclass
@@ -142,6 +144,7 @@ class ValidationConfig:
command_timeout_s: int = 900
working_directory: str = "."
keep_project_copy: bool = False
+ parallel_safe: bool = False
@dataclass
diff --git a/src/re_agent/core/identity.py b/src/re_agent/core/identity.py
index 56eaaac4..36a63857 100644
--- a/src/re_agent/core/identity.py
+++ b/src/re_agent/core/identity.py
@@ -19,6 +19,7 @@ def project_fingerprint(config: ReAgentConfig) -> str:
"parity": asdict(config.parity),
"backend": asdict(config.backend),
}
+ values["validation"].pop("parallel_safe", None)
digest.update(json.dumps(values, sort_keys=True).encode())
root = Path(config.project_profile.source_root).resolve()
digest.update(str(root).encode())
diff --git a/tests/test_config/test_parallel.py b/tests/test_config/test_parallel.py
new file mode 100644
index 00000000..46c77ab9
--- /dev/null
+++ b/tests/test_config/test_parallel.py
@@ -0,0 +1,28 @@
+import pytest
+
+from re_agent.config.loader import validate_config
+from re_agent.config.schema import ReAgentConfig
+from re_agent.core.identity import project_fingerprint
+
+
+@pytest.mark.parametrize("value", [0, -1, 33, True, 1.5])
+def test_parallel_limits(value):
+ config = ReAgentConfig()
+ config.orchestrator.max_parallel_functions = value
+ with pytest.raises(ValueError, match="max_parallel_functions"):
+ validate_config(config)
+
+
+def test_validation_contract_and_identity(tmp_path):
+ c = ReAgentConfig()
+ c.project_profile.source_root = str(tmp_path)
+ original = project_fingerprint(c)
+ c.orchestrator.max_parallel_functions = 4
+ c.orchestrator.max_parallel_validations = 2
+ with pytest.raises(ValueError, match="parallel_safe"):
+ validate_config(c)
+ c.validation.parallel_safe = True
+ c.validation.copy_project = True
+ validate_config(c)
+ c.validation.copy_project = False
+ assert project_fingerprint(c) == original
From 8c70c36981d07f7d6b698578bb7f51734594f7e8 Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 09:19:18 +1000
Subject: [PATCH 14/34] Add isolated execution controls and cancellable
validation waits
---
src/re_agent/llm/observed.py | 8 ++++
src/re_agent/orchestrator/execution.py | 65 ++++++++++++++++++++++++++
src/re_agent/orchestrator/single.py | 12 +++++
src/re_agent/utils/process.py | 34 +++++++++++++-
tests/test_execution.py | 36 ++++++++++++++
5 files changed, 154 insertions(+), 1 deletion(-)
create mode 100644 src/re_agent/orchestrator/execution.py
create mode 100644 tests/test_execution.py
diff --git a/src/re_agent/llm/observed.py b/src/re_agent/llm/observed.py
index a5f7e521..f8c5df9b 100644
--- a/src/re_agent/llm/observed.py
+++ b/src/re_agent/llm/observed.py
@@ -48,6 +48,12 @@ def resume(self, conversation_id: str, message: str) -> str:
return self._call(conversation_id=conversation_id, message=message)
def _call(self, **request: Any) -> str:
+ from re_agent.orchestrator.execution import current, progress
+
+ progress(self.role)
+ context = current()
+ if context:
+ context.emit("call", self.role)
number = self.budget.consume()
start = time.monotonic()
event: dict[str, Any] = {"role": self.role, "call": number, "request": request}
@@ -57,6 +63,8 @@ def _call(self, **request: Any) -> str:
else:
response = self.provider.resume(request["conversation_id"], request["message"])
event["response"] = response
+ if context:
+ context.check()
return response
except Exception as exc:
event["error"] = str(exc)
diff --git a/src/re_agent/orchestrator/execution.py b/src/re_agent/orchestrator/execution.py
new file mode 100644
index 00000000..f756335d
--- /dev/null
+++ b/src/re_agent/orchestrator/execution.py
@@ -0,0 +1,65 @@
+"""Thread-local execution controls shared by providers and validation gates."""
+from __future__ import annotations
+
+import threading
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass
+from typing import Any
+
+
+class Cancelled(RuntimeError):
+ pass
+
+
+@dataclass
+class Execution:
+ cancel: threading.Event
+ validations: threading.Semaphore
+ emit: Callable[[str, Any], None]
+
+ def check(self) -> None:
+ if self.cancel.is_set():
+ raise Cancelled("Execution cancelled")
+
+
+_local = threading.local()
+
+
+def current() -> Execution | None:
+ return getattr(_local, "execution", None)
+
+
+@contextmanager
+def executing(context: Execution) -> Iterator[None]:
+ previous = current()
+ _local.execution = context
+ try:
+ context.check()
+ yield
+ finally:
+ _local.execution = previous
+
+
+def progress(stage: str) -> None:
+ context = current()
+ if context:
+ context.check()
+ context.emit("stage", stage)
+
+
+@contextmanager
+def validation_lane() -> Iterator[None]:
+ context = current()
+ if context is None:
+ yield
+ return
+ progress("waiting-for-validation")
+ while not context.validations.acquire(timeout=.1):
+ context.check()
+ try:
+ progress("validating")
+ yield
+ context.check()
+ finally:
+ context.validations.release()
diff --git a/src/re_agent/orchestrator/single.py b/src/re_agent/orchestrator/single.py
index 71e20e6f..a8236ad5 100644
--- a/src/re_agent/orchestrator/single.py
+++ b/src/re_agent/orchestrator/single.py
@@ -122,6 +122,18 @@ def validate_result(
config: ReAgentConfig,
backend: REBackend,
indexer: SourceIndexer | None = None,
+) -> ReversalResult:
+ from re_agent.orchestrator.execution import validation_lane
+
+ with validation_lane():
+ return _validate_result(result, config, backend, indexer)
+
+
+def _validate_result(
+ result: ReversalResult,
+ config: ReAgentConfig,
+ backend: REBackend,
+ indexer: SourceIndexer | None = None,
) -> ReversalResult:
"""Validate one round and return its candidate-level diagnostics."""
target = result.target
diff --git a/src/re_agent/utils/process.py b/src/re_agent/utils/process.py
index c6505b4f..4d93fb84 100644
--- a/src/re_agent/utils/process.py
+++ b/src/re_agent/utils/process.py
@@ -83,8 +83,40 @@ def run_process(
start_new_session=os.name != "nt",
)
try:
- proc.communicate(input_text, timeout=timeout_s)
+ import time
+
+ from re_agent.orchestrator.execution import current
+
+ context = current()
+ deadline = time.monotonic() + timeout_s
+ first = True
+ while True:
+ if context:
+ context.check()
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ raise subprocess.TimeoutExpired(args, timeout_s)
+ try:
+ proc.communicate(input_text if first else None,
+ timeout=min(.1, remaining) if context else remaining)
+ break
+ except subprocess.TimeoutExpired:
+ first = False
+ if not context:
+ raise
except BaseException:
+ # Capture descendants before killing their parent, including children
+ # that have detached into a separate POSIX session.
+ try:
+ import psutil
+
+ with contextlib.suppress(psutil.Error):
+ descendants = psutil.Process(proc.pid).children(recursive=True)
+ for child in reversed(descendants):
+ with contextlib.suppress(psutil.Error):
+ child.kill()
+ except ImportError:
+ pass
if os.name == "nt":
subprocess.run(["taskkill", "/PID", str(proc.pid), "/T", "/F"], capture_output=True, check=False)
else:
diff --git a/tests/test_execution.py b/tests/test_execution.py
new file mode 100644
index 00000000..04374b73
--- /dev/null
+++ b/tests/test_execution.py
@@ -0,0 +1,36 @@
+import sys
+import threading
+from concurrent.futures import ThreadPoolExecutor
+
+import pytest
+
+from re_agent.orchestrator.execution import Cancelled, Execution, executing, validation_lane
+from re_agent.utils.process import run_process
+
+
+def test_cancelled_process_and_context_cleanup():
+ cancel = threading.Event()
+ context = Execution(cancel, threading.Semaphore(1), lambda *_: None)
+ cancel.set()
+ with pytest.raises(Cancelled), executing(context):
+ pass
+ # An unrelated call outside the scope is not cancelled.
+ assert run_process([sys.executable, "-c", "print('ok')"]).stdout.strip() == "ok"
+
+
+def test_validation_wait_is_cancellable():
+ cancel = threading.Event()
+ waiting = threading.Event()
+ semaphore = threading.Semaphore(0)
+ context = Execution(cancel, semaphore, lambda *_: waiting.set())
+
+ def worker():
+ with executing(context), validation_lane():
+ pytest.fail("Validation must not run")
+
+ with ThreadPoolExecutor(1) as executor:
+ future = executor.submit(worker)
+ assert waiting.wait(5)
+ cancel.set()
+ with pytest.raises(Cancelled):
+ future.result(timeout=5)
From b4704879319b82a70c694b5c9f7d46b6756ca9fa Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 09:20:13 +1000
Subject: [PATCH 15/34] Plan deterministic parallel dependencies and exclusive
coordinator leases
---
src/re_agent/orchestrator/dependencies.py | 47 +++++++++++++++++++++++
src/re_agent/utils/storage.py | 6 +--
tests/test_dependencies.py | 19 +++++++++
3 files changed, 69 insertions(+), 3 deletions(-)
create mode 100644 src/re_agent/orchestrator/dependencies.py
create mode 100644 tests/test_dependencies.py
diff --git a/src/re_agent/orchestrator/dependencies.py b/src/re_agent/orchestrator/dependencies.py
new file mode 100644
index 00000000..c7df4fb7
--- /dev/null
+++ b/src/re_agent/orchestrator/dependencies.py
@@ -0,0 +1,47 @@
+"""Deterministic prerequisite chains with iterative strongly connected components."""
+from __future__ import annotations
+
+
+def prerequisites(order: list[str], edges: dict[str, set[str]]) -> dict[str, set[str]]:
+ """Collapse cycles, then serialize each cycle while allowing independent SCCs."""
+ nodes = set(order)
+ edges = {n: edges.get(n, set()) & nodes for n in order}
+ seen: set[str] = set()
+ finish: list[str] = []
+ for root in order:
+ stack = [(root, False)]
+ while stack:
+ node, exiting = stack.pop()
+ if exiting:
+ finish.append(node)
+ elif node not in seen:
+ seen.add(node)
+ stack.append((node, True))
+ stack.extend((child, False) for child in sorted(edges[node], reverse=True) if child not in seen)
+ reverse: dict[str, set[str]] = {n: set() for n in order}
+ for node, children in edges.items():
+ for child in children:
+ reverse[child].add(node)
+ groups: list[list[str]] = []
+ membership: dict[str, int] = {}
+ rank = {n: i for i, n in enumerate(order)}
+ for root in reversed(finish):
+ if root in membership:
+ continue
+ group: list[str] = []
+ todo = [root]
+ while todo:
+ node = todo.pop()
+ if node in membership:
+ continue
+ membership[node] = len(groups)
+ group.append(node)
+ todo.extend(reverse[node])
+ groups.append(sorted(group, key=rank.__getitem__))
+ result: dict[str, set[str]] = {}
+ for group in groups:
+ required = {groups[membership[child]][-1] for node in group for child in edges[node]
+ if membership[child] != membership[node]}
+ for index, node in enumerate(group):
+ result[node] = required if index == 0 else {group[index - 1]}
+ return result
diff --git a/src/re_agent/utils/storage.py b/src/re_agent/utils/storage.py
index d984d1b1..9c4b042e 100644
--- a/src/re_agent/utils/storage.py
+++ b/src/re_agent/utils/storage.py
@@ -25,7 +25,7 @@ def atomic_json(path: Path, data: Any) -> None:
@contextmanager
-def file_lock(path: Path) -> Iterator[None]:
+def file_lock(path: Path, *, blocking: bool = True) -> Iterator[None]:
path.parent.mkdir(parents=True, exist_ok=True)
with path.with_suffix(path.suffix + ".lock").open("a+b") as handle:
if os.name == "nt":
@@ -35,11 +35,11 @@ def file_lock(path: Path) -> Iterator[None]:
handle.write(b"0")
handle.flush()
handle.seek(0)
- msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1) # type: ignore[attr-defined]
+ msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK if blocking else msvcrt.LK_NBLCK, 1) # type: ignore[attr-defined]
else:
import fcntl
- fcntl.flock(handle, fcntl.LOCK_EX)
+ fcntl.flock(handle, fcntl.LOCK_EX | (0 if blocking else fcntl.LOCK_NB))
try:
yield
finally:
diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py
new file mode 100644
index 00000000..55f3f02c
--- /dev/null
+++ b/tests/test_dependencies.py
@@ -0,0 +1,19 @@
+from re_agent.orchestrator.dependencies import prerequisites
+
+
+def test_diamond_and_external_dependencies():
+ result = prerequisites(["a", "b", "c", "d"], {
+ "a": {"b", "c"}, "b": {"d"}, "c": {"d", "external"}})
+ assert result == {"a": {"b", "c"}, "b": {"d"}, "c": {"d"}, "d": set()}
+
+
+def test_cycle_has_deterministic_serial_order():
+ result = prerequisites(["a", "b", "c"], {"a": {"b"}, "b": {"a", "c"}})
+ assert result == {"a": {"c"}, "b": {"a"}, "c": set()}
+
+
+def test_deep_graph_does_not_use_python_recursion():
+ order = [str(n) for n in range(5000)]
+ result = prerequisites(order, {str(n): {str(n+1)} for n in range(4999)})
+ assert result["0"] == {"1"}
+ assert result["4999"] == set()
From d647f6685f2770a161e8d56027909be62f5067c8 Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 09:27:39 +1000
Subject: [PATCH 16/34] Execute bounded isolated jobs with durable resumable
checkpoints
---
src/re_agent/core/session.py | 7 +-
src/re_agent/llm/observed.py | 2 +-
src/re_agent/orchestrator/parallel.py | 324 ++++++++++++++++++++++++++
src/re_agent/orchestrator/snapshot.py | 35 +++
tests/test_parallel.py | 126 ++++++++++
5 files changed, 492 insertions(+), 2 deletions(-)
create mode 100644 src/re_agent/orchestrator/parallel.py
create mode 100644 src/re_agent/orchestrator/snapshot.py
create mode 100644 tests/test_parallel.py
diff --git a/src/re_agent/core/session.py b/src/re_agent/core/session.py
index eca67969..e24f339c 100644
--- a/src/re_agent/core/session.py
+++ b/src/re_agent/core/session.py
@@ -62,7 +62,10 @@ def previous_feedback(self, address: str) -> str:
entry = self._data.get("checkpoints", {}).get(normalize_address(address))
return json.dumps(entry, indent=2) if entry else ""
- def record_result(self, result: ReversalResult) -> None:
+ def record_result_once(self, result: ReversalResult) -> None:
+ self.record_result(result, idempotent=True)
+
+ def record_result(self, result: ReversalResult, *, idempotent: bool = False) -> None:
addr = normalize_address(result.target.address)
entry = {
"address": result.target.address,
@@ -87,6 +90,8 @@ def record_result(self, result: ReversalResult) -> None:
with file_lock(self.path):
if self.path.exists():
self.load()
+ if idempotent and result.run_id and any(r.get("run_id") == result.run_id for r in self._data["runs"]):
+ return
self._data["functions"][addr] = entry
self._data["runs"].append(entry)
self.save()
diff --git a/src/re_agent/llm/observed.py b/src/re_agent/llm/observed.py
index f8c5df9b..0f1cd0af 100644
--- a/src/re_agent/llm/observed.py
+++ b/src/re_agent/llm/observed.py
@@ -52,9 +52,9 @@ def _call(self, **request: Any) -> str:
progress(self.role)
context = current()
+ number = self.budget.consume()
if context:
context.emit("call", self.role)
- number = self.budget.consume()
start = time.monotonic()
event: dict[str, Any] = {"role": self.role, "call": number, "request": request}
try:
diff --git a/src/re_agent/orchestrator/parallel.py b/src/re_agent/orchestrator/parallel.py
new file mode 100644
index 00000000..ed9097ca
--- /dev/null
+++ b/src/re_agent/orchestrator/parallel.py
@@ -0,0 +1,324 @@
+"""Bounded function workers with coordinator-owned journals and checkpoints."""
+from __future__ import annotations
+
+import copy
+import hashlib
+import json
+import os
+import queue
+import threading
+import time
+import uuid
+from collections.abc import Callable
+from concurrent.futures import Future, ThreadPoolExecutor
+from contextlib import nullcontext
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any, cast
+
+from re_agent.backend.protocol import REBackend
+from re_agent.config.loader import validate_config
+from re_agent.config.schema import LLMConfig, ReAgentConfig
+from re_agent.core.identity import project_fingerprint
+from re_agent.core.models import (
+ CheckerVerdict,
+ Finding,
+ FunctionTarget,
+ ObjectiveVerdict,
+ ParityStatus,
+ ReversalResult,
+ ValidationVerdict,
+ Verdict,
+)
+from re_agent.core.session import Session
+from re_agent.llm.protocol import LLMProvider
+from re_agent.orchestrator.dependencies import prerequisites
+from re_agent.orchestrator.execution import Cancelled, Execution, executing
+from re_agent.orchestrator.single import reverse_single
+from re_agent.reports.formatter import _result_to_dict
+from re_agent.utils.address import normalize_address
+from re_agent.utils.storage import atomic_json, file_lock
+
+ProviderFactory = Callable[[LLMConfig], LLMProvider]
+
+
+def decode_result(data: dict[str, Any]) -> ReversalResult:
+ result = ReversalResult(FunctionTarget(data["address"], data["class_name"], data["function_name"]),
+ code=data.get("code", ""), success=data.get("success", False),
+ rounds_used=data.get("rounds_used", 0), run_id=data.get("run_id", ""),
+ error=data.get("error"))
+ if data.get("verdict"):
+ result.checker_verdict = CheckerVerdict(Verdict(data["verdict"]), data.get("summary", ""),
+ data.get("issues", []))
+ if data.get("objective_verdict"):
+ result.objective_verdict = ObjectiveVerdict(Verdict(data["objective_verdict"]),
+ data.get("objective_summary", ""), data.get("objective_findings", []))
+ if data.get("validation_verdict"):
+ result.validation_verdict = ValidationVerdict(Verdict(data["validation_verdict"]),
+ data.get("validation_summary", ""), data.get("validation_findings", []),
+ overlay_file=data.get("candidate_overlay"), checks=data.get("validation_checks", []))
+ if data.get("parity_status"):
+ result.parity_status = ParityStatus(data["parity_status"])
+ result.parity_findings = [Finding(**f) for f in data.get("parity_findings", [])]
+ return result
+
+
+class LockedBackend:
+ """Serialize stateful backend calls; do not share provider or indexer state."""
+
+ def __init__(self, backend: REBackend, lock: threading.Lock, cancel: threading.Event):
+ self.backend, self.lock, self.cancel = backend, lock, cancel
+
+ def __getattr__(self, name: str) -> Any:
+ value = getattr(self.backend, name)
+ if not callable(value):
+ return value
+
+ def invoke(*args: Any, **kwargs: Any) -> Any:
+ while not self.lock.acquire(timeout=.1):
+ if self.cancel.is_set():
+ raise Cancelled("Backend wait cancelled")
+ try:
+ if self.cancel.is_set():
+ raise Cancelled("Backend call cancelled")
+ return copy.deepcopy(value(*args, **kwargs))
+ finally:
+ self.lock.release()
+ return invoke
+
+
+class WorkerSession(Session):
+ def __init__(self, feedback: str, emit: Callable[[str, Any], None], offset: int):
+ self.feedback, self.emit, self.offset = feedback, emit, offset
+
+ def previous_feedback(self, address: str) -> str:
+ return self.feedback
+
+ def record_checkpoint(self, result: ReversalResult) -> None:
+ saved = copy.deepcopy(result)
+ saved.rounds_used += self.offset
+ self.emit("checkpoint", saved)
+
+ def record_result(self, result: ReversalResult, *, idempotent: bool = False) -> None:
+ pass # Only the coordinator publishes final results.
+
+
+@dataclass
+class Event:
+ job: str
+ kind: str
+ value: Any
+ ack: threading.Event
+
+
+def reverse_parallel(
+ targets: list[FunctionTarget], config: ReAgentConfig, backend: REBackend, session: Session,
+ provider_factory: ProviderFactory, limit: int, *, cancel: threading.Event | None = None,
+ promote: Callable[[ReversalResult], ReversalResult] | None = None,
+ identity: str | None = None,
+) -> list[ReversalResult]:
+ """Execute jobs; promotion, when supplied, is serialized by planned target order."""
+ validate_config(config)
+ cancel = cancel or threading.Event()
+ if limit < 1:
+ raise ValueError("Function attempt limit must be positive")
+ policy = asdict(config.orchestrator)
+ for key in ("max_parallel_functions", "max_parallel_validations", "max_functions_per_class"):
+ policy.pop(key, None)
+ models = [asdict(c) for c in (config.agents.reverser or config.llm, config.agents.checker or config.llm)]
+ for model in models:
+ model.pop("api_key", None)
+ identity = identity or project_fingerprint(config)
+ key = hashlib.sha256(json.dumps([identity, str(session.path.resolve()), policy, models],
+ sort_keys=True).encode()).hexdigest()[:24]
+ root = Path(config.output.report_dir).resolve() / "parallel" / key
+ root.mkdir(parents=True, exist_ok=True)
+ with file_lock(session.path.with_suffix(".coordinator"), blocking=False):
+ return _run(targets, config, backend, session, provider_factory, limit, cancel, root, promote)
+
+
+def _run(targets: list[FunctionTarget], config: ReAgentConfig, backend: REBackend, session: Session,
+ factory: ProviderFactory, limit: int, cancel: threading.Event, root: Path,
+ promote: Callable[[ReversalResult], ReversalResult] | None) -> list[ReversalResult]:
+ by_address = {normalize_address(t.address): t for t in targets}
+ order = list(by_address)
+ rank = {a: i for i, a in enumerate(order)}
+ deps = prerequisites(order, {
+ a: {normalize_address(x.address) for x in backend.xrefs_from(t.address)}
+ for a, t in by_address.items()
+ }) if config.orchestrator.selection_strategy == "dependency-order" else {a: set() for a in order}
+ journal = root / "jobs.json"
+ jobs: dict[str, Any] = json.loads(journal.read_text()) if journal.exists() else {}
+ status = session.path.with_suffix(session.path.suffix + ".execution.json")
+ stop = status.with_suffix(".stop")
+ stop.unlink(missing_ok=True)
+ events: queue.Queue[Event] = queue.Queue()
+ gate = threading.Semaphore(config.orchestrator.max_parallel_validations)
+ backend_lock = threading.Lock()
+ generation_lock = threading.Lock()
+ futures: dict[Future[ReversalResult], str] = {}
+ proposed: dict[str, ReversalResult] = {}
+ results: list[ReversalResult] = []
+ submitted = 0
+
+ def save() -> None:
+ atomic_json(journal, jobs)
+ atomic_json(status, {"schema_version": 1, "phase": "stopping" if cancel.is_set() else "running",
+ "pid": os.getpid(), "limit": config.orchestrator.max_parallel_functions,
+ "queued": sum(not session.is_completed(a) and a not in
+ {v["address"] for v in jobs.values() if v["state"] == "running"}
+ and session.attempt_count(a) < config.orchestrator.max_attempts_per_function for a in order),
+ "jobs": [{k: v for k, v in j.items() if k not in {"checkpoint", "result"}} for j in jobs.values()],
+ "updated": time.time(), "stop_file": str(stop.resolve())})
+
+ def publish(job: str, result: ReversalResult) -> None:
+ # Journal first, then idempotent session publication: recovery can finish
+ # either half of this transaction after a crash.
+ jobs[job].update(state="completed" if result.success else "failed", result=_result_to_dict(result))
+ save()
+ session.record_result_once(result)
+ results.append(result)
+
+ for value in jobs.values():
+ if value["state"] in {"completed", "failed"}:
+ session.record_result_once(decode_result(value["result"]))
+ else:
+ value["state"] = "interrupted"
+
+ def emit(job: str, kind: str, value: Any) -> None:
+ event = Event(job, kind, value, threading.Event())
+ events.put(event)
+ while not event.ack.wait(.1):
+ if cancel.is_set():
+ raise Cancelled("Checkpoint publication cancelled")
+
+ def worker(job: str, snapshot: dict[str, Any]) -> ReversalResult:
+ t = by_address[snapshot["address"]]
+ isolated = copy.deepcopy(config)
+ directory = root / job
+ directory.mkdir(exist_ok=True)
+ isolated.output.report_dir = str(directory)
+ isolated.output.log_dir = str(directory / "logs")
+ isolated.parity.cache_dir = str(directory / "parity-cache")
+ isolated.orchestrator.max_parallel_functions = 1
+ offset = int(snapshot.get("rounds", 0))
+ isolated.orchestrator.max_review_rounds -= offset
+ isolated.orchestrator.max_llm_calls_per_function -= int(snapshot.get("calls", 0))
+ if min(isolated.orchestrator.max_review_rounds, isolated.orchestrator.max_llm_calls_per_function) <= 0:
+ return ReversalResult(t, code="", success=False,
+ error="Interrupted attempt exhausted its saved budget", run_id=job)
+ providers: list[LLMProvider] = []
+ try:
+ from re_agent.orchestrator.snapshot import project_snapshot
+
+ with (executing(Execution(cancel, gate, lambda kind, value: emit(job, kind, value))),
+ project_snapshot(isolated, generation_lock) if promote else nullcontext(isolated) as isolated):
+ providers = [factory(isolated.agents.reverser or isolated.llm)]
+ providers.append(factory(isolated.agents.checker or isolated.llm))
+ feedback = json.dumps(snapshot.get("checkpoint", {}))
+ result = reverse_single(t, isolated, cast(REBackend, LockedBackend(backend, backend_lock, cancel)),
+ providers[0], checker_llm=providers[1], session=WorkerSession(
+ feedback, lambda kind, value: emit(job, kind, value), offset))
+ result.rounds_used += offset
+ result.run_id = job
+ return result
+ finally:
+ for provider in providers:
+ close = getattr(provider, "close", None)
+ if callable(close):
+ close()
+
+ def drain() -> None:
+ while True:
+ try:
+ event = events.get_nowait()
+ except queue.Empty:
+ return
+ try:
+ if cancel.is_set():
+ continue
+ value = jobs[event.job]
+ if event.kind == "checkpoint":
+ value.update(checkpoint=_result_to_dict(event.value), rounds=event.value.rounds_used)
+ event.value.run_id = event.job
+ session.record_checkpoint(event.value)
+ elif event.kind == "call":
+ value["calls"] = value.get("calls", 0) + 1
+ else:
+ value["stage"] = event.value
+ save()
+ finally:
+ event.ack.set()
+
+ try:
+ with ThreadPoolExecutor(max_workers=config.orchestrator.max_parallel_functions) as executor:
+ try:
+ while True:
+ if stop.exists():
+ cancel.set()
+ drain()
+ for future in list(futures):
+ if not future.done():
+ continue
+ job = futures.pop(future)
+ if cancel.is_set():
+ jobs[job]["state"] = "interrupted"
+ continue
+ try:
+ result = future.result()
+ except Exception as exc:
+ result = ReversalResult(by_address[jobs[job]["address"]], code="", success=False,
+ error=str(exc), run_id=job)
+ proposed[job] = result
+ jobs[job]["state"] = "proposed"
+ # Cumulative promotion follows dispatch order, never completion order.
+ for job in sorted(list(proposed), key=lambda j: jobs[j]["sequence"]):
+ if promote and any(jobs[j]["sequence"] < jobs[job]["sequence"] for j in futures.values()):
+ break
+ result = proposed.pop(job)
+ if cancel.is_set():
+ jobs[job]["state"] = "interrupted"
+ continue
+ if promote and result.success:
+ with generation_lock, executing(Execution(cancel, gate, lambda k, v: None)):
+ result = promote(result)
+ if cancel.is_set():
+ jobs[job]["state"] = "interrupted"
+ else:
+ publish(job, result)
+ active_addresses = {jobs[j]["address"] for j in [*futures.values(), *proposed]}
+ pending = {a for a in order if not session.is_completed(a)
+ and session.attempt_count(a) < config.orchestrator.max_attempts_per_function}
+ terminal = set(order) - pending
+ while (not cancel.is_set() and
+ len(futures) + len(proposed) < config.orchestrator.max_parallel_functions and
+ submitted < limit):
+ ready = next((a for a in order if a in pending and a not in active_addresses
+ and deps[a] <= terminal), None)
+ if ready is None:
+ break
+ job = next((j for j, v in jobs.items()
+ if v["address"] == ready and v["state"] == "interrupted"), uuid.uuid4().hex)
+ value = jobs.setdefault(job, {"id": job, "address": ready, "calls": 0, "rounds": 0,
+ "sequence": len(jobs), "started": time.time()})
+ value["state"] = "running"
+ save()
+ futures[executor.submit(worker, job, copy.deepcopy(value))] = job
+ active_addresses.add(ready)
+ submitted += 1
+ if not futures and not proposed:
+ break
+ time.sleep(.05)
+ finally:
+ # Ensure worker acknowledgement waits cannot deadlock shutdown.
+ if futures:
+ cancel.set()
+ finally:
+ for value in jobs.values():
+ if value["state"] in {"running", "proposed"}:
+ value["state"] = "interrupted"
+ save()
+ data = json.loads(status.read_text())
+ data["phase"] = "stopped" if cancel.is_set() else "complete"
+ atomic_json(status, data)
+ return sorted(results, key=lambda r: (rank[normalize_address(r.target.address)], jobs[r.run_id]["sequence"]))
diff --git a/src/re_agent/orchestrator/snapshot.py b/src/re_agent/orchestrator/snapshot.py
new file mode 100644
index 00000000..1d29b957
--- /dev/null
+++ b/src/re_agent/orchestrator/snapshot.py
@@ -0,0 +1,35 @@
+"""Immutable per-worker project snapshots for cumulative proposals."""
+from __future__ import annotations
+
+import copy
+import json
+import shutil
+import tempfile
+import threading
+from collections.abc import Iterator
+from contextlib import contextmanager
+from pathlib import Path
+
+from re_agent.config.schema import ReAgentConfig
+from re_agent.verification.candidate import _remap_links
+
+
+@contextmanager
+def project_snapshot(config: ReAgentConfig, lock: threading.Lock) -> Iterator[ReAgentConfig]:
+ with tempfile.TemporaryDirectory(prefix="re-agent-worker-") as directory:
+ scratch = Path(directory)
+ original = Path(config.validation.project_root).resolve()
+ source = Path(config.project_profile.source_root).resolve().relative_to(original)
+ isolated = copy.deepcopy(config)
+ with lock:
+ shutil.copytree(original, scratch, dirs_exist_ok=True, symlinks=True,
+ ignore=shutil.ignore_patterns(".git", ".venv", "build", "reports", "__pycache__"))
+ _remap_links(scratch, original)
+ if config.project_profile.compilation_database:
+ database = json.loads(Path(config.project_profile.compilation_database).read_text(encoding="utf-8"))
+ destination = scratch / ".re-agent-compile_commands.json"
+ destination.write_text(json.dumps(database).replace(str(original), str(scratch)), encoding="utf-8")
+ isolated.project_profile.compilation_database = str(destination)
+ isolated.validation.project_root = str(scratch)
+ isolated.project_profile.source_root = str(scratch / source)
+ yield isolated
diff --git a/tests/test_parallel.py b/tests/test_parallel.py
new file mode 100644
index 00000000..c139978d
--- /dev/null
+++ b/tests/test_parallel.py
@@ -0,0 +1,126 @@
+"""Deterministic scheduler tests without network/model requests."""
+import json
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from unittest.mock import Mock
+
+import pytest
+
+from re_agent.config.schema import ReAgentConfig
+from re_agent.core.models import FunctionTarget, ReversalResult
+from re_agent.core.session import Session
+from re_agent.orchestrator.execution import current
+from re_agent.orchestrator.parallel import reverse_parallel
+
+
+@pytest.fixture
+def setup(tmp_path):
+ config = ReAgentConfig()
+ config.output.report_dir = str(tmp_path / "reports")
+ config.output.session_file = str(tmp_path / "session.json")
+ config.project_profile.source_root = str(tmp_path / "source")
+ config.orchestrator.max_parallel_functions = 2
+ config.orchestrator.max_attempts_per_function = 1
+ config.orchestrator.selection_strategy = "high-impact"
+ config.validation.enabled = False
+ config.parity.enabled = False
+ backend = Mock()
+ backend.xrefs_from.return_value = []
+ targets = [FunctionTarget(hex(i), "C", f"f{i}") for i in range(1, 5)]
+ return config, backend, targets, Session(config.output.session_file)
+
+
+@pytest.mark.parametrize("workers", [1, 2, 4])
+def test_bounded_overlap_isolation_order_and_checkpoints(setup, monkeypatch, workers):
+ config, backend, targets, session = setup
+ config.orchestrator.max_parallel_functions = workers
+ barrier = threading.Barrier(workers)
+ paths, providers = [], []
+ lock = threading.Lock()
+
+ def fake(target, cfg, backend, llm, *, checker_llm, session):
+ with lock:
+ paths.append(cfg.output.report_dir)
+ providers.extend([llm, checker_llm])
+ barrier.wait(timeout=10)
+ current().emit("call", "reverser")
+ result = ReversalResult(target, code="int f() { return 1; }", success=True, rounds_used=1)
+ session.record_checkpoint(result)
+ return result
+
+ monkeypatch.setattr("re_agent.orchestrator.parallel.reverse_single", fake)
+ result = reverse_parallel([*targets, targets[0]], config, backend, session, lambda c: Mock(), 4)
+ assert [r.target for r in result] == targets
+ assert len(set(paths)) == 4
+ assert len({id(p) for p in providers}) == 8
+ assert all(p.close.call_count == 1 for p in providers)
+ assert all(session.attempt_count(t.address) == 1 for t in targets)
+ journal = json.loads(next(Path(config.output.report_dir).rglob("jobs.json")).read_text())
+ assert all(j["calls"] == 1 and j["rounds"] == 1 for j in journal.values())
+ # Recovery publication is idempotent, including completed jobs.
+ assert reverse_parallel(targets, config, backend, session, lambda c: Mock(), 4) == []
+ assert all(session.attempt_count(t.address) == 1 for t in targets)
+
+
+def test_cancel_late_result_and_resume_budget(setup, monkeypatch):
+ config, backend, targets, session = setup
+ cancel = threading.Event()
+ budgets = []
+
+ def interrupted(target, cfg, backend, llm, **kwargs):
+ current().emit("call", "reverser")
+ current().emit("checkpoint", ReversalResult(target, code="draft", rounds_used=1))
+ cancel.set()
+ return ReversalResult(target, code="late", success=True)
+
+ monkeypatch.setattr("re_agent.orchestrator.parallel.reverse_single", interrupted)
+ assert reverse_parallel(targets[:1], config, backend, session, lambda c: Mock(), 1, cancel=cancel) == []
+ assert session.attempt_count(targets[0].address) == 0
+
+ def resumed(target, cfg, backend, llm, **kwargs):
+ budgets.append((cfg.orchestrator.max_llm_calls_per_function, cfg.orchestrator.max_review_rounds))
+ return ReversalResult(target, code="accepted", success=True, rounds_used=1)
+
+ monkeypatch.setattr("re_agent.orchestrator.parallel.reverse_single", resumed)
+ config.orchestrator.max_parallel_functions = 4
+ result = reverse_parallel(targets[:1], config, backend, session, lambda c: Mock(), 1)
+ assert budgets == [(config.orchestrator.max_llm_calls_per_function - 1,
+ config.orchestrator.max_review_rounds - 1)]
+ assert result[0].rounds_used == 2
+ assert session.attempt_count(targets[0].address) == 1
+
+
+def test_retry_limit_and_failure_independence(setup, monkeypatch):
+ config, backend, targets, session = setup
+ config.orchestrator.max_attempts_per_function = 2
+
+ def fake(target, *args, **kwargs):
+ return ReversalResult(target, code="", success=target != targets[0])
+
+ monkeypatch.setattr("re_agent.orchestrator.parallel.reverse_single", fake)
+ result = reverse_parallel(targets, config, backend, session, lambda c: Mock(), 5)
+ assert len(result) == 5
+ assert session.attempt_count(targets[0].address) == 2
+ assert all(session.is_completed(t.address) for t in targets[1:])
+
+
+def test_competing_coordinator_lease(setup, monkeypatch):
+ config, backend, targets, session = setup
+ entered, release = threading.Event(), threading.Event()
+
+ def fake(target, *args, **kwargs):
+ entered.set()
+ assert release.wait(10)
+ return ReversalResult(target, code="", success=True)
+
+ monkeypatch.setattr("re_agent.orchestrator.parallel.reverse_single", fake)
+ with ThreadPoolExecutor() as pool:
+ first = pool.submit(reverse_parallel, targets[:1], config, backend, session, lambda c: Mock(), 1)
+ assert entered.wait(10)
+ try:
+ with pytest.raises(OSError):
+ reverse_parallel(targets[:1], config, backend, session, lambda c: Mock(), 1)
+ finally:
+ release.set()
+ assert len(first.result(timeout=10)) == 1
From 4d36572aea9d9f4bf95dfb0ae40b7608a54d3935 Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 09:28:41 +1000
Subject: [PATCH 17/34] Cancel provider subprocess trees and close per-job
resources
---
src/re_agent/llm/claude.py | 3 +++
src/re_agent/llm/claude_cli.py | 8 +++---
src/re_agent/llm/codex_cli.py | 14 ++++------
src/re_agent/llm/grok_cli.py | 3 +++
src/re_agent/llm/openai_compat.py | 3 +++
src/re_agent/utils/process.py | 2 ++
tests/test_execution.py | 45 +++++++++++++++++++++++++++++++
tests/test_llm/test_claude_cli.py | 6 ++---
tests/test_llm/test_codex_cli.py | 10 +++----
9 files changed, 72 insertions(+), 22 deletions(-)
diff --git a/src/re_agent/llm/claude.py b/src/re_agent/llm/claude.py
index a7a04914..7f57fd65 100644
--- a/src/re_agent/llm/claude.py
+++ b/src/re_agent/llm/claude.py
@@ -40,6 +40,9 @@ def __init__(
# -- LLMProvider interface ------------------------------------------------
+ def close(self) -> None:
+ self._client.close()
+
def send(self, messages: list[Message], **kwargs: Any) -> str:
"""Send messages to Claude and return the assistant response text."""
system_text: str | None = None
diff --git a/src/re_agent/llm/claude_cli.py b/src/re_agent/llm/claude_cli.py
index 1d905b18..31fa5211 100644
--- a/src/re_agent/llm/claude_cli.py
+++ b/src/re_agent/llm/claude_cli.py
@@ -8,6 +8,7 @@
from typing import Any
from re_agent.llm.protocol import Message
+from re_agent.utils.process import run_process
@dataclass
@@ -109,12 +110,9 @@ def _run(
cmd.append(prompt)
try:
- proc = subprocess.run(
+ proc = run_process(
cmd,
- capture_output=True,
- text=True,
- timeout=self._timeout_s,
- check=False,
+ timeout_s=self._timeout_s,
)
except subprocess.TimeoutExpired as exc:
raise RuntimeError(f"claude CLI timed out after {self._timeout_s}s") from exc
diff --git a/src/re_agent/llm/codex_cli.py b/src/re_agent/llm/codex_cli.py
index c913441c..4aa2d63f 100644
--- a/src/re_agent/llm/codex_cli.py
+++ b/src/re_agent/llm/codex_cli.py
@@ -8,6 +8,7 @@
from typing import Any
from re_agent.llm.protocol import Message
+from re_agent.utils.process import run_process
class CodexCLIProvider:
@@ -31,7 +32,7 @@ def send(self, messages: list[Message], **kwargs: Any) -> str:
out_path = Path(tmp.name)
try:
- proc = subprocess.run(
+ proc = run_process(
[
self._codex_bin,
"exec",
@@ -46,17 +47,12 @@ def send(self, messages: list[Message], **kwargs: Any) -> str:
str(model),
"-",
],
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT,
- input=prompt,
- text=True,
- encoding="utf-8",
- timeout=self._timeout_s,
- check=False,
+ input_text=prompt,
+ timeout_s=self._timeout_s,
)
if proc.returncode != 0:
raise RuntimeError(
- f"codex exec failed with exit code {proc.returncode}\n{proc.stdout}"
+ f"codex exec failed with exit code {proc.returncode}\n{proc.stdout}\n{proc.stderr or ''}"
)
return out_path.read_text(encoding="utf-8")
except subprocess.TimeoutExpired as exc:
diff --git a/src/re_agent/llm/grok_cli.py b/src/re_agent/llm/grok_cli.py
index 855592f6..a9cbda18 100644
--- a/src/re_agent/llm/grok_cli.py
+++ b/src/re_agent/llm/grok_cli.py
@@ -44,6 +44,9 @@ def resume(self, conversation_id: str, message: str) -> str:
self._conversations[conversation_id] = (system, True)
return text
+ def close(self) -> None:
+ self._workspace.cleanup()
+
def send(self, messages: list[Message], **kwargs: Any) -> str:
system = "\n\n".join(m.content for m in messages if m.role == "system")
prompt = "\n\n".join(f"[{m.role.upper()}]\n{m.content}" for m in messages if m.role != "system")
diff --git a/src/re_agent/llm/openai_compat.py b/src/re_agent/llm/openai_compat.py
index 10e8a2a7..a9597e29 100644
--- a/src/re_agent/llm/openai_compat.py
+++ b/src/re_agent/llm/openai_compat.py
@@ -46,6 +46,9 @@ def __init__(
# -- LLMProvider interface ------------------------------------------------
+ def close(self) -> None:
+ self._client.close()
+
def send(self, messages: list[Message], **kwargs: Any) -> str:
"""Send messages via the chat completions API and return the response."""
api_messages: list[dict[str, str]] = [{"role": m.role, "content": m.content} for m in messages]
diff --git a/src/re_agent/utils/process.py b/src/re_agent/utils/process.py
index 4d93fb84..651cf796 100644
--- a/src/re_agent/utils/process.py
+++ b/src/re_agent/utils/process.py
@@ -80,6 +80,8 @@ def run_process(
stdout=stdout,
stderr=stderr,
text=True,
+ encoding="utf-8",
+ creationflags=int(getattr(subprocess, "CREATE_NO_WINDOW", 0)) if os.name == "nt" else 0,
start_new_session=os.name != "nt",
)
try:
diff --git a/tests/test_execution.py b/tests/test_execution.py
index 04374b73..a1b267d4 100644
--- a/tests/test_execution.py
+++ b/tests/test_execution.py
@@ -34,3 +34,48 @@ def worker():
cancel.set()
with pytest.raises(Cancelled):
future.result(timeout=5)
+
+
+def test_cancel_reaps_detached_process_tree(tmp_path):
+ import json
+ import os
+ import sys
+ import time
+ from concurrent.futures import ThreadPoolExecutor
+
+ import psutil
+ import pytest
+
+ from re_agent.utils.process import run_process
+
+ ready = tmp_path / "ready.json"
+ script = tmp_path / "parent.py"
+ script.write_text(
+ "import subprocess, sys, pathlib, json, os, time\n"
+ "child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'], "
+ "start_new_session=os.name != 'nt')\n"
+ "pathlib.Path(sys.argv[1]).write_text(json.dumps([os.getpid(), child.pid]))\n"
+ "time.sleep(60)\n"
+ )
+ cancel = threading.Event()
+
+ def invoke():
+ with executing(Execution(cancel, threading.Semaphore(1), lambda *args: None)):
+ run_process([sys.executable, str(script), str(ready)], timeout_s=20)
+
+ with ThreadPoolExecutor() as pool:
+ future = pool.submit(invoke)
+ deadline = time.monotonic() + 10
+ while not ready.exists() and time.monotonic() < deadline:
+ time.sleep(.02)
+ try:
+ assert ready.exists()
+ pids = json.loads(ready.read_text())
+ finally:
+ cancel.set()
+ with pytest.raises(Cancelled):
+ future.result(timeout=10)
+ for pid in pids:
+ if psutil.pid_exists(pid):
+ # POSIX init may not have reaped an already-dead grandchild yet.
+ assert os.name != "nt" and psutil.Process(pid).status() == psutil.STATUS_ZOMBIE
diff --git a/tests/test_llm/test_claude_cli.py b/tests/test_llm/test_claude_cli.py
index 327a7e86..81617648 100644
--- a/tests/test_llm/test_claude_cli.py
+++ b/tests/test_llm/test_claude_cli.py
@@ -24,7 +24,7 @@ def _completed(session_id: str = "session-1") -> CompletedProcess[str]:
def test_claude_cli_send_parses_result_and_usage() -> None:
provider = ClaudeCLIProvider(model="sonnet", max_budget_usd=1.5, effort="high")
- with patch("re_agent.llm.claude_cli.subprocess.run", return_value=_completed()) as run:
+ with patch("re_agent.llm.claude_cli.run_process", return_value=_completed()) as run:
result = provider.send([
Message(role="system", content="system"),
Message(role="user", content="reverse this"),
@@ -42,7 +42,7 @@ def test_claude_cli_send_parses_result_and_usage() -> None:
def test_claude_cli_uses_real_session_resume() -> None:
provider = ClaudeCLIProvider()
conversation_id = provider.new_conversation("system")
- with patch("re_agent.llm.claude_cli.subprocess.run", return_value=_completed()) as run:
+ with patch("re_agent.llm.claude_cli.run_process", return_value=_completed()) as run:
provider.resume(conversation_id, "first")
first = run.call_args.args[0]
provider.resume(conversation_id, "second")
@@ -57,7 +57,7 @@ def test_claude_cli_surfaces_structured_error() -> None:
completed = CompletedProcess(["claude"], 1, stdout=payload, stderr="")
provider = ClaudeCLIProvider()
with (
- patch("re_agent.llm.claude_cli.subprocess.run", return_value=completed),
+ patch("re_agent.llm.claude_cli.run_process", return_value=completed),
pytest.raises(RuntimeError, match="Not logged in"),
):
provider.send([Message(role="user", content="hello")])
diff --git a/tests/test_llm/test_codex_cli.py b/tests/test_llm/test_codex_cli.py
index e8b254cd..046f02ce 100644
--- a/tests/test_llm/test_codex_cli.py
+++ b/tests/test_llm/test_codex_cli.py
@@ -22,17 +22,17 @@ def test_large_unicode_prompt_uses_utf8_stdin(tmp_path, monkeypatch):
"sys.stdout.buffer.write('\u8a3a\u65ad \u2713'.encode('utf-8'))\n",
encoding="utf-8",
)
- run = subprocess.run
+ from re_agent.utils.process import run_process
+ run = run_process
output_paths = []
def invoke(args, **kwargs):
assert sum(len(arg) for arg in args) < 4096
- assert kwargs.get("encoding") == "utf-8"
- assert kwargs.get("input") == expected
+ assert kwargs.get("input_text") == expected
output_paths.append(Path(args[args.index('--output-last-message') + 1]))
return run([sys.executable, str(fake), *args[1:]], **kwargs)
- monkeypatch.setattr("re_agent.llm.codex_cli.subprocess.run", invoke)
+ monkeypatch.setattr("re_agent.llm.codex_cli.run_process", invoke)
content = "\u65e5\u672c\u8a9e evidence & | > \" ' " * 5000
expected = "[USER]\n" + content.strip()
assert CodexCLIProvider().send([Message(role="user", content=content)]) == expected
@@ -53,7 +53,7 @@ def invoke(args, **kwargs):
args, 1, "The selected model requires a newer version of Codex. \u8a3a\u65ad"
)
- monkeypatch.setattr("re_agent.llm.codex_cli.subprocess.run", invoke)
+ monkeypatch.setattr("re_agent.llm.codex_cli.run_process", invoke)
match = {"model": "requires a newer version", "timeout": "timed out after 7s", "missing": "CLI not found"}
with pytest.raises(RuntimeError, match=match[failure]):
CodexCLIProvider(timeout_s=7).send([Message(role="user", content="test")])
From 4824ac582fa749a4ab433db8eb60395d1b0ae3a6 Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 09:38:00 +1000
Subject: [PATCH 18/34] Invalidate accepted results when models or acceptance
policies change
---
src/re_agent/core/identity.py | 7 +++++++
tests/test_config/test_parallel.py | 18 ++++++++++++++++++
2 files changed, 25 insertions(+)
diff --git a/src/re_agent/core/identity.py b/src/re_agent/core/identity.py
index 36a63857..73582538 100644
--- a/src/re_agent/core/identity.py
+++ b/src/re_agent/core/identity.py
@@ -20,6 +20,13 @@ def project_fingerprint(config: ReAgentConfig) -> str:
"backend": asdict(config.backend),
}
values["validation"].pop("parallel_safe", None)
+ values["models"] = {role: asdict(model) for role, model in (
+ ("reverser", config.agents.reverser or config.llm), ("checker", config.agents.checker or config.llm))}
+ for model in values["models"].values():
+ for key in ("api_key", "timeout_s", "max_budget_usd"):
+ model.pop(key, None)
+ values["acceptance"] = {key: value for key, value in asdict(config.orchestrator).items()
+ if key.startswith("objective_") or key == "cumulative_validation"}
digest.update(json.dumps(values, sort_keys=True).encode())
root = Path(config.project_profile.source_root).resolve()
digest.update(str(root).encode())
diff --git a/tests/test_config/test_parallel.py b/tests/test_config/test_parallel.py
index 46c77ab9..ec1ae97f 100644
--- a/tests/test_config/test_parallel.py
+++ b/tests/test_config/test_parallel.py
@@ -26,3 +26,21 @@ def test_validation_contract_and_identity(tmp_path):
validate_config(c)
c.validation.copy_project = False
assert project_fingerprint(c) == original
+
+
+def test_semantic_identity_tracks_models_and_acceptance_not_scheduling(tmp_path):
+ from re_agent.config.schema import ReAgentConfig
+ from re_agent.core.identity import project_fingerprint
+
+ config = ReAgentConfig()
+ config.project_profile.source_root = str(tmp_path)
+ original = project_fingerprint(config)
+ config.orchestrator.max_parallel_functions = 4
+ config.orchestrator.max_parallel_validations = 2
+ config.validation.parallel_safe = True
+ assert project_fingerprint(config) == original
+ config.llm.model = "different-model"
+ assert project_fingerprint(config) != original
+ config.llm.model = ReAgentConfig().llm.model
+ config.orchestrator.objective_verifier_enabled = not config.orchestrator.objective_verifier_enabled
+ assert project_fingerprint(config) != original
From d64070214ee50e0ea67645502bd9f68aef160ac2 Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 09:38:00 +1000
Subject: [PATCH 19/34] Integrate parallel class and manifest runs with safe
cumulative promotion
---
pyproject.toml | 1 +
src/re_agent/cli/cmd_reverse.py | 183 ++++++++++++----------
src/re_agent/cli/main.py | 2 +
src/re_agent/core/session.py | 26 +++
src/re_agent/llm/observed.py | 3 +
src/re_agent/orchestrator/batch_runner.py | 7 +-
src/re_agent/orchestrator/class_runner.py | 59 ++++++-
src/re_agent/orchestrator/execution.py | 22 +++
src/re_agent/orchestrator/parallel.py | 117 +++++++++++---
src/re_agent/orchestrator/snapshot.py | 3 +-
src/re_agent/verification/candidate.py | 3 +-
tests/test_parallel.py | 165 ++++++++++++++++++-
12 files changed, 474 insertions(+), 117 deletions(-)
diff --git a/pyproject.toml b/pyproject.toml
index d0a83f42..c93f0cc0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -40,6 +40,7 @@ dependencies = [
"pyyaml>=6.0",
"anthropic>=0.40",
"openai>=1.0",
+ "psutil>=7.0",
]
[project.optional-dependencies]
diff --git a/src/re_agent/cli/cmd_reverse.py b/src/re_agent/cli/cmd_reverse.py
index f7c40e76..4ce0c17d 100644
--- a/src/re_agent/cli/cmd_reverse.py
+++ b/src/re_agent/cli/cmd_reverse.py
@@ -28,6 +28,10 @@ def cmd_reverse(args: argparse.Namespace) -> int:
if args.skip_parity:
config.parity.enabled = False
+ for name in ("max_parallel_functions", "max_parallel_validations"):
+ value = getattr(args, name, None)
+ if value is not None:
+ setattr(config.orchestrator, name, value)
validate_config(config)
from re_agent.core.identity import project_fingerprint
@@ -57,92 +61,113 @@ def cmd_reverse(args: argparse.Namespace) -> int:
from re_agent.core.session import Session
from re_agent.llm.registry import create_provider
- reverser_llm = create_provider(config.agents.reverser or config.llm)
- checker_llm = create_provider(config.agents.checker or config.llm)
+ # Parallel jobs construct their own providers; no unused shared clients.
+ reverser_llm = checker_llm = None
+ if args.address or config.orchestrator.max_parallel_functions == 1:
+ reverser_llm = create_provider(config.agents.reverser or config.llm)
+ checker_llm = create_provider(config.agents.checker or config.llm)
backend = create_backend(config.backend)
session = Session(config.output.session_file)
- session.bind(project_fingerprint(config))
-
- if args.address:
- from re_agent.orchestrator.single import reverse_single
-
- class_name = args.class_name or ""
- function_name = ""
-
- dec = backend.decompile(args.address)
- if dec.name:
- resolved_class, _, function_name = dec.name.rpartition("::")
- function_name = function_name or dec.name
- class_name = class_name or resolved_class
- # Project hooks provide identity when legacy decompilation uses FUN_* names.
- from re_agent.parity.source_indexer import SourceIndexer
- from re_agent.utils.address import normalize_address
-
- source_index = SourceIndexer(Path(config.project_profile.source_root), config.project_profile)
- for address, (hook_class, hook_name) in source_index.hook_address_index.items():
- if normalize_address(address) == normalize_address(args.address):
- class_name = hook_class or class_name
- function_name = hook_name
- break
-
- target = FunctionTarget(
- address=args.address,
- class_name=class_name,
- function_name=function_name,
- )
- result = reverse_single(
- target,
- config,
- backend,
- reverser_llm,
- checker_llm=checker_llm,
- session=session,
- indexer=source_index,
- )
- from re_agent.reports.formatter import results_to_json, results_to_markdown
-
- if config.output.format == "json":
- print(results_to_json([result]))
- elif config.output.format == "markdown":
- print(results_to_markdown([result]))
- else:
- print(format_result(result))
- return 0 if result.success else 1
-
- if args.class_name or plan is not None:
- from re_agent.orchestrator.class_runner import reverse_class
-
- if plan is not None:
- from re_agent.orchestrator.batch_runner import reverse_manifest
-
- results = reverse_manifest(plan, config, backend, reverser_llm, session,
- args.max_functions, checker_llm)
- else:
- results = reverse_class(
- class_name=args.class_name,
- config=config,
- backend=backend,
- llm=reverser_llm,
+ from contextlib import ExitStack
+
+ with session.coordinator(), ExitStack() as resources:
+ for provider in (reverser_llm, checker_llm):
+ close = getattr(provider, "close", None)
+ if callable(close):
+ resources.callback(close)
+ session.bind(project_fingerprint(config))
+
+ if args.address:
+ from re_agent.orchestrator.single import reverse_single
+
+ class_name = args.class_name or ""
+ function_name = ""
+
+ dec = backend.decompile(args.address)
+ if dec.name:
+ resolved_class, _, function_name = dec.name.rpartition("::")
+ function_name = function_name or dec.name
+ class_name = class_name or resolved_class
+ # Project hooks provide identity when legacy decompilation uses FUN_* names.
+ from re_agent.parity.source_indexer import SourceIndexer
+ from re_agent.utils.address import normalize_address
+
+ source_index = SourceIndexer(Path(config.project_profile.source_root), config.project_profile)
+ for address, (hook_class, hook_name) in source_index.hook_address_index.items():
+ if normalize_address(address) == normalize_address(args.address):
+ class_name = hook_class or class_name
+ function_name = hook_name
+ break
+
+ target = FunctionTarget(
+ address=args.address,
+ class_name=class_name,
+ function_name=function_name,
+ )
+ assert reverser_llm is not None
+ result = reverse_single(
+ target,
+ config,
+ backend,
+ reverser_llm,
checker_llm=checker_llm,
session=session,
- max_functions=args.max_functions,
+ indexer=source_index,
)
- from re_agent.reports.formatter import results_to_json, results_to_markdown
-
- if config.output.format == "json":
- print(results_to_json(results))
- elif config.output.format == "markdown":
- print(results_to_markdown(results))
- else:
- for result in results:
+ from re_agent.reports.formatter import results_to_json, results_to_markdown
+
+ if config.output.format == "json":
+ print(results_to_json([result]))
+ elif config.output.format == "markdown":
+ print(results_to_markdown([result]))
+ else:
print(format_result(result))
- passed = sum(1 for r in results if r.success)
- total = len(results)
- print(f"Results: {passed}/{total} passed", file=sys.stderr)
- return 0 if passed == total else 1
+ return 0 if result.success else 1
+
+ if args.class_name or plan is not None:
+ from re_agent.orchestrator.class_runner import reverse_class
+
+ if plan is not None:
+ from re_agent.orchestrator.batch_runner import reverse_manifest
+
+ results = reverse_manifest(plan, config, backend, reverser_llm, session,
+ args.max_functions, checker_llm, provider_factory=create_provider)
+ else:
+ results = reverse_class(
+ class_name=args.class_name,
+ config=config,
+ backend=backend,
+ llm=reverser_llm,
+ checker_llm=checker_llm,
+ session=session,
+ max_functions=args.max_functions,
+ provider_factory=create_provider,
+ )
+ from re_agent.reports.formatter import results_to_json, results_to_markdown
+
+ if config.output.format == "json":
+ print(results_to_json(results))
+ elif config.output.format == "markdown":
+ print(results_to_markdown(results))
+ else:
+ for result in results:
+ print(format_result(result))
+ passed = sum(1 for r in results if r.success)
+ total = len(results)
+ print(f"Results: {passed}/{total} passed", file=sys.stderr)
+ import json
- print("Error: specify --address, --class, or --manifest", file=sys.stderr)
- return 1
+ status_path = session.path.with_suffix(session.path.suffix + ".execution.json")
+ if config.orchestrator.max_parallel_functions > 1 and status_path.exists():
+ phase = json.loads(status_path.read_text(encoding="utf-8")).get("phase")
+ if phase == "stopped":
+ return 130
+ if phase == "failed":
+ return 1
+ return 0 if passed == total else 1
+
+ print("Error: specify --address, --class, or --manifest", file=sys.stderr)
+ return 1
def _dry_run(args: argparse.Namespace, config: ReAgentConfig) -> int:
diff --git a/src/re_agent/cli/main.py b/src/re_agent/cli/main.py
index 5282bc8b..671ea6e1 100644
--- a/src/re_agent/cli/main.py
+++ b/src/re_agent/cli/main.py
@@ -51,6 +51,8 @@ def build_parser() -> argparse.ArgumentParser:
rev_p.add_argument("--address", help="Single function address to reverse")
rev_p.add_argument("--class", dest="class_name", help="Class name for class-level reversal")
rev_p.add_argument("--max-functions", type=int, default=None, help="Max functions per class")
+ rev_p.add_argument("--max-parallel-functions", type=int, default=None, help="Concurrent functions (1-32)")
+ rev_p.add_argument("--max-parallel-validations", type=int, default=None, help="Concurrent isolated validations")
rev_p.add_argument("--max-rounds", type=int, default=None, help="Max review rounds per function")
rev_p.add_argument("--dry-run", action="store_true", help="Show plan without executing")
rev_p.add_argument("--skip-parity", action="store_true", help="Skip parity check after PASS")
diff --git a/src/re_agent/core/session.py b/src/re_agent/core/session.py
index e24f339c..fb913338 100644
--- a/src/re_agent/core/session.py
+++ b/src/re_agent/core/session.py
@@ -4,7 +4,11 @@
import hashlib
import json
+import threading
import time
+from collections import Counter
+from collections.abc import Iterator
+from contextlib import contextmanager
from dataclasses import asdict
from pathlib import Path
from typing import Any
@@ -19,10 +23,27 @@ class Session:
def __init__(self, path: str | Path = "re-agent-progress.json") -> None:
self.path = Path(path)
+ self._lease_owner: int | None = None
self._data: dict[str, Any] = {"functions": {}, "runs": []}
if self.path.exists():
self.load()
+ @contextmanager
+ def coordinator(self) -> Iterator[None]:
+ """Lease one session across selection, identity binding, and publication."""
+ owner = threading.get_ident()
+ if getattr(self, "_lease_owner", None) == owner:
+ yield
+ return
+ with file_lock(self.path.with_suffix(".coordinator"), blocking=False):
+ self._lease_owner = owner
+ try:
+ if self.path.exists():
+ self.load()
+ yield
+ finally:
+ self._lease_owner = None
+
def load(self) -> None:
data = json.loads(self.path.read_text(encoding="utf-8"))
if (
@@ -106,6 +127,11 @@ def is_attempted(self, address: str) -> bool:
addr = normalize_address(address)
return addr in self._data["functions"]
+ def attempt_counts(self) -> dict[str, int]:
+ """Build one attempt index for a coordinator, avoiding repeated history scans."""
+ return dict(Counter(normalize_address(str(entry.get("address", "")))
+ for entry in self._data.get("runs", [])))
+
def attempt_count(self, address: str) -> int:
"""Return the number of recorded runs for an address."""
addr = normalize_address(address)
diff --git a/src/re_agent/llm/observed.py b/src/re_agent/llm/observed.py
index 0f1cd0af..871ce838 100644
--- a/src/re_agent/llm/observed.py
+++ b/src/re_agent/llm/observed.py
@@ -67,6 +67,9 @@ def _call(self, **request: Any) -> str:
context.check()
return response
except Exception as exc:
+ status = getattr(exc, "status_code", None)
+ if context and status in (401, 403):
+ context.fail("authentication", str(exc))
event["error"] = str(exc)
raise
finally:
diff --git a/src/re_agent/orchestrator/batch_runner.py b/src/re_agent/orchestrator/batch_runner.py
index e7f59c83..11efeac4 100644
--- a/src/re_agent/orchestrator/batch_runner.py
+++ b/src/re_agent/orchestrator/batch_runner.py
@@ -10,6 +10,7 @@
from re_agent.core.target_plan import TargetPlan
from re_agent.llm.protocol import LLMProvider
from re_agent.orchestrator.class_runner import reverse_class
+from re_agent.orchestrator.parallel import ProviderFactory
class _ManifestBackend:
@@ -31,9 +32,11 @@ def unimplemented(self, filter_pattern: str | None = None) -> list[FunctionEntry
def reverse_manifest(plan: TargetPlan, config: ReAgentConfig, backend: REBackend,
- llm: LLMProvider, session: Session, max_functions: int | None = None,
- checker_llm: LLMProvider | None = None) -> list[ReversalResult]:
+ llm: LLMProvider | None, session: Session, max_functions: int | None = None,
+ checker_llm: LLMProvider | None = None, *,
+ provider_factory: ProviderFactory | None = None) -> list[ReversalResult]:
return reverse_class(
"", config, cast(REBackend, _ManifestBackend(backend, plan)), llm, session,
max_functions, checker_llm, target_addresses={target.address for target in plan.functions},
+ provider_factory=provider_factory,
)
diff --git a/src/re_agent/orchestrator/class_runner.py b/src/re_agent/orchestrator/class_runner.py
index a1f343eb..05d9975b 100644
--- a/src/re_agent/orchestrator/class_runner.py
+++ b/src/re_agent/orchestrator/class_runner.py
@@ -16,7 +16,8 @@
from re_agent.core.models import ReversalResult
from re_agent.core.session import Session
from re_agent.llm.protocol import LLMProvider
-from re_agent.orchestrator.single import reverse_single
+from re_agent.orchestrator.parallel import ProviderFactory, reverse_parallel
+from re_agent.orchestrator.single import reverse_single, validate_result
from re_agent.parity.source_indexer import SourceIndexer
from re_agent.verification.candidate import _remap_links, create_candidate_overlay
@@ -27,7 +28,7 @@ def _reverse_class(
class_name: str,
config: ReAgentConfig,
backend: REBackend,
- llm: LLMProvider,
+ llm: LLMProvider | None,
session: Session | None = None,
max_functions: int | None = None,
checker_llm: LLMProvider | None = None,
@@ -45,6 +46,8 @@ def _reverse_class(
Returns:
List of ReversalResult for each attempted function
"""
+ if llm is None:
+ raise ValueError("Sequential execution requires a provider")
if session is None:
session = Session(config.output.session_file)
@@ -111,20 +114,50 @@ def _promote(result: ReversalResult, config: ReAgentConfig) -> None:
Path(matches[0].path).write_text(candidate.read_text(encoding="utf-8"), encoding="utf-8")
-def reverse_class(
+def _reverse_class_run(
class_name: str,
config: ReAgentConfig,
backend: REBackend,
- llm: LLMProvider,
+ llm: LLMProvider | None,
session: Session | None = None,
max_functions: int | None = None,
checker_llm: LLMProvider | None = None,
*,
target_addresses: set[str] | None = None,
+ provider_factory: ProviderFactory | None = None,
) -> list[ReversalResult]:
"""Validate a class cumulatively in an isolated scratch project."""
+ from re_agent.core.identity import project_fingerprint
+
+ identity = project_fingerprint(config)
+ parallel = config.orchestrator.max_parallel_functions > 1
+ if parallel and provider_factory is None:
+ raise ValueError("Parallel execution requires a provider_factory; live providers cannot be shared")
+
+ def run(effective: ReAgentConfig, cumulative: bool = False) -> list[ReversalResult]:
+ if not parallel:
+ return _reverse_class(class_name, effective, backend, llm, session, max_functions, checker_llm)
+ from re_agent.core.models import FunctionTarget
+
+ entries = backend.remaining(class_name) or backend.unimplemented(class_name)
+ reverse_rank = effective.orchestrator.selection_strategy == "high-impact"
+ entries.sort(key=lambda f: (-f.caller_count if reverse_rank else f.caller_count, f.name, f.address))
+ targets = [FunctionTarget(f.address, f.class_name or class_name, f.name, f.caller_count) for f in entries]
+
+ def promote(result: ReversalResult, view: REBackend) -> ReversalResult:
+ # Provisional successes must pass unchanged gates on the latest generation.
+ result = validate_result(result, effective, view)
+ if result.success:
+ _promote(result, effective)
+ return result
+
+ assert provider_factory is not None
+ return reverse_parallel(targets, effective, backend, session or Session(config.output.session_file),
+ provider_factory, max_functions or config.orchestrator.max_functions_per_class,
+ promote=promote if cumulative else None, identity=identity)
+
if not (config.validation.copy_project and config.orchestrator.cumulative_validation):
- return _reverse_class(class_name, config, backend, llm, session, max_functions, checker_llm)
+ return run(config)
original = Path(config.validation.project_root).resolve()
relative_source = Path(config.project_profile.source_root).resolve().relative_to(original)
session = session or Session(config.output.session_file)
@@ -135,7 +168,7 @@ def reverse_class(
scratch,
dirs_exist_ok=True,
symlinks=True,
- ignore=shutil.ignore_patterns(".git", ".venv", "build", "reports", "__pycache__"),
+ ignore=shutil.ignore_patterns(".git", ".venv", "build", "reports", "__pycache__", "*.coordinator.lock"),
)
_remap_links(scratch, original)
isolated = copy.deepcopy(config)
@@ -164,4 +197,16 @@ def reverse_class(
),
isolated,
)
- return _reverse_class(class_name, isolated, backend, llm, session, max_functions, checker_llm)
+ return run(isolated, cumulative=True)
+
+
+def reverse_class(
+ class_name: str, config: ReAgentConfig, backend: REBackend, llm: LLMProvider | None,
+ session: Session | None = None, max_functions: int | None = None,
+ checker_llm: LLMProvider | None = None, *, target_addresses: set[str] | None = None,
+ provider_factory: ProviderFactory | None = None,
+) -> list[ReversalResult]:
+ session = session or Session(config.output.session_file)
+ with session.coordinator():
+ return _reverse_class_run(class_name, config, backend, llm, session, max_functions, checker_llm,
+ target_addresses=target_addresses, provider_factory=provider_factory)
diff --git a/src/re_agent/orchestrator/execution.py b/src/re_agent/orchestrator/execution.py
index f756335d..0df6ebac 100644
--- a/src/re_agent/orchestrator/execution.py
+++ b/src/re_agent/orchestrator/execution.py
@@ -18,6 +18,10 @@ class Execution:
validations: threading.Semaphore
emit: Callable[[str, Any], None]
+ def fail(self, category: str, message: str) -> None:
+ self.emit("fatal", {"category": category, "message": message})
+ self.cancel.set()
+
def check(self) -> None:
if self.cancel.is_set():
raise Cancelled("Execution cancelled")
@@ -63,3 +67,21 @@ def validation_lane() -> Iterator[None]:
context.check()
finally:
context.validations.release()
+
+
+@contextmanager
+def cancellation_signals(cancel: threading.Event) -> Iterator[None]:
+ """Turn interrupt/termination into orderly shutdown on CLI main threads."""
+ import signal
+
+ if threading.current_thread() is not threading.main_thread():
+ yield
+ return
+ previous = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)}
+ try:
+ for sig in previous:
+ signal.signal(sig, lambda signum, frame: cancel.set())
+ yield
+ finally:
+ for sig, handler in previous.items():
+ signal.signal(sig, handler)
diff --git a/src/re_agent/orchestrator/parallel.py b/src/re_agent/orchestrator/parallel.py
index ed9097ca..083eb1a6 100644
--- a/src/re_agent/orchestrator/parallel.py
+++ b/src/re_agent/orchestrator/parallel.py
@@ -37,7 +37,7 @@
from re_agent.orchestrator.single import reverse_single
from re_agent.reports.formatter import _result_to_dict
from re_agent.utils.address import normalize_address
-from re_agent.utils.storage import atomic_json, file_lock
+from re_agent.utils.storage import atomic_json
ProviderFactory = Callable[[LLMConfig], LLMProvider]
@@ -82,6 +82,13 @@ def invoke(*args: Any, **kwargs: Any) -> Any:
if self.cancel.is_set():
raise Cancelled("Backend call cancelled")
return copy.deepcopy(value(*args, **kwargs))
+ except OSError as exc:
+ from re_agent.orchestrator.execution import current
+
+ context = current()
+ if context:
+ context.fail("evidence", str(exc))
+ raise
finally:
self.lock.release()
return invoke
@@ -114,7 +121,7 @@ class Event:
def reverse_parallel(
targets: list[FunctionTarget], config: ReAgentConfig, backend: REBackend, session: Session,
provider_factory: ProviderFactory, limit: int, *, cancel: threading.Event | None = None,
- promote: Callable[[ReversalResult], ReversalResult] | None = None,
+ promote: Callable[[ReversalResult, REBackend], ReversalResult] | None = None,
identity: str | None = None,
) -> list[ReversalResult]:
"""Execute jobs; promotion, when supplied, is serialized by planned target order."""
@@ -133,13 +140,15 @@ def reverse_parallel(
sort_keys=True).encode()).hexdigest()[:24]
root = Path(config.output.report_dir).resolve() / "parallel" / key
root.mkdir(parents=True, exist_ok=True)
- with file_lock(session.path.with_suffix(".coordinator"), blocking=False):
+ from re_agent.orchestrator.execution import cancellation_signals
+
+ with session.coordinator(), cancellation_signals(cancel):
return _run(targets, config, backend, session, provider_factory, limit, cancel, root, promote)
def _run(targets: list[FunctionTarget], config: ReAgentConfig, backend: REBackend, session: Session,
factory: ProviderFactory, limit: int, cancel: threading.Event, root: Path,
- promote: Callable[[ReversalResult], ReversalResult] | None) -> list[ReversalResult]:
+ promote: Callable[[ReversalResult, REBackend], ReversalResult] | None) -> list[ReversalResult]:
by_address = {normalize_address(t.address): t for t in targets}
order = list(by_address)
rank = {a: i for i, a in enumerate(order)}
@@ -147,8 +156,12 @@ def _run(targets: list[FunctionTarget], config: ReAgentConfig, backend: REBacken
a: {normalize_address(x.address) for x in backend.xrefs_from(t.address)}
for a, t in by_address.items()
}) if config.orchestrator.selection_strategy == "dependency-order" else {a: set() for a in order}
- journal = root / "jobs.json"
- jobs: dict[str, Any] = json.loads(journal.read_text()) if journal.exists() else {}
+ journal = root / "jobs"
+ journal.mkdir(exist_ok=True)
+ jobs: dict[str, Any] = {p.stem: json.loads(p.read_text(encoding="utf-8")) for p in sorted(journal.glob("*.json"))}
+ jobs = dict(sorted(jobs.items(), key=lambda pair: pair[1]["sequence"]))
+ attempts = session.attempt_counts()
+ completed = {normalize_address(v["address"]) for v in session.get_all_functions() if v.get("success")}
status = session.path.with_suffix(session.path.suffix + ".execution.json")
stop = status.with_suffix(".stop")
stop.unlink(missing_ok=True)
@@ -160,34 +173,66 @@ def _run(targets: list[FunctionTarget], config: ReAgentConfig, backend: REBacken
proposed: dict[str, ReversalResult] = {}
results: list[ReversalResult] = []
submitted = 0
+ failure: dict[str, str] | None = None
+ watcher_done = threading.Event()
+
+ def watch_stop() -> None:
+ while not watcher_done.wait(.1):
+ if stop.exists():
+ cancel.set()
+
+ watcher = threading.Thread(target=watch_stop, daemon=True)
+
+ import psutil
- def save() -> None:
- atomic_json(journal, jobs)
+ created = psutil.Process().create_time()
+
+ def save(changed: str | None = None) -> None:
+ if changed is not None:
+ atomic_json(journal / f"{changed}.json", jobs[changed])
+ active = {v["address"] for v in jobs.values() if v["state"] in {"running", "proposed"}}
+ visible = [v for v in jobs.values() if v["state"] in {"running", "proposed"}]
+ visible += [v for v in jobs.values() if v["state"] not in {"running", "proposed"}][-32:]
atomic_json(status, {"schema_version": 1, "phase": "stopping" if cancel.is_set() else "running",
- "pid": os.getpid(), "limit": config.orchestrator.max_parallel_functions,
- "queued": sum(not session.is_completed(a) and a not in
- {v["address"] for v in jobs.values() if v["state"] == "running"}
- and session.attempt_count(a) < config.orchestrator.max_attempts_per_function for a in order),
- "jobs": [{k: v for k, v in j.items() if k not in {"checkpoint", "result"}} for j in jobs.values()],
- "updated": time.time(), "stop_file": str(stop.resolve())})
+ "pid": os.getpid(), "created": created, "run_id": root.name,
+ "limit": config.orchestrator.max_parallel_functions,
+ "queued": sum(a not in completed and a not in active and
+ attempts.get(a, 0) < config.orchestrator.max_attempts_per_function for a in order),
+ "jobs": [{k: v for k, v in j.items() if k not in {"checkpoint", "result"}} for j in visible],
+ "counts": {state: sum(v["state"] == state for v in jobs.values())
+ for state in ("running", "proposed", "completed", "failed", "interrupted")},
+ "updated": time.time(), "error": failure, "stop_file": str(stop.resolve())})
def publish(job: str, result: ReversalResult) -> None:
# Journal first, then idempotent session publication: recovery can finish
# either half of this transaction after a crash.
- jobs[job].update(state="completed" if result.success else "failed", result=_result_to_dict(result))
- save()
+ jobs[job].update(state="completed" if result.success else "failed",
+ result=_result_to_dict(result), finished=time.time())
+ save(job)
session.record_result_once(result)
+ address = normalize_address(result.target.address)
+ attempts[address] = attempts.get(address, 0) + 1
+ if result.success:
+ completed.add(address)
results.append(result)
for value in jobs.values():
if value["state"] in {"completed", "failed"}:
- session.record_result_once(decode_result(value["result"]))
+ recovered = decode_result(value["result"])
+ if promote and recovered.success and not session.is_completed(recovered.target.address):
+ recovered = promote(recovered, backend)
+ session.record_result_once(recovered)
else:
value["state"] = "interrupted"
+ save(value["id"])
+ attempts = session.attempt_counts()
+ completed = {normalize_address(v["address"]) for v in session.get_all_functions() if v.get("success")}
def emit(job: str, kind: str, value: Any) -> None:
event = Event(job, kind, value, threading.Event())
events.put(event)
+ if kind in {"stage", "fatal"}:
+ return
while not event.ack.wait(.1):
if cancel.is_set():
raise Cancelled("Checkpoint publication cancelled")
@@ -213,8 +258,13 @@ def worker(job: str, snapshot: dict[str, Any]) -> ReversalResult:
with (executing(Execution(cancel, gate, lambda kind, value: emit(job, kind, value))),
project_snapshot(isolated, generation_lock) if promote else nullcontext(isolated) as isolated):
- providers = [factory(isolated.agents.reverser or isolated.llm)]
- providers.append(factory(isolated.agents.checker or isolated.llm))
+ try:
+ providers = [factory(isolated.agents.reverser or isolated.llm)]
+ providers.append(factory(isolated.agents.checker or isolated.llm))
+ except (ValueError, OSError) as exc:
+ emit(job, "fatal", {"category": "configuration", "message": str(exc)})
+ cancel.set()
+ raise
feedback = json.dumps(snapshot.get("checkpoint", {}))
result = reverse_single(t, isolated, cast(REBackend, LockedBackend(backend, backend_lock, cancel)),
providers[0], checker_llm=providers[1], session=WorkerSession(
@@ -229,12 +279,17 @@ def worker(job: str, snapshot: dict[str, Any]) -> ReversalResult:
close()
def drain() -> None:
+ nonlocal failure
while True:
try:
event = events.get_nowait()
except queue.Empty:
return
try:
+ if event.kind == "fatal":
+ failure = event.value
+ cancel.set()
+ continue
if cancel.is_set():
continue
value = jobs[event.job]
@@ -246,10 +301,11 @@ def drain() -> None:
value["calls"] = value.get("calls", 0) + 1
else:
value["stage"] = event.value
- save()
+ save(event.job)
finally:
event.ack.set()
+ watcher.start()
try:
with ThreadPoolExecutor(max_workers=config.orchestrator.max_parallel_functions) as executor:
try:
@@ -281,14 +337,14 @@ def drain() -> None:
continue
if promote and result.success:
with generation_lock, executing(Execution(cancel, gate, lambda k, v: None)):
- result = promote(result)
+ result = promote(result, cast(REBackend, LockedBackend(backend, backend_lock, cancel)))
if cancel.is_set():
jobs[job]["state"] = "interrupted"
else:
publish(job, result)
active_addresses = {jobs[j]["address"] for j in [*futures.values(), *proposed]}
- pending = {a for a in order if not session.is_completed(a)
- and session.attempt_count(a) < config.orchestrator.max_attempts_per_function}
+ pending = {a for a in order if a not in completed
+ and attempts.get(a, 0) < config.orchestrator.max_attempts_per_function}
terminal = set(order) - pending
while (not cancel.is_set() and
len(futures) + len(proposed) < config.orchestrator.max_parallel_functions and
@@ -300,9 +356,10 @@ def drain() -> None:
job = next((j for j, v in jobs.items()
if v["address"] == ready and v["state"] == "interrupted"), uuid.uuid4().hex)
value = jobs.setdefault(job, {"id": job, "address": ready, "calls": 0, "rounds": 0,
- "sequence": len(jobs), "started": time.time()})
+ "sequence": len(jobs), "started": time.time(),
+ "checkpoint": session.get_checkpoint(ready) or {}})
value["state"] = "running"
- save()
+ save(job)
futures[executor.submit(worker, job, copy.deepcopy(value))] = job
active_addresses.add(ready)
submitted += 1
@@ -313,12 +370,20 @@ def drain() -> None:
# Ensure worker acknowledgement waits cannot deadlock shutdown.
if futures:
cancel.set()
+ except BaseException:
+ cancel.set()
+ raise
finally:
+ drain()
+ watcher_done.set()
+ watcher.join()
for value in jobs.values():
if value["state"] in {"running", "proposed"}:
value["state"] = "interrupted"
+ if value["state"] == "interrupted":
+ save(value["id"])
save()
data = json.loads(status.read_text())
- data["phase"] = "stopped" if cancel.is_set() else "complete"
+ data["phase"] = "failed" if failure else "stopped" if cancel.is_set() else "complete"
atomic_json(status, data)
return sorted(results, key=lambda r: (rank[normalize_address(r.target.address)], jobs[r.run_id]["sequence"]))
diff --git a/src/re_agent/orchestrator/snapshot.py b/src/re_agent/orchestrator/snapshot.py
index 1d29b957..a5598610 100644
--- a/src/re_agent/orchestrator/snapshot.py
+++ b/src/re_agent/orchestrator/snapshot.py
@@ -23,7 +23,8 @@ def project_snapshot(config: ReAgentConfig, lock: threading.Lock) -> Iterator[Re
isolated = copy.deepcopy(config)
with lock:
shutil.copytree(original, scratch, dirs_exist_ok=True, symlinks=True,
- ignore=shutil.ignore_patterns(".git", ".venv", "build", "reports", "__pycache__"))
+ ignore=shutil.ignore_patterns(".git", ".venv", "build", "reports", "__pycache__",
+ "*.coordinator.lock"))
_remap_links(scratch, original)
if config.project_profile.compilation_database:
database = json.loads(Path(config.project_profile.compilation_database).read_text(encoding="utf-8"))
diff --git a/src/re_agent/verification/candidate.py b/src/re_agent/verification/candidate.py
index 2e630a0d..8f052672 100644
--- a/src/re_agent/verification/candidate.py
+++ b/src/re_agent/verification/candidate.py
@@ -63,7 +63,8 @@ def create_candidate_overlay(
overlay_root,
dirs_exist_ok=True,
symlinks=True,
- ignore=shutil.ignore_patterns(".git", ".venv", "build", "reports", "__pycache__", "*.pyc"),
+ ignore=shutil.ignore_patterns(".git", ".venv", "build", "reports", "__pycache__",
+ "*.pyc", "*.coordinator.lock"),
)
_remap_links(overlay_root, project_root)
else:
diff --git a/tests/test_parallel.py b/tests/test_parallel.py
index c139978d..ae2267c6 100644
--- a/tests/test_parallel.py
+++ b/tests/test_parallel.py
@@ -56,7 +56,7 @@ def fake(target, cfg, backend, llm, *, checker_llm, session):
assert len({id(p) for p in providers}) == 8
assert all(p.close.call_count == 1 for p in providers)
assert all(session.attempt_count(t.address) == 1 for t in targets)
- journal = json.loads(next(Path(config.output.report_dir).rglob("jobs.json")).read_text())
+ journal = {p.stem: json.loads(p.read_text()) for p in Path(config.output.report_dir).glob("parallel/*/jobs/*.json")}
assert all(j["calls"] == 1 and j["rounds"] == 1 for j in journal.values())
# Recovery publication is idempotent, including completed jobs.
assert reverse_parallel(targets, config, backend, session, lambda c: Mock(), 4) == []
@@ -124,3 +124,166 @@ def fake(target, *args, **kwargs):
finally:
release.set()
assert len(first.result(timeout=10)) == 1
+
+
+@pytest.mark.parametrize("workers", [1, 2, 4])
+def test_full_manifest_pipeline_all_workers(tmp_path, workers):
+ from re_agent.backend.stub import StubBackend
+ from re_agent.config.schema import LLMConfig
+ from re_agent.core.target_plan import TargetPlan
+ from re_agent.orchestrator.batch_runner import reverse_manifest
+ from tests.test_agents.test_loop import MockLLM
+ from tests.test_audit_regressions import config_for
+
+ config = config_for(tmp_path)
+ config.validation.enabled = False
+ config.orchestrator.max_parallel_functions = workers
+ config.agents.reverser = LLMConfig(model="reverse")
+ config.agents.checker = LLMConfig(model="check")
+ targets = [FunctionTarget(hex(i), "", "f") for i in range(1, 5)]
+
+ def factory(cfg):
+ return MockLLM(['{"verdict":"PASS"}'] if cfg.model == "check" else ['```cpp\nint f() { return 1; }\n```'])
+
+ result = reverse_manifest(TargetPlan("a" * 64, [], targets), config, StubBackend(),
+ factory(config.agents.reverser), Session(config.output.session_file), 4,
+ factory(config.agents.checker), provider_factory=factory)
+ assert len(result) == 4
+ assert all(r.success and r.checker_verdict.verdict.value == "PASS" for r in result)
+
+
+def test_cumulative_proposals_same_file_revalidate_latest_generation(tmp_path, monkeypatch):
+ from re_agent.backend.stub import StubBackend
+ from re_agent.core.target_plan import TargetPlan
+ from re_agent.orchestrator.batch_runner import reverse_manifest
+ from tests.test_audit_regressions import config_for
+
+ config = config_for(tmp_path)
+ config.validation.enabled = False
+ config.validation.copy_project = True
+ config.validation.project_root = str(tmp_path)
+ config.orchestrator.max_parallel_functions = 2
+ config.orchestrator.max_attempts_per_function = 1
+ source = Path(config.project_profile.source_root) / "f.cpp"
+ original = "int a() { return 0; }\nint b() { return 0; }\n"
+ source.write_text(original)
+ targets = [FunctionTarget("1", "", "a"), FunctionTarget("2", "", "b")]
+ barrier = threading.Barrier(2)
+ snapshots, generations = [], []
+
+ def fake(target, cfg, *args, **kwargs):
+ snapshot = Path(cfg.project_profile.source_root) / "f.cpp"
+ snapshots.append(snapshot)
+ assert snapshot.read_text() == original
+ barrier.wait(timeout=10)
+ return ReversalResult(target, f"int {target.function_name}() {{ return 7; }}", success=True)
+
+ def validate(result, cfg, backend):
+ current_source = (Path(cfg.project_profile.source_root) / "f.cpp").read_text()
+ generations.append(current_source)
+ if result.target.function_name == "b":
+ assert "int a() { return 7; }" in current_source
+ result.success = False # A stale proposal cannot leak a provisional PASS.
+ return result
+
+ monkeypatch.setattr("re_agent.orchestrator.parallel.reverse_single", fake)
+ monkeypatch.setattr("re_agent.orchestrator.class_runner.validate_result", validate)
+ session = Session(config.output.session_file)
+ result = reverse_manifest(TargetPlan("a" * 64, [], targets), config, StubBackend(), None,
+ session, 2, provider_factory=lambda c: Mock())
+ assert [r.success for r in result] == [True, False]
+ assert source.read_text() == original
+ assert len(set(snapshots)) == 2 and all(not p.exists() for p in snapshots)
+ assert not session.is_completed("2")
+
+
+def test_fatal_configuration_stops_dispatch(setup):
+ config, backend, targets, session = setup
+
+ def factory(cfg):
+ raise ValueError("invalid provider settings")
+
+ assert reverse_parallel(targets, config, backend, session, factory, 4) == []
+ status = json.loads(session.path.with_suffix(".json.execution.json").read_text())
+ assert status["phase"] == "failed"
+ assert status["error"]["category"] == "configuration"
+ assert len(status["jobs"]) <= 2
+ assert all(session.attempt_count(t.address) == 0 for t in targets)
+
+
+def test_journal_recovers_crash_before_session_publication(setup, monkeypatch):
+ config, backend, targets, session = setup
+ monkeypatch.setattr("re_agent.orchestrator.parallel.reverse_single",
+ lambda target, *args, **kwargs: ReversalResult(target, "accepted", success=True))
+ publish = session.record_result_once
+ monkeypatch.setattr(session, "record_result_once", Mock(side_effect=OSError("interrupted write")))
+ with pytest.raises(OSError, match="interrupted write"):
+ reverse_parallel(targets[:1], config, backend, session, lambda c: Mock(), 1)
+ monkeypatch.setattr(session, "record_result_once", publish)
+ assert reverse_parallel(targets[:1], config, backend, session, lambda c: Mock(), 1) == []
+ assert session.is_completed(targets[0].address)
+ assert session.attempt_count(targets[0].address) == 1
+
+
+def test_scheduler_waits_for_dependencies_and_serializes_cycles(setup, monkeypatch):
+ from re_agent.core.models import XRef
+
+ config, backend, targets, session = setup
+ config.orchestrator.selection_strategy = "dependency-order"
+ edges = {"0x1": ["0x2", "0x3"], "0x2": ["0x4"], "0x3": ["0x4"], "0x4": []}
+ backend.xrefs_from.side_effect = lambda a: [XRef(b, "", "CALL") for b in edges[a]]
+ done, lock = set(), threading.Lock()
+
+ def fake(target, *args, **kwargs):
+ with lock:
+ assert set(edges[target.address]) <= done
+ done.add(target.address)
+ return ReversalResult(target, "ok", success=True)
+
+ monkeypatch.setattr("re_agent.orchestrator.parallel.reverse_single", fake)
+ assert len(reverse_parallel(targets, config, backend, session, lambda c: Mock(), 4)) == 4
+ assert len(done) == 4
+
+
+def test_actual_validation_uses_separate_writable_roots(setup, monkeypatch, tmp_path):
+ import sys
+
+ from re_agent.backend.stub import StubBackend
+ from re_agent.orchestrator.execution import Execution, executing
+ from re_agent.orchestrator.single import validate_result
+ from re_agent.verification.candidate import run_process as run
+
+ config, _, targets, _ = setup
+ targets = [FunctionTarget(t.address, "", t.function_name) for t in targets]
+ root = Path(config.project_profile.source_root)
+ root.mkdir()
+ (root / "f.cpp").write_text("int f1() { return 0; }\nint f2() { return 0; }\n")
+ config.validation.enabled = True
+ config.validation.copy_project = True
+ config.validation.parallel_safe = True
+ config.validation.project_root = str(tmp_path)
+ config.validation.trust_configured_commands = True
+ config.validation.build_commands = [[sys.executable, "-c",
+ "from pathlib import Path; p=Path('shared-build-name'); assert not p.exists(); p.write_text('ok')"]]
+ config.orchestrator.max_parallel_validations = 2
+ barrier = threading.Barrier(2)
+ roots = []
+
+ def invoke(args, **kwargs):
+ roots.append(kwargs["cwd"])
+ barrier.wait(10)
+ return run(args, **kwargs)
+
+ monkeypatch.setattr("re_agent.verification.candidate.run_process", invoke)
+ context = Execution(threading.Event(), threading.Semaphore(2), lambda *args: None)
+
+ def validate(target):
+ with executing(context):
+ return validate_result(ReversalResult(target, f"int {target.function_name}() {{ return 1; }}",
+ success=True), config, StubBackend())
+
+ with ThreadPoolExecutor() as pool:
+ results = list(pool.map(validate, targets[:2]))
+ assert all(r.success for r in results), [r.validation_verdict for r in results]
+ assert len(set(roots)) == 2
+ assert all(not Path(r).exists() for r in roots)
From 2cb76d384c2423a3e3ba6ff676514040ab3ee6aa Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 09:40:15 +1000
Subject: [PATCH 20/34] Keep API retries within explicit function call budgets
---
src/re_agent/llm/claude.py | 2 +-
src/re_agent/llm/openai_compat.py | 2 +-
tests/test_execution.py | 17 +++++++++++++++++
3 files changed, 19 insertions(+), 2 deletions(-)
diff --git a/src/re_agent/llm/claude.py b/src/re_agent/llm/claude.py
index 7f57fd65..58347ac9 100644
--- a/src/re_agent/llm/claude.py
+++ b/src/re_agent/llm/claude.py
@@ -31,7 +31,7 @@ def __init__(
temperature: float = 0.0,
timeout_s: int = 1800,
) -> None:
- self._client = anthropic.Anthropic(api_key=api_key, timeout=timeout_s)
+ self._client = anthropic.Anthropic(api_key=api_key, timeout=timeout_s, max_retries=0)
self.last_metadata: dict[str, Any] = {}
self._model = model
self._max_tokens = max_tokens
diff --git a/src/re_agent/llm/openai_compat.py b/src/re_agent/llm/openai_compat.py
index a9597e29..f871ea3c 100644
--- a/src/re_agent/llm/openai_compat.py
+++ b/src/re_agent/llm/openai_compat.py
@@ -37,7 +37,7 @@ def __init__(
timeout_s: int = 1800,
base_url: str | None = None,
) -> None:
- self._client = openai.OpenAI(api_key=api_key, base_url=base_url, timeout=timeout_s)
+ self._client = openai.OpenAI(api_key=api_key, base_url=base_url, timeout=timeout_s, max_retries=0)
self.last_metadata: dict[str, Any] = {}
self._model = model
self._max_tokens = max_tokens
diff --git a/tests/test_execution.py b/tests/test_execution.py
index a1b267d4..34a8bfa2 100644
--- a/tests/test_execution.py
+++ b/tests/test_execution.py
@@ -79,3 +79,20 @@ def invoke():
if psutil.pid_exists(pid):
# POSIX init may not have reaped an already-dead grandchild yet.
assert os.name != "nt" and psutil.Process(pid).status() == psutil.STATUS_ZOMBIE
+
+
+def test_api_transports_do_not_hide_retries_from_call_budget(monkeypatch):
+ from unittest.mock import Mock
+
+ from re_agent.llm.claude import ClaudeProvider
+ from re_agent.llm.openai_compat import OpenAIProvider
+
+ for path, provider in [("re_agent.llm.claude.anthropic.Anthropic", ClaudeProvider),
+ ("re_agent.llm.openai_compat.openai.OpenAI", OpenAIProvider)]:
+ constructor = Mock()
+ monkeypatch.setattr(path, constructor)
+ client = provider(api_key="test", timeout_s=7)
+ assert constructor.call_args.kwargs["max_retries"] == 0
+ assert constructor.call_args.kwargs["timeout"] == 7
+ client.close()
+ constructor.return_value.close.assert_called_once()
From 2ae170a17298e163445b837e26bd001d804570e8 Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 09:40:15 +1000
Subject: [PATCH 21/34] Report parallel worker progress with graceful and
forced stop controls
---
src/re_agent/monitor/index.html | 27 ++++++++++++++
src/re_agent/monitor/server.py | 51 ++++++++++++++++++++++++---
src/re_agent/orchestrator/parallel.py | 4 +++
tests/test_monitor.py | 41 +++++++++++++++++++++
4 files changed, 119 insertions(+), 4 deletions(-)
diff --git a/src/re_agent/monitor/index.html b/src/re_agent/monitor/index.html
index 0ad65da4..94396d85 100644
--- a/src/re_agent/monitor/index.html
+++ b/src/re_agent/monitor/index.html
@@ -268,6 +268,12 @@
Reconstruction, live.
Saved source remains separate from accepted implementations.
+
+
Function workers
+
+
Address
Stage
Calls
Elapsed
+
+
Recent results
@@ -276,6 +282,7 @@
Recent results
Address
Outcome
+
Validation
Rounds
@@ -341,6 +348,7 @@
Worker activity
for (let v of [
row.address,
row.success ? "Review passed" : "Needs attention",
+ row.validation_verdict || "Not recorded",
row.rounds_used || 0,
]) {
let td = document.createElement("td");
@@ -350,6 +358,25 @@
Worker activity
tr.children[1].className = row.success ? "green" : "amber";
el("results").append(tr);
}
+ const runs = s.executions || [];
+ el("parallel").hidden = !runs.length;
+ el("queue").textContent = runs.map(run =>
+ `${run.phase} · ${run.limit} worker limit · ${run.queued || 0} queued`
+ ).join(" / ");
+ el("jobs").replaceChildren();
+ for (const run of runs) for (const job of run.jobs || []) {
+ if (!["running", "proposed", "interrupted"].includes(job.state)) continue;
+ const tr = document.createElement("tr");
+ const end = job.finished || (run.alive ? Date.now() / 1000 : run.updated);
+ for (const value of [job.address, job.state === "running" ? job.stage || "starting" : job.state,
+ job.calls || 0, `${Math.max(0, Math.floor(end - job.started))}s`]) {
+ const td = document.createElement("td");
+ td.textContent = value;
+ tr.append(td);
+ }
+ el("jobs").append(tr);
+ }
+ el("stop").textContent = s.phase === "stopping" ? "Force stop" : "Stop run";
el("start").disabled = busy || s.active || !s.controls;
el("stop").disabled = busy || !s.active || !s.controls;
el("connection").textContent =
diff --git a/src/re_agent/monitor/server.py b/src/re_agent/monitor/server.py
index dc23ac7b..ed175696 100644
--- a/src/re_agent/monitor/server.py
+++ b/src/re_agent/monitor/server.py
@@ -102,12 +102,50 @@ def start(self) -> dict[str, Any]:
"cwd": str(self.work_dir), "phase": "running"})
return {"message": "Worker started", "pid": self.process.pid}
+ def executions(self) -> list[tuple[Path, dict[str, Any]]]:
+ """Read adjacent status files, verifying PID birth time before displaying activity."""
+ import psutil
+
+ paths = {p for pattern in self.session_globs
+ for p in self.work_dir.glob(pattern + ".execution.json") if p.is_file()}
+ result = []
+ for path in sorted(paths):
+ data = read_json(path)
+ if data.get("schema_version") != 1:
+ continue
+ alive = False
+ try:
+ process = psutil.Process(data["pid"])
+ alive = process.create_time() == data["created"] and process.status() != psutil.STATUS_ZOMBIE
+ except (psutil.Error, KeyError, TypeError):
+ pass
+ if not alive and data.get("phase") in {"running", "stopping"}:
+ data["phase"] = "interrupted"
+ for job in data.get("jobs", []):
+ if job.get("state") in {"running", "proposed"}:
+ job.update(state="interrupted", stage="interrupted")
+ data["alive"] = alive
+ result.append((path, data))
+ return result
+
def stop(self) -> dict[str, str]:
if not self.worker:
raise ValueError("Read-only monitor: no worker command configured")
with self.lock, file_lock(self.record):
self._adopt()
if self.active():
+ descendants = self.process.children(recursive=True)
+ owned = {self.process.pid, *(child.pid for child in descendants)}
+ executions = [(path, data) for path, data in self.executions()
+ if data.get("alive") and data.get("pid") in owned
+ and data.get("phase") in {"running", "stopping"}]
+ saved = read_json(self.record)
+ if executions and saved.get("phase") != "stopping":
+ for path, _data in executions:
+ path.with_suffix(".stop").touch()
+ saved.update(phase="stopping", stop_requested_at=time.time())
+ atomic_json(self.record, saved)
+ return {"message": "Stop requested; waiting for requests and cleanup. Stop again to force."}
if os.name == "nt":
subprocess.run(["taskkill", "/PID", str(self.process.pid), "/T", "/F"],
capture_output=True, timeout=20, check=True)
@@ -122,6 +160,10 @@ def stop(self) -> dict[str, str]:
# Descendants can ignore SIGTERM even when the parent exits.
with contextlib.suppress(ProcessLookupError):
os.killpg(self.process.pid, signal.SIGKILL)
+ for child in reversed(descendants):
+ with contextlib.suppress(self.psutil.Error):
+ child.kill()
+ self.psutil.wait_procs(descendants, timeout=5)
self.process.wait(timeout=20)
saved = read_json(self.record)
saved.update(phase="stopped", stopped_at=time.time())
@@ -143,7 +185,8 @@ def snapshot(self) -> dict[str, Any]:
data = read_json(path).get("functions")
if isinstance(data, dict):
rows = [{key: row.get(key) for key in
- ("address", "function_name", "success", "rounds_used", "timestamp")}
+ ("address", "function_name", "success", "rounds_used", "timestamp",
+ "verdict", "validation_verdict")}
for row in data.values()
if isinstance(row, dict) and isinstance(row.get("address"), str)]
self.cache[path] = (stamp, rows)
@@ -172,11 +215,11 @@ def snapshot(self) -> dict[str, Any]:
matching = saved.get("command") == self.worker and saved.get("cwd") == str(self.work_dir)
phase = "idle" if self.worker else "read-only"
if active:
- phase = "running"
+ phase = "stopping" if saved.get("phase") == "stopping" else "running"
elif self.worker and matching:
- phase = "stopped" if saved.get("phase") == "stopped" else "exited"
+ phase = "stopped" if saved.get("phase") in {"stopped", "stopping"} else "exited"
return {"active": active, "controls": bool(self.worker),
- "phase": phase,
+ "phase": phase, "executions": [data for _, data in self.executions()],
"targets": self.total, "completed": len(recent), "passed": passed,
"failed": len(recent) - passed, "rounds": rounds, "recent": recent[:18], "log": tail,
"updated": time.strftime("%H:%M:%S"), "output": str(self.work_dir)}
diff --git a/src/re_agent/orchestrator/parallel.py b/src/re_agent/orchestrator/parallel.py
index 083eb1a6..dec97765 100644
--- a/src/re_agent/orchestrator/parallel.py
+++ b/src/re_agent/orchestrator/parallel.py
@@ -309,7 +309,11 @@ def drain() -> None:
try:
with ThreadPoolExecutor(max_workers=config.orchestrator.max_parallel_functions) as executor:
try:
+ stopping_reported = False
while True:
+ if cancel.is_set() and not stopping_reported:
+ save()
+ stopping_reported = True
if stop.exists():
cancel.set()
drain()
diff --git a/tests/test_monitor.py b/tests/test_monitor.py
index 836677a5..2d39bc82 100644
--- a/tests/test_monitor.py
+++ b/tests/test_monitor.py
@@ -215,3 +215,44 @@ def test_packaged_html_uses_text_nodes_for_untrusted_results():
html = Path(server.__file__).with_name("index.html").read_text(encoding="utf-8")
assert "textContent" in html
assert "innerHTML" not in html
+
+
+def test_execution_status_reconciles_dead_process_and_preserves_verdicts(tmp_path):
+ session = tmp_path / "session.json"
+ save(session, {"1": {"address": "1", "success": False, "verdict": "PASS", "validation_verdict": "FAIL"}})
+ status = session.with_suffix(".json.execution.json")
+ status.write_text(json.dumps({"schema_version": 1, "phase": "running", "pid": 99999999, "created": 0,
+ "jobs": [{"address": "2", "state": "running", "stage": "reverser"}]}))
+ snapshot = Monitor(tmp_path, tmp_path / "state", ["session.json"]).snapshot()
+ assert snapshot["executions"][0]["phase"] == "interrupted"
+ assert snapshot["executions"][0]["jobs"][0]["stage"] == "interrupted"
+ assert snapshot["recent"][0]["verdict"] == "PASS"
+ assert snapshot["recent"][0]["validation_verdict"] == "FAIL"
+
+
+def test_parallel_stop_requests_cleanup_then_force_stops(tmp_path):
+ script = tmp_path / "worker.py"
+ script.write_text(
+ "import json, os, pathlib, psutil, time\n"
+ "pathlib.Path('session.json').write_text('{}')\n"
+ "pathlib.Path('session.json.execution.json').write_text(json.dumps({"
+ "'schema_version': 1, 'phase': 'running', 'pid': os.getpid(), "
+ "'created': psutil.Process().create_time(), 'jobs': []}))\n"
+ "time.sleep(60)\n"
+ )
+ monitor = Monitor(tmp_path, tmp_path / "state", ["session.json"], worker=[sys.executable, str(script)])
+ monitor.start()
+ try:
+ deadline = time.monotonic() + 10
+ while not monitor.executions() and time.monotonic() < deadline:
+ time.sleep(.02)
+ assert monitor.executions()
+ assert "Stop requested" in monitor.stop()["message"]
+ assert (tmp_path / "session.json.execution.stop").exists()
+ assert monitor.snapshot()["phase"] == "stopping"
+ assert monitor.active()
+ monitor.stop()
+ assert not monitor.active()
+ finally:
+ if monitor.active():
+ monitor.stop()
From 32205757dda65bdba8b27e412675a75c0fdeefe1 Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 09:43:18 +1000
Subject: [PATCH 22/34] Document parallel execution recovery and monitor
controls
---
.gitignore | 7 ++
README.md | 6 ++
docs/parallel-functions-plan.md | 4 +-
docs/parallel-functions.md | 123 ++++++++++++++++++++++++++++++++
4 files changed, 139 insertions(+), 1 deletion(-)
create mode 100644 docs/parallel-functions.md
diff --git a/.gitignore b/.gitignore
index c9027395..36ed1e3e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,3 +13,10 @@ build/
.DS_Store
.coverage
+
+# Local reconstruction state and generated reports
+/reports/re-agent/
+/re-agent-progress.json*
+*.execution.json
+*.execution.stop
+*.coordinator.lock
diff --git a/README.md b/README.md
index 862a87e1..ba35443c 100644
--- a/README.md
+++ b/README.md
@@ -543,3 +543,9 @@ mypy src
## License
MIT
+
+### Bounded parallel function processing
+
+Class and manifest runs support provider-independent concurrency with isolated
+workers, durable recovery, and live progress. Sequential execution remains the
+default. See [configuration and behavior](docs/parallel-functions.md).
diff --git a/docs/parallel-functions-plan.md b/docs/parallel-functions-plan.md
index c0872c97..b246b26a 100644
--- a/docs/parallel-functions-plan.md
+++ b/docs/parallel-functions-plan.md
@@ -1,6 +1,8 @@
# Bounded parallel function processing
-Status: implementation plan; no parallel execution is enabled by this document.
+Status: implemented. This document preserves the staged design and intermediate
+constraints. See [the usage guide](parallel-functions.md) for the final supported
+configuration, including cumulative promotion and parallel validation.
## Objective
diff --git a/docs/parallel-functions.md b/docs/parallel-functions.md
new file mode 100644
index 00000000..6b87f22a
--- /dev/null
+++ b/docs/parallel-functions.md
@@ -0,0 +1,123 @@
+# Parallel function processing
+
+Class and manifest reconstruction can use bounded concurrent workers with any
+registered provider. The default remains the existing sequential pipeline.
+
+## Configure a run
+
+```yaml
+orchestrator:
+ max_parallel_functions: 4
+ max_parallel_validations: 1
+```
+
+Or override the limits for an invocation:
+
+```sh
+re-agent --config project.yaml reverse --manifest targets.json --max-parallel-functions 4
+re-agent --config project.yaml reverse --class Example --max-parallel-functions 2
+```
+
+Function concurrency accepts integers from 1 through 32. This is a ReAgent limit;
+your provider's account, request limits, and available resources still apply.
+`--max-functions` remains a total function-attempt limit, including retries.
+`--address` remains a single-function operation.
+
+To run multiple validation commands concurrently, explicitly opt in:
+
+```yaml
+orchestrator:
+ max_parallel_functions: 4
+ max_parallel_validations: 2
+validation:
+ copy_project: true
+ parallel_safe: true
+```
+
+`parallel_safe` declares that your commands and reference harnesses can overlap
+without conflicting through external services, absolute output paths, or shared
+state. ReAgent gives each candidate its own writable project copy. A single
+validation lane is the default. Invalid limits fail before dispatch.
+
+## Execution and acceptance
+
+Each function gets fresh reverser and checker providers, conversations, model-call
+budgets, source-index state, artifact directories, and parity caches. Calls into a
+shared backend are serialized, including promotion validation. Provider factories
+must return fresh instances. Library users pass `provider_factory=create_provider`
+to `reverse_class` or `reverse_manifest`; passing only an existing provider remains
+supported at one worker.
+
+Addresses are normalized and deduplicated. The coordinator admits only as many
+jobs as there are slots and returns results in stable target order. Dependency
+selection uses strongly connected components: callees reach a terminal outcome
+before callers start, and members of a cycle run serially in deterministic order.
+A failed callee does not automatically reject its callers. In independent draft
+mode, generated callee code is not added to the source snapshot.
+
+When `validation.copy_project` and `orchestrator.cumulative_validation` are both
+true, workers propose changes against isolated snapshots of the accepted scratch
+project. Promotion follows dispatch order. Each provisional success must pass the
+same validation and parity gates against the latest accepted source before it is
+promoted and published as successful. A rejected proposal consumes an attempt;
+any retry uses the existing attempt and invocation limits. Original source files
+are never the cumulative scratch project. Full project copies and serial promotion
+can limit speedup for large projects.
+
+## Checkpoints and recovery
+
+The coordinator holds an exclusive session lease across identity binding,
+selection, and publication. A competing coordinator fails without starting jobs.
+Worker checkpoints and model-call reservations are acknowledged after atomic
+storage. Only the coordinator writes final session results.
+
+Per-attempt journals live below the configured report directory in
+`parallel//jobs/.json`; worker artifacts are in sibling
+attempt directories. Final publication is idempotent by attempt ID, including
+recovery after a crash between journal and session writes. Interrupted attempts
+retain completed rounds, previous feedback, and spent call budget. They resume
+with the remaining budget; exhaustion records a failure instead of granting free
+calls. A later attempt follows the normal attempt limit.
+
+Worker counts do not change semantic identity. Changing source, evidence, model,
+or acceptance policy invalidates incompatible accepted results. Changing other
+per-attempt execution policy can start a new parallel journal. Do not change
+input files during a run. Keep generated sessions, journals, source proposals,
+and benchmark reports outside version control.
+
+## Monitor and stop
+
+An adjacent `.execution.json` file reports concurrency, queue size,
+per-job stages/calls/times, and aggregate attempt states. The monitor reads this
+versioned file without importing the scheduler. Final session results remain the
+authority for accepted counts. Only active jobs and the latest 32 terminal jobs
+are included in the compact status payload; full history remains in the journals.
+
+The monitor verifies the process ID and creation time before showing activity.
+Dead workers display interrupted stages. Existing sequential/older sessions keep
+their previous display, and review verdicts and validation outcomes are separate.
+
+Stop first closes dispatch and requests cooperative cleanup. Waiting jobs and
+local subprocesses observe cancellation; cancelled late results are discarded.
+Synchronous API calls finish or reach their configured transport timeout. SDK
+retries are disabled so they cannot silently spend calls beyond the recorded
+budget. The monitor shows Stopping during cleanup; **Force stop** terminates the
+owned process tree when a request is unresponsive. Ctrl+C/SIGTERM also request
+orderly shutdown in parallel CLI runs. The CLI returns 130 after cancellation.
+
+Authentication failures with explicit HTTP 401/403 status, provider configuration
+failures, and inaccessible backend evidence stop dispatch and record a run-level
+category. Ordinary candidate failures leave independent jobs running.
+
+## Validation
+
+The regression suite exercises 1/2/4-worker full fake-provider pipelines,
+barrier-controlled overlap, unique providers/artifacts, retry caps, dependency
+ordering and cycles, lease contention, checkpoint crash recovery, spent budgets,
+late cancellation, real disposable subprocess trees, isolated validation commands,
+cumulative promotion, and monitor stop/reconnect behavior. CI runs Linux, Windows,
+and macOS jobs. Fake-provider timing checks measure scheduler overhead and overlap;
+they do not predict real model throughput or reconstruction quality.
+
+Grok Build uses this same scheduler. Grok-managed subagents are still disabled;
+this feature does not enable a provider-specific swarm mode.
From 63714d30307201de9a7426da7c9a0be7578c727e Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 09:44:48 +1000
Subject: [PATCH 23/34] Preserve backend enumeration fallback and provider
cleanup on startup
---
src/re_agent/cli/cmd_reverse.py | 16 +++++++++-------
src/re_agent/orchestrator/class_runner.py | 9 ++++++++-
tests/test_parallel.py | 15 +++++++++++++++
3 files changed, 32 insertions(+), 8 deletions(-)
diff --git a/src/re_agent/cli/cmd_reverse.py b/src/re_agent/cli/cmd_reverse.py
index 4ce0c17d..b04cf2b8 100644
--- a/src/re_agent/cli/cmd_reverse.py
+++ b/src/re_agent/cli/cmd_reverse.py
@@ -63,18 +63,20 @@ def cmd_reverse(args: argparse.Namespace) -> int:
# Parallel jobs construct their own providers; no unused shared clients.
reverser_llm = checker_llm = None
- if args.address or config.orchestrator.max_parallel_functions == 1:
- reverser_llm = create_provider(config.agents.reverser or config.llm)
- checker_llm = create_provider(config.agents.checker or config.llm)
backend = create_backend(config.backend)
session = Session(config.output.session_file)
from contextlib import ExitStack
with session.coordinator(), ExitStack() as resources:
- for provider in (reverser_llm, checker_llm):
- close = getattr(provider, "close", None)
- if callable(close):
- resources.callback(close)
+ if args.address or config.orchestrator.max_parallel_functions == 1:
+ providers = []
+ for role in (config.agents.reverser or config.llm, config.agents.checker or config.llm):
+ provider = create_provider(role)
+ providers.append(provider)
+ close = getattr(provider, "close", None)
+ if callable(close):
+ resources.callback(close)
+ reverser_llm, checker_llm = providers
session.bind(project_fingerprint(config))
if args.address:
diff --git a/src/re_agent/orchestrator/class_runner.py b/src/re_agent/orchestrator/class_runner.py
index 05d9975b..5ea092bc 100644
--- a/src/re_agent/orchestrator/class_runner.py
+++ b/src/re_agent/orchestrator/class_runner.py
@@ -139,7 +139,12 @@ def run(effective: ReAgentConfig, cumulative: bool = False) -> list[ReversalResu
return _reverse_class(class_name, effective, backend, llm, session, max_functions, checker_llm)
from re_agent.core.models import FunctionTarget
- entries = backend.remaining(class_name) or backend.unimplemented(class_name)
+ try:
+ entries = backend.remaining(class_name)
+ except Exception:
+ entries = []
+ if not entries:
+ entries = backend.unimplemented(class_name)
reverse_rank = effective.orchestrator.selection_strategy == "high-impact"
entries.sort(key=lambda f: (-f.caller_count if reverse_rank else f.caller_count, f.name, f.address))
targets = [FunctionTarget(f.address, f.class_name or class_name, f.name, f.caller_count) for f in entries]
@@ -206,6 +211,8 @@ def reverse_class(
checker_llm: LLMProvider | None = None, *, target_addresses: set[str] | None = None,
provider_factory: ProviderFactory | None = None,
) -> list[ReversalResult]:
+ if max_functions is not None and max_functions < 1:
+ raise ValueError("Function attempt limit must be positive")
session = session or Session(config.output.session_file)
with session.coordinator():
return _reverse_class_run(class_name, config, backend, llm, session, max_functions, checker_llm,
diff --git a/tests/test_parallel.py b/tests/test_parallel.py
index ae2267c6..b0a84d65 100644
--- a/tests/test_parallel.py
+++ b/tests/test_parallel.py
@@ -287,3 +287,18 @@ def validate(target):
assert all(r.success for r in results), [r.validation_verdict for r in results]
assert len(set(roots)) == 2
assert all(not Path(r).exists() for r in roots)
+
+
+def test_parallel_enumeration_falls_back_without_sharing_provider(setup, monkeypatch):
+ from re_agent.core.models import FunctionEntry
+ from re_agent.orchestrator.class_runner import reverse_class
+
+ config, backend, targets, session = setup
+ backend.remaining.side_effect = NotImplementedError("remaining unavailable")
+ backend.unimplemented.return_value = [FunctionEntry(t.address, t.function_name, t.class_name) for t in targets]
+ monkeypatch.setattr("re_agent.orchestrator.parallel.reverse_single",
+ lambda target, *args, **kwargs: ReversalResult(target, "ok", success=True))
+ result = reverse_class("C", config, backend, None, session, 4, provider_factory=lambda c: Mock())
+ assert len(result) == 4
+ with pytest.raises(ValueError, match="positive"):
+ reverse_class("C", config, backend, None, session, 0, provider_factory=lambda c: Mock())
From 3efaf15975de57674248151e0c23d3ac1813673f Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 09:44:48 +1000
Subject: [PATCH 24/34] Distinguish accepted results from review and validation
verdicts
---
src/re_agent/monitor/index.html | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/src/re_agent/monitor/index.html b/src/re_agent/monitor/index.html
index 94396d85..9d22ea5a 100644
--- a/src/re_agent/monitor/index.html
+++ b/src/re_agent/monitor/index.html
@@ -246,9 +246,9 @@
Reconstruction, live.
Saved final results
-
REVIEW PASSED
+
ACCEPTED
0
-
Not a proof of equivalence
+
Passed the configured acceptance gates
NEEDS ATTENTION
@@ -262,10 +262,10 @@
Reconstruction, live.
- Draft pass — not a verified build.
+ Acceptance follows the configured checks.
Review passes do not establish compilation or binary equivalence.
- Saved source remains separate from accepted implementations.
+ Inspect the validation outcome separately from model review.
@@ -282,13 +282,14 @@
Recent results
Address
Outcome
+
Review
Validation
Rounds
-
Results appear as functions finish.
+
Results appear as functions finish.
@@ -347,7 +348,8 @@
Worker activity
let tr = document.createElement("tr");
for (let v of [
row.address,
- row.success ? "Review passed" : "Needs attention",
+ row.success ? "Accepted" : "Needs attention",
+ row.verdict || "Not recorded",
row.validation_verdict || "Not recorded",
row.rounds_used || 0,
]) {
From 576ed397d403187dfd2451ba99c47a32f3472479 Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 09:46:00 +1000
Subject: [PATCH 25/34] Fail shared provider startup errors and preserve
recovered promotion outcomes
---
src/re_agent/orchestrator/parallel.py | 7 +++++--
tests/test_parallel.py | 5 +++--
2 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/src/re_agent/orchestrator/parallel.py b/src/re_agent/orchestrator/parallel.py
index dec97765..16bc5c90 100644
--- a/src/re_agent/orchestrator/parallel.py
+++ b/src/re_agent/orchestrator/parallel.py
@@ -220,7 +220,10 @@ def publish(job: str, result: ReversalResult) -> None:
if value["state"] in {"completed", "failed"}:
recovered = decode_result(value["result"])
if promote and recovered.success and not session.is_completed(recovered.target.address):
- recovered = promote(recovered, backend)
+ with executing(Execution(cancel, gate, lambda k, v: None)):
+ recovered = promote(recovered, backend)
+ value.update(state="completed" if recovered.success else "failed", result=_result_to_dict(recovered))
+ save(value["id"])
session.record_result_once(recovered)
else:
value["state"] = "interrupted"
@@ -261,7 +264,7 @@ def worker(job: str, snapshot: dict[str, Any]) -> ReversalResult:
try:
providers = [factory(isolated.agents.reverser or isolated.llm)]
providers.append(factory(isolated.agents.checker or isolated.llm))
- except (ValueError, OSError) as exc:
+ except Exception as exc:
emit(job, "fatal", {"category": "configuration", "message": str(exc)})
cancel.set()
raise
diff --git a/tests/test_parallel.py b/tests/test_parallel.py
index b0a84d65..a2d5f673 100644
--- a/tests/test_parallel.py
+++ b/tests/test_parallel.py
@@ -197,11 +197,12 @@ def validate(result, cfg, backend):
assert not session.is_completed("2")
-def test_fatal_configuration_stops_dispatch(setup):
+@pytest.mark.parametrize("error", [ValueError, RuntimeError])
+def test_fatal_configuration_stops_dispatch(setup, error):
config, backend, targets, session = setup
def factory(cfg):
- raise ValueError("invalid provider settings")
+ raise error("invalid provider settings")
assert reverse_parallel(targets, config, backend, session, factory, 4) == []
status = json.loads(session.path.with_suffix(".json.execution.json").read_text())
From 5ca7c81beaa89573d7caa9fe732983f26381a86d Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 09:47:34 +1000
Subject: [PATCH 26/34] Bind parallel library sessions before restoring
accepted sources
---
src/re_agent/orchestrator/class_runner.py | 6 ++++++
src/re_agent/orchestrator/parallel.py | 2 ++
tests/test_parallel.py | 13 +++++++++++++
3 files changed, 21 insertions(+)
diff --git a/src/re_agent/orchestrator/class_runner.py b/src/re_agent/orchestrator/class_runner.py
index 5ea092bc..15e69c55 100644
--- a/src/re_agent/orchestrator/class_runner.py
+++ b/src/re_agent/orchestrator/class_runner.py
@@ -215,5 +215,11 @@ def reverse_class(
raise ValueError("Function attempt limit must be positive")
session = session or Session(config.output.session_file)
with session.coordinator():
+ if config.orchestrator.max_parallel_functions > 1:
+ from re_agent.core.identity import project_fingerprint
+
+ identity = project_fingerprint(config)
+ if session.identity != identity:
+ session.bind(identity)
return _reverse_class_run(class_name, config, backend, llm, session, max_functions, checker_llm,
target_addresses=target_addresses, provider_factory=provider_factory)
diff --git a/src/re_agent/orchestrator/parallel.py b/src/re_agent/orchestrator/parallel.py
index 16bc5c90..d8a36149 100644
--- a/src/re_agent/orchestrator/parallel.py
+++ b/src/re_agent/orchestrator/parallel.py
@@ -143,6 +143,8 @@ def reverse_parallel(
from re_agent.orchestrator.execution import cancellation_signals
with session.coordinator(), cancellation_signals(cancel):
+ if session.identity != identity:
+ session.bind(identity)
return _run(targets, config, backend, session, provider_factory, limit, cancel, root, promote)
diff --git a/tests/test_parallel.py b/tests/test_parallel.py
index a2d5f673..f07efdf7 100644
--- a/tests/test_parallel.py
+++ b/tests/test_parallel.py
@@ -303,3 +303,16 @@ def test_parallel_enumeration_falls_back_without_sharing_provider(setup, monkeyp
assert len(result) == 4
with pytest.raises(ValueError, match="positive"):
reverse_class("C", config, backend, None, session, 0, provider_factory=lambda c: Mock())
+
+
+def test_library_parallel_run_rebinds_changed_semantic_identity(setup, monkeypatch):
+ config, backend, targets, session = setup
+ monkeypatch.setattr("re_agent.orchestrator.parallel.reverse_single",
+ lambda target, *args, **kwargs: ReversalResult(target, "ok", success=True))
+ assert len(reverse_parallel(targets[:1], config, backend, session, lambda c: Mock(), 1)) == 1
+ original = session.identity
+ config.llm.model = "new-model"
+ assert len(reverse_parallel(targets[:1], config, backend, session, lambda c: Mock(), 1)) == 1
+ assert original is not None and session.identity != original
+ assert session.attempt_count(targets[0].address) == 1
+ assert len(json.loads(session.path.read_text())["history"]) == 1
From 3eaa1baaacc470833418f91aeab75d4191a0f141 Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 10:14:44 +1000
Subject: [PATCH 27/34] Isolate Grok CLI configuration and remove tool exposure
from text requests
---
docs/grok-build.md | 31 ++++++++++++++++++---
src/re_agent/llm/grok_cli.py | 48 ++++++++++++++++++++++++++++-----
tests/test_llm/test_grok_cli.py | 39 ++++++++++++++++++++++++++-
3 files changed, 107 insertions(+), 11 deletions(-)
diff --git a/docs/grok-build.md b/docs/grok-build.md
index ce92bcef..cab35c07 100644
--- a/docs/grok-build.md
+++ b/docs/grok-build.md
@@ -5,18 +5,43 @@ Install Grok Build and authenticate using its own `grok login` workflow. ReAgent
```yaml
llm:
provider: grok-cli
- model: "" # Use the model configured in Grok Build, or supply a model from grok models.
+ model: "" # Use Grok's isolated default, or supply a model from grok models.
cli_path: grok # An absolute path to grok.exe also works on Windows.
timeout_s: 600
```
-Set `model` explicitly, including an empty string to use Grok's configured model. Omitting it inherits ReAgent's general default model, which is not a Grok model.
+Set `model` explicitly, including an empty string to use Grok's built-in default. Omitting it inherits ReAgent's general default model, which is not a Grok model.
The same configuration can be used under `agents.reverser` or `agents.checker` for mixed-provider runs. Optional `effort` is forwarded to Grok's `--effort` argument.
The adapter uses `--prompt-file` for large evidence inputs, native session IDs for repair conversations, and native JSON output (`text`, `sessionId`, `stopReason`). It requires a completed `end_turn` response and rejects empty, malformed, errored, or incomplete results. Usage and cost fields are retained in call metadata when supplied by Grok.
-Requests run in a temporary working directory with built-in tools disabled, a deny-all permission rule, web tools disabled, and subagents disabled. ReAgent supplies evidence and controls compilation/validation. The CLI still uses its own global configuration and stores its own sessions. Temporary prompt files are removed after each request; workspace cleanup does not delete Grok's persistent sessions.
+Requests run in a temporary working directory and a separate `GROK_HOME` per provider.
+`GROK_AUTH_PATH` points Grok at the existing login store (respecting an explicit override);
+ReAgent does not read, copy, or log credentials. Global user plugins, MCP configuration,
+and compatibility discovery are not inherited. Local `requirements.toml` and
+`managed_config.toml` are preserved, and system policies still apply. Custom user
+model/default/endpoint and authentication-helper configuration is not copied; select
+an explicit supported model and use supported environment-based endpoint/auth settings
+when necessary. An enforced managed integration may still initialize.
+
+The adapter selects `read_file` with `--tools`, then removes it together with the
+always-on MCP meta-tools using `--disallowed-tools read_file,search_tool,use_tool`.
+This deliberately avoids the empty allowlist behavior observed in Grok Build 1.0.25,
+which left built-ins available. A deny rule, disabled web search, and disabled
+subagents remain additional restrictions. `--max-turns 1` is unchanged. In a native
+reconstruction trace, the old filter produced tool calls and hit that limit; the
+explicit filter completed with `end_turn` and one model turn.
+
+Grok SDK retries are disabled in the isolated configuration. Native error JSON retains
+stop reason, turn count, and reported usage in call metadata, even on nonzero exit.
+Provider cleanup removes its temporary home and sessions; ReAgent's configured call
+logs and reports persist separately. Original Grok sessions and user settings are unchanged.
+
+References: [headless tool filtering](https://github.com/xai-org/grok-build/blob/main/crates/codegen/xai-grok-pager/docs/user-guide/14-headless-mode.md),
+[configuration](https://github.com/xai-org/grok-build/blob/main/crates/codegen/xai-grok-pager/docs/user-guide/05-configuration.md),
+and [native auth path resolution](https://github.com/xai-org/grok-build/blob/main/crates/codegen/xai-grok-login/src/storage.rs).
+
The shared subprocess runner terminates the child process tree on timeout. ReAgent's `max_tokens`, `temperature`, and API-key settings are not forwarded to this CLI. `max_budget_usd` is rejected because the adapter cannot enforce it through the supported CLI flags. Configure account/model limits in Grok Build.
diff --git a/src/re_agent/llm/grok_cli.py b/src/re_agent/llm/grok_cli.py
index a9cbda18..968724bb 100644
--- a/src/re_agent/llm/grok_cli.py
+++ b/src/re_agent/llm/grok_cli.py
@@ -2,6 +2,8 @@
from __future__ import annotations
import json
+import os
+import shutil
import subprocess
import tempfile
import uuid
@@ -22,9 +24,32 @@ def __init__(self, model: str = "", timeout_s: int = 1800,
self._grok_bin = grok_bin
self._effort = effort
self._workspace = tempfile.TemporaryDirectory(prefix="re-agent-grok-")
+ self._env = self._isolated_environment(Path(self._workspace.name))
self._conversations: dict[str, tuple[str, bool]] = {}
self.last_metadata: dict[str, Any] = {}
+ @staticmethod
+ def _isolated_environment(workspace: Path) -> dict[str, str]:
+ """Use Grok's supported home/auth overrides without copying credentials."""
+ original = Path(os.environ.get("GROK_HOME", str(Path.home() / ".grok"))).resolve()
+ home = workspace / "grok-home"
+ home.mkdir(mode=0o700)
+ # Keep local managed requirements as well as system-level policies.
+ for name in ("requirements.toml", "managed_config.toml"):
+ source = original / name
+ if source.is_file():
+ shutil.copy2(source, home / name)
+ settings = "[cli]\nauto_update = false\nuse_leader = false\n"
+ settings += "[models]\nmax_retries = 0\n"
+ settings += "[managed_mcps]\nenabled = false\ngateway_tools_enabled = false\n"
+ for vendor in ("claude", "cursor"):
+ settings += f"[compat.{vendor}]\n"
+ settings += "".join(f"{surface} = false\n" for surface in
+ ("mcps", "skills", "rules", "agents", "hooks", "sessions"))
+ (home / "config.toml").write_text(settings, encoding="utf-8")
+ return {**os.environ, "GROK_HOME": str(home),
+ "GROK_AUTH_PATH": os.environ.get("GROK_AUTH_PATH", str(original / "auth.json"))}
+
@property
def supports_conversations(self) -> bool:
return True
@@ -58,7 +83,10 @@ def _run(self, prompt: str, *, system: str | None = None, session_id: str | None
path = Path(self._workspace.name) / f"prompt-{uuid.uuid4().hex}.txt"
path.write_text(prompt, encoding="utf-8")
command = [self._grok_bin, "--prompt-file", str(path), "--output-format", "json",
- "--tools", "", "--deny", "*", "--no-subagents", "--disable-web-search",
+ # Empty --tools is not reliably tool-free in Grok Build. Select a
+ # real tool, remove it, and remove always-on MCP meta-tools.
+ "--tools", "read_file", "--disallowed-tools", "read_file,search_tool,use_tool",
+ "--deny", "*", "--no-subagents", "--disable-web-search",
"--max-turns", "1"]
for flag, value in [("--model", model if model is not None else self._model),
("--system-prompt-override", system), ("--session-id", session_id),
@@ -66,20 +94,26 @@ def _run(self, prompt: str, *, system: str | None = None, session_id: str | None
if value:
command.extend([flag, value])
try:
- proc = run_process(command, cwd=self._workspace.name, timeout_s=self._timeout_s)
- if proc.returncode != 0:
- detail = (proc.stderr.strip() or proc.stdout.strip())[-2000:]
- raise RuntimeError(f"Grok CLI failed with exit code {proc.returncode}: {detail}")
+ proc = run_process(command, cwd=self._workspace.name, env=self._env, timeout_s=self._timeout_s)
try:
payload = json.loads(proc.stdout)
except json.JSONDecodeError as exc:
+ if proc.returncode:
+ raise RuntimeError(
+ f"Grok CLI failed with exit code {proc.returncode}: {proc.stderr[-2000:]}"
+ ) from exc
raise RuntimeError("Grok CLI returned invalid JSON") from exc
if not isinstance(payload, dict):
raise RuntimeError("Grok CLI returned an unexpected JSON payload")
self.last_metadata = {key: payload[key] for key in
- ("sessionId", "requestId", "usage", "modelUsage", "total_cost_usd", "stopReason")
+ ("sessionId", "requestId", "usage", "modelUsage", "total_cost_usd",
+ "stopReason", "num_turns", "type", "message")
if key in payload}
- if payload.get("error") or payload.get("is_error") or payload.get("stopReason") != "end_turn":
+ if proc.returncode:
+ detail = payload.get("message") or proc.stderr.strip() or "request failed"
+ raise RuntimeError(f"Grok CLI failed with exit code {proc.returncode}: {str(detail)[-2000:]}")
+ if (payload.get("type") == "error" or payload.get("error") or payload.get("is_error")
+ or payload.get("stopReason") != "end_turn"):
raise RuntimeError(f"Grok CLI did not complete a text response: {payload.get('stopReason', 'error')}")
text = payload.get("text")
if not isinstance(text, str) or not text.strip():
diff --git a/tests/test_llm/test_grok_cli.py b/tests/test_llm/test_grok_cli.py
index 448c2b7e..45b1ec5f 100644
--- a/tests/test_llm/test_grok_cli.py
+++ b/tests/test_llm/test_grok_cli.py
@@ -34,8 +34,11 @@ def fake(command, **kwargs):
assert first[3] == prompt
assert prompt not in first[0]
assert first[0][0] == "grok path"
- assert first[0][first[0].index("--tools") + 1] == ""
+ assert first[0][first[0].index("--tools") + 1] == "read_file"
assert first[0][first[0].index("--deny") + 1] == "*"
+ assert first[0][first[0].index("--disallowed-tools") + 1] == "read_file,search_tool,use_tool"
+ assert first[1]["env"]["GROK_HOME"] != str(Path.home() / ".grok")
+ assert not (Path(first[1]["env"]["GROK_HOME"]) / "auth.json").exists()
assert "--no-subagents" in first[0] and "--disable-web-search" in first[0]
assert first[0][first[0].index("--max-turns") + 1] == "1"
assert first[0][first[0].index("--system-prompt-override") + 1] == "Preserve behavior"
@@ -93,3 +96,37 @@ def test_unknown_conversation_and_unsupported_budget():
GrokCLIProvider().resume("missing", "prompt")
with pytest.raises(ValueError, match="max_budget_usd"):
create_provider(LLMConfig(provider="grok-cli", max_budget_usd=1))
+
+
+def test_isolated_home_preserves_auth_location_and_local_requirements(tmp_path, monkeypatch):
+ original = tmp_path / "original"
+ original.mkdir()
+ (original / "auth.json").write_text("DO NOT COPY")
+ (original / "requirements.toml").write_text("[models]\nmax_retries = 0\n")
+ monkeypatch.setenv("GROK_HOME", str(original))
+ monkeypatch.delenv("GROK_AUTH_PATH", raising=False)
+ provider = GrokCLIProvider()
+ isolated = Path(provider._env["GROK_HOME"])
+ assert provider._env["GROK_AUTH_PATH"] == str(original / "auth.json")
+ assert not (isolated / "auth.json").exists()
+ assert (isolated / "requirements.toml").read_text() == (original / "requirements.toml").read_text()
+ assert "use_leader = false" in (isolated / "config.toml").read_text()
+ provider.close()
+ assert not isolated.exists()
+ assert (original / "auth.json").read_text() == "DO NOT COPY"
+
+
+def test_error_json_retains_native_stop_reason_and_usage(monkeypatch):
+ payload = {"type": "error", "message": "max turns reached", "stopReason": "cancelled",
+ "num_turns": 1, "usage": {"output_tokens": 100}}
+ monkeypatch.setattr("re_agent.llm.grok_cli.run_process", lambda *args, **kwargs:
+ subprocess.CompletedProcess([], 1, json.dumps(payload), "secondary warning"))
+ provider = GrokCLIProvider()
+ try:
+ with pytest.raises(RuntimeError, match="max turns reached"):
+ provider.send([])
+ assert provider.last_metadata["num_turns"] == 1
+ assert provider.last_metadata["stopReason"] == "cancelled"
+ assert provider.last_metadata["usage"]["output_tokens"] == 100
+ finally:
+ provider.close()
From fbb0022983e9eadd2c96fa5c78c11adb6509f6e6 Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 10:14:44 +1000
Subject: [PATCH 28/34] Bound model requests independently with cancellable
admission and budgeted retries
---
docs/parallel-functions.md | 24 +++++-
src/re_agent/cli/cmd_reverse.py | 2 +-
src/re_agent/cli/main.py | 1 +
src/re_agent/config/loader.py | 6 ++
src/re_agent/config/schema.py | 2 +
src/re_agent/llm/observed.py | 33 +++++++-
src/re_agent/monitor/index.html | 3 +-
src/re_agent/orchestrator/execution.py | 51 +++++++++++++
src/re_agent/orchestrator/parallel.py | 16 ++--
tests/test_config/test_parallel.py | 12 +++
tests/test_parallel.py | 1 +
tests/test_request_queue.py | 102 +++++++++++++++++++++++++
12 files changed, 244 insertions(+), 9 deletions(-)
create mode 100644 tests/test_request_queue.py
diff --git a/docs/parallel-functions.md b/docs/parallel-functions.md
index 6b87f22a..ac22b270 100644
--- a/docs/parallel-functions.md
+++ b/docs/parallel-functions.md
@@ -7,8 +7,10 @@ registered provider. The default remains the existing sequential pipeline.
```yaml
orchestrator:
- max_parallel_functions: 4
+ max_parallel_functions: 32
+ max_parallel_requests: 4
max_parallel_validations: 1
+ max_request_retries: 0
```
Or override the limits for an invocation:
@@ -121,3 +123,23 @@ they do not predict real model throughput or reconstruction quality.
Grok Build uses this same scheduler. Grok-managed subagents are still disabled;
this feature does not enable a provider-specific swarm mode.
+
+
+## Model request admission
+
+`max_parallel_requests` (1–32, default 1) is independent of function and validation
+limits. One FIFO queue serves reverser and checker calls across the run. Waiting
+for a slot is cancellable and does not spend a model call. Providers are invoked
+only after admission and durable call-budget reservation. Slots are released on
+success, failure, and cancellation. Each call log records `queue_wait_s` separately
+from its request `duration_s`; execution status exposes active and queued requests.
+The monitor displays both function and model-request counts.
+
+`max_request_retries` (0–3, default 0) enables bounded retries for explicit HTTP 429
+exceptions only. Every retry consumes the same function call budget. Retry-After
+seconds and exponential delay feed a shared cooldown, capped at 60 seconds; waits
+are cancellable and do not hold a request slot. Authentication, ambiguous network
+errors, and CLI diagnostics are not guessed to be retryable. Grok CLI errors do
+not currently expose a typed HTTP status and therefore are not automatically retried.
+Request retries apply to the parallel worker execution context. Changing the request
+concurrency limit preserves interrupted attempt budgets and accepted results.
diff --git a/src/re_agent/cli/cmd_reverse.py b/src/re_agent/cli/cmd_reverse.py
index b04cf2b8..cde20979 100644
--- a/src/re_agent/cli/cmd_reverse.py
+++ b/src/re_agent/cli/cmd_reverse.py
@@ -28,7 +28,7 @@ def cmd_reverse(args: argparse.Namespace) -> int:
if args.skip_parity:
config.parity.enabled = False
- for name in ("max_parallel_functions", "max_parallel_validations"):
+ for name in ("max_parallel_functions", "max_parallel_validations", "max_parallel_requests"):
value = getattr(args, name, None)
if value is not None:
setattr(config.orchestrator, name, value)
diff --git a/src/re_agent/cli/main.py b/src/re_agent/cli/main.py
index 671ea6e1..f71477ca 100644
--- a/src/re_agent/cli/main.py
+++ b/src/re_agent/cli/main.py
@@ -52,6 +52,7 @@ def build_parser() -> argparse.ArgumentParser:
rev_p.add_argument("--class", dest="class_name", help="Class name for class-level reversal")
rev_p.add_argument("--max-functions", type=int, default=None, help="Max functions per class")
rev_p.add_argument("--max-parallel-functions", type=int, default=None, help="Concurrent functions (1-32)")
+ rev_p.add_argument("--max-parallel-requests", type=int, default=None, help="Concurrent model requests (1-32)")
rev_p.add_argument("--max-parallel-validations", type=int, default=None, help="Concurrent isolated validations")
rev_p.add_argument("--max-rounds", type=int, default=None, help="Max review rounds per function")
rev_p.add_argument("--dry-run", action="store_true", help="Show plan without executing")
diff --git a/src/re_agent/config/loader.py b/src/re_agent/config/loader.py
index a619442d..5d7ca893 100644
--- a/src/re_agent/config/loader.py
+++ b/src/re_agent/config/loader.py
@@ -241,6 +241,12 @@ def load_config(
def validate_config(config: ReAgentConfig) -> None:
workers = config.orchestrator.max_parallel_functions
+ requests = config.orchestrator.max_parallel_requests
+ retries = config.orchestrator.max_request_retries
+ if type(requests) is not int or not 1 <= requests <= 32:
+ raise ValueError("max_parallel_requests must be an integer from 1 to 32")
+ if type(retries) is not int or not 0 <= retries <= 3:
+ raise ValueError("max_request_retries must be an integer from 0 to 3")
validations = config.orchestrator.max_parallel_validations
if type(workers) is not int or not 1 <= workers <= 32:
raise ValueError("max_parallel_functions must be an integer from 1 to 32")
diff --git a/src/re_agent/config/schema.py b/src/re_agent/config/schema.py
index 5e7c18b2..ab05defb 100644
--- a/src/re_agent/config/schema.py
+++ b/src/re_agent/config/schema.py
@@ -119,6 +119,8 @@ class OrchestratorConfig:
max_attempts_per_function: int = 3
max_parallel_functions: int = 1
max_parallel_validations: int = 1
+ max_parallel_requests: int = 1
+ max_request_retries: int = 0
@dataclass
diff --git a/src/re_agent/llm/observed.py b/src/re_agent/llm/observed.py
index 871ce838..69710ba3 100644
--- a/src/re_agent/llm/observed.py
+++ b/src/re_agent/llm/observed.py
@@ -4,6 +4,7 @@
import json
import time
+from contextlib import nullcontext, suppress
from dataclasses import asdict, dataclass, is_dataclass
from pathlib import Path
from typing import Any
@@ -50,13 +51,41 @@ def resume(self, conversation_id: str, message: str) -> str:
def _call(self, **request: Any) -> str:
from re_agent.orchestrator.execution import current, progress
+ context = current()
+ retries = context.request_retries if context else 0
+ for attempt in range(retries + 1):
+ progress("waiting-for-model")
+ admission = context.requests.acquire(context) if context and context.requests else nullcontext(0.0)
+ try:
+ with admission as waited:
+ return self._call_once(waited, **request)
+ except Exception as exc:
+ # Only explicit rate limits are safe to retry automatically.
+ if getattr(exc, "status_code", None) != 429:
+ raise
+ delay = min(60.0, 2.0 ** attempt)
+ with suppress(AttributeError, TypeError, ValueError):
+ headers = getattr(getattr(exc, "response", None), "headers", {})
+ delay = min(60.0, max(delay, float(headers.get("retry-after", delay))))
+ if context and context.requests:
+ context.requests.defer(delay)
+ if attempt == retries:
+ raise
+ progress("rate-limit-backoff")
+ if context and context.cancel.wait(delay):
+ context.check()
+ raise AssertionError("unreachable")
+
+ def _call_once(self, waited: float, **request: Any) -> str:
+ from re_agent.orchestrator.execution import current, progress
+
progress(self.role)
context = current()
number = self.budget.consume()
if context:
context.emit("call", self.role)
start = time.monotonic()
- event: dict[str, Any] = {"role": self.role, "call": number, "request": request}
+ event: dict[str, Any] = {"role": self.role, "call": number, "request": request, "queue_wait_s": waited}
try:
if "messages" in request:
response = self.provider.send(request["messages"], **request["kwargs"])
@@ -75,6 +104,8 @@ def _call(self, **request: Any) -> str:
finally:
event["duration_s"] = time.monotonic() - start
event["metadata"] = self.last_metadata
+ if context:
+ context.emit("request_timing", {"queue_wait_s": waited, "request_s": event["duration_s"]})
if self.log_dir:
def encode(value: Any) -> Any:
diff --git a/src/re_agent/monitor/index.html b/src/re_agent/monitor/index.html
index 9d22ea5a..691b701b 100644
--- a/src/re_agent/monitor/index.html
+++ b/src/re_agent/monitor/index.html
@@ -363,7 +363,8 @@
Worker activity
const runs = s.executions || [];
el("parallel").hidden = !runs.length;
el("queue").textContent = runs.map(run =>
- `${run.phase} · ${run.limit} worker limit · ${run.queued || 0} queued`
+ `${run.phase} · ${run.limit} function limit · ${run.queued || 0} functions queued` +
+ (run.requests ? ` · ${run.requests.active}/${run.requests.limit} model requests · ${run.requests.queued} waiting` : "")
).join(" / ");
el("jobs").replaceChildren();
for (const run of runs) for (const job of run.jobs || []) {
diff --git a/src/re_agent/orchestrator/execution.py b/src/re_agent/orchestrator/execution.py
index 0df6ebac..60dc6be6 100644
--- a/src/re_agent/orchestrator/execution.py
+++ b/src/re_agent/orchestrator/execution.py
@@ -2,6 +2,8 @@
from __future__ import annotations
import threading
+import time
+from collections import deque
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from dataclasses import dataclass
@@ -17,6 +19,8 @@ class Execution:
cancel: threading.Event
validations: threading.Semaphore
emit: Callable[[str, Any], None]
+ requests: RequestQueue | None = None
+ request_retries: int = 0
def fail(self, category: str, message: str) -> None:
self.emit("fatal", {"category": category, "message": message})
@@ -85,3 +89,50 @@ def cancellation_signals(cancel: threading.Event) -> Iterator[None]:
finally:
for sig, handler in previous.items():
signal.signal(sig, handler)
+
+
+class RequestQueue:
+ """FIFO admission and shared rate-limit cooldown, independent of function slots."""
+
+ def __init__(self, limit: int) -> None:
+ self.limit = limit
+ self.active = 0
+ self.waiting: deque[object] = deque()
+ self.condition = threading.Condition()
+ self.ready_at = 0.0
+
+ def snapshot(self) -> dict[str, int]:
+ with self.condition:
+ return {"limit": self.limit, "active": self.active, "queued": len(self.waiting)}
+
+ def defer(self, seconds: float) -> None:
+ with self.condition:
+ self.ready_at = max(self.ready_at, time.monotonic() + min(60, max(0, seconds)))
+ self.condition.notify_all()
+
+ @contextmanager
+ def acquire(self, context: Execution) -> Iterator[float]:
+ ticket = object()
+ start = time.monotonic()
+ with self.condition:
+ self.waiting.append(ticket)
+ try:
+ while True:
+ context.check()
+ if (self.waiting[0] is ticket and self.active < self.limit and
+ time.monotonic() >= self.ready_at):
+ self.waiting.popleft()
+ self.active += 1
+ self.condition.notify_all()
+ break
+ self.condition.wait(.1)
+ except BaseException:
+ self.waiting.remove(ticket)
+ self.condition.notify_all()
+ raise
+ try:
+ yield time.monotonic() - start
+ finally:
+ with self.condition:
+ self.active -= 1
+ self.condition.notify_all()
diff --git a/src/re_agent/orchestrator/parallel.py b/src/re_agent/orchestrator/parallel.py
index d8a36149..79149458 100644
--- a/src/re_agent/orchestrator/parallel.py
+++ b/src/re_agent/orchestrator/parallel.py
@@ -33,7 +33,7 @@
from re_agent.core.session import Session
from re_agent.llm.protocol import LLMProvider
from re_agent.orchestrator.dependencies import prerequisites
-from re_agent.orchestrator.execution import Cancelled, Execution, executing
+from re_agent.orchestrator.execution import Cancelled, Execution, RequestQueue, executing
from re_agent.orchestrator.single import reverse_single
from re_agent.reports.formatter import _result_to_dict
from re_agent.utils.address import normalize_address
@@ -130,7 +130,8 @@ def reverse_parallel(
if limit < 1:
raise ValueError("Function attempt limit must be positive")
policy = asdict(config.orchestrator)
- for key in ("max_parallel_functions", "max_parallel_validations", "max_functions_per_class"):
+ for key in ("max_parallel_functions", "max_parallel_validations", "max_parallel_requests",
+ "max_functions_per_class"):
policy.pop(key, None)
models = [asdict(c) for c in (config.agents.reverser or config.llm, config.agents.checker or config.llm)]
for model in models:
@@ -169,6 +170,7 @@ def _run(targets: list[FunctionTarget], config: ReAgentConfig, backend: REBacken
stop.unlink(missing_ok=True)
events: queue.Queue[Event] = queue.Queue()
gate = threading.Semaphore(config.orchestrator.max_parallel_validations)
+ requests = RequestQueue(config.orchestrator.max_parallel_requests)
backend_lock = threading.Lock()
generation_lock = threading.Lock()
futures: dict[Future[ReversalResult], str] = {}
@@ -197,7 +199,7 @@ def save(changed: str | None = None) -> None:
visible += [v for v in jobs.values() if v["state"] not in {"running", "proposed"}][-32:]
atomic_json(status, {"schema_version": 1, "phase": "stopping" if cancel.is_set() else "running",
"pid": os.getpid(), "created": created, "run_id": root.name,
- "limit": config.orchestrator.max_parallel_functions,
+ "limit": config.orchestrator.max_parallel_functions, "requests": requests.snapshot(),
"queued": sum(a not in completed and a not in active and
attempts.get(a, 0) < config.orchestrator.max_attempts_per_function for a in order),
"jobs": [{k: v for k, v in j.items() if k not in {"checkpoint", "result"}} for j in visible],
@@ -236,7 +238,7 @@ def publish(job: str, result: ReversalResult) -> None:
def emit(job: str, kind: str, value: Any) -> None:
event = Event(job, kind, value, threading.Event())
events.put(event)
- if kind in {"stage", "fatal"}:
+ if kind in {"stage", "fatal", "request_timing"}:
return
while not event.ack.wait(.1):
if cancel.is_set():
@@ -261,7 +263,8 @@ def worker(job: str, snapshot: dict[str, Any]) -> ReversalResult:
try:
from re_agent.orchestrator.snapshot import project_snapshot
- with (executing(Execution(cancel, gate, lambda kind, value: emit(job, kind, value))),
+ with (executing(Execution(cancel, gate, lambda kind, value: emit(job, kind, value),
+ requests, config.orchestrator.max_request_retries)),
project_snapshot(isolated, generation_lock) if promote else nullcontext(isolated) as isolated):
try:
providers = [factory(isolated.agents.reverser or isolated.llm)]
@@ -302,6 +305,9 @@ def drain() -> None:
value.update(checkpoint=_result_to_dict(event.value), rounds=event.value.rounds_used)
event.value.run_id = event.job
session.record_checkpoint(event.value)
+ elif event.kind == "request_timing":
+ for key, duration in event.value.items():
+ value[key] = value.get(key, 0.0) + duration
elif event.kind == "call":
value["calls"] = value.get("calls", 0) + 1
else:
diff --git a/tests/test_config/test_parallel.py b/tests/test_config/test_parallel.py
index ec1ae97f..ae35d76b 100644
--- a/tests/test_config/test_parallel.py
+++ b/tests/test_config/test_parallel.py
@@ -44,3 +44,15 @@ def test_semantic_identity_tracks_models_and_acceptance_not_scheduling(tmp_path)
config.llm.model = ReAgentConfig().llm.model
config.orchestrator.objective_verifier_enabled = not config.orchestrator.objective_verifier_enabled
assert project_fingerprint(config) != original
+
+
+@pytest.mark.parametrize("name,value", [("max_parallel_requests", 0), ("max_parallel_requests", 33),
+ ("max_parallel_requests", True), ("max_request_retries", -1),
+ ("max_request_retries", 4)])
+def test_invalid_request_controls(name, value):
+ from re_agent.config.loader import validate_config
+ from re_agent.config.schema import ReAgentConfig
+ config = ReAgentConfig()
+ setattr(config.orchestrator, name, value)
+ with pytest.raises(ValueError, match=name):
+ validate_config(config)
diff --git a/tests/test_parallel.py b/tests/test_parallel.py
index f07efdf7..9722ab47 100644
--- a/tests/test_parallel.py
+++ b/tests/test_parallel.py
@@ -84,6 +84,7 @@ def resumed(target, cfg, backend, llm, **kwargs):
monkeypatch.setattr("re_agent.orchestrator.parallel.reverse_single", resumed)
config.orchestrator.max_parallel_functions = 4
+ config.orchestrator.max_parallel_requests = 2
result = reverse_parallel(targets[:1], config, backend, session, lambda c: Mock(), 1)
assert budgets == [(config.orchestrator.max_llm_calls_per_function - 1,
config.orchestrator.max_review_rounds - 1)]
diff --git a/tests/test_request_queue.py b/tests/test_request_queue.py
new file mode 100644
index 00000000..257ee873
--- /dev/null
+++ b/tests/test_request_queue.py
@@ -0,0 +1,102 @@
+"""Real thread overlap, cancellation, and explicit request-budget tests."""
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from unittest.mock import Mock
+
+import pytest
+
+from re_agent.llm.observed import CallBudget, ObservedProvider
+from re_agent.llm.protocol import Message
+from re_agent.orchestrator.execution import Cancelled, Execution, RequestQueue, executing
+
+
+def test_32_functions_share_four_request_slots():
+ queue = RequestQueue(4)
+ barrier = threading.Barrier(4)
+ lock = threading.Lock()
+ active = peak = 0
+
+ def send(*args, **kwargs):
+ nonlocal active, peak
+ with lock:
+ active += 1
+ peak = max(peak, active)
+ barrier.wait(10)
+ with lock:
+ active -= 1
+ return "ok"
+
+ def job(_):
+ context = Execution(threading.Event(), threading.Semaphore(1), lambda *args: None, queue)
+ with executing(context):
+ budget = CallBudget(1)
+ assert ObservedProvider(Mock(send=send), budget, "reverser", None).send([Message("user", "test")]) == "ok"
+ assert budget.used == 1
+
+ with ThreadPoolExecutor(max_workers=32) as pool:
+ list(pool.map(job, range(32)))
+ assert peak == 4
+ assert queue.snapshot() == {"limit": 4, "active": 0, "queued": 0}
+
+
+def test_cancel_queued_request_does_not_spend_budget():
+ queue = RequestQueue(1)
+ cancel = threading.Event()
+ context = Execution(cancel, threading.Semaphore(1), lambda *args: None, queue)
+ queued = threading.Event()
+ context.emit = lambda *args: queued.set()
+ provider, budget = Mock(), CallBudget(1)
+
+ def waiting():
+ with executing(context):
+ ObservedProvider(provider, budget, "checker", None).send([])
+
+ with (queue.acquire(Execution(threading.Event(), threading.Semaphore(1), lambda *args: None)),
+ ThreadPoolExecutor() as pool):
+ future = pool.submit(waiting)
+ assert queued.wait(10)
+ cancel.set()
+ with pytest.raises(Cancelled):
+ future.result(timeout=10)
+ assert budget.used == 0
+ provider.send.assert_not_called()
+ assert queue.snapshot()["queued"] == 0
+
+
+class RateLimit(Exception):
+ status_code = 429
+
+
+@pytest.mark.parametrize("limit,expected_calls", [(3, 3), (2, 2)])
+def test_rate_limit_retries_spend_budget_and_release_slots(monkeypatch, limit, expected_calls):
+ queue = RequestQueue(1)
+ cancel = threading.Event()
+ # Skip real backoff in this test; still verify the shared cooldown is set.
+ monkeypatch.setattr(cancel, "wait", lambda seconds: False)
+ monkeypatch.setattr(queue, "defer", Mock())
+ events = []
+ context = Execution(cancel, threading.Semaphore(1), lambda *args: events.append(args), queue, 2)
+ provider = Mock(send=Mock(side_effect=[RateLimit(), RateLimit(), "ok"]))
+ budget = CallBudget(limit)
+ with executing(context):
+ observed = ObservedProvider(provider, budget, "reverser", None)
+ if limit == 3:
+ assert observed.send([]) == "ok"
+ else:
+ with pytest.raises(RuntimeError, match="budget exhausted"):
+ observed.send([])
+ assert provider.send.call_count == expected_calls
+ assert budget.used == expected_calls
+ assert queue.snapshot()["active"] == 0
+ assert queue.defer.call_count == 2
+ assert len([e for e in events if e[0] == "request_timing"]) == expected_calls
+
+
+def test_failure_is_not_retried_and_slot_is_released():
+ queue = RequestQueue(1)
+ context = Execution(threading.Event(), threading.Semaphore(1), lambda *args: None, queue, 3)
+ provider = Mock(send=Mock(side_effect=ValueError("invalid output")))
+ with executing(context), pytest.raises(ValueError):
+ ObservedProvider(provider, CallBudget(4), "checker", None).send([])
+ assert provider.send.call_count == 1
+ assert queue.snapshot()["active"] == 0
From 4ccadfde09a8792bde3109b1b4ca5a7d9e6c920f Mon Sep 17 00:00:00 2001
From: samuel <13449963+samuelGrahame@users.noreply.github.com>
Date: Thu, 10 Sep 2026 11:16:27 +1000
Subject: [PATCH 29/34] Integrate external batch progress into the existing
live monitor
---
docs/live-monitor.md | 27 +++++++++++
src/re_agent/cli/cmd_monitor.py | 2 +-
src/re_agent/cli/main.py | 2 +
src/re_agent/monitor/index.html | 69 ++++++++++++++++++++------
src/re_agent/monitor/server.py | 85 ++++++++++++++++++++++++++++++++-
tests/test_monitor.py | 39 +++++++++++++++
6 files changed, 207 insertions(+), 17 deletions(-)
diff --git a/docs/live-monitor.md b/docs/live-monitor.md
index 75ca2cf5..713bffa4 100644
--- a/docs/live-monitor.md
+++ b/docs/live-monitor.md
@@ -42,3 +42,30 @@ Control records and worker stdout/stderr live in `reports/monitor` by default; o
## Local access
The server binds only to IPv4 loopback. Host-header checks reject DNS-rebinding requests. Control endpoints require a random per-host token and reject cross-origin requests. HTTP clients cannot supply a different worker command or arbitrary file path. No external scripts, fonts, telemetry or services are used by the dashboard. This is a local tool, not a multi-user network service.
+# External batch progress
+
+Use the existing dashboard for a separately launched batch runner:
+
+```console
+re-agent monitor --work-dir /path/to/run --progress-file status.json --stop-file STOP --log-glob "batch-*/stderr.log"
+```
+
+Both file paths must stay within the working directory. The monitor does not adopt
+or restart the external process. Stop creates the configured cooperative signal;
+the runner must watch it and cancel its own children. Omit `--stop-file` for
+read-only reporting. External progress cannot be combined with `--worker`.
+
+The progress JSON uses `phase`, Unix-second `started`, `updated` and
+`batch_started` timestamps, and integer `total`, `completed`, `compiled`, `failed`,
+`verified`, `batch`, `batches`, `child_started`, `child_returned` and
+`active_children` counters. `recent` contains rows with `address`, `compiled` and
+optional `diagnostic`; optional `error` describes a run-level failure.
+Active phases are `opening-analysis`, `exporting-evidence`, `native-subagents` and
+`validating-candidates`. Updates older than 60 seconds are marked stale; this is
+a freshness indication, not proof that a process has exited.
+
+Compiled drafts are explicitly distinguished from accepted reconstructions.
+Child counts describe starts and collected results, not measured model-request
+concurrency. The existing layout displays elapsed time, throughput, batch
+progress, diagnostics and source-data age. Browser refreshes run every two
+seconds, do not overlap, and time out after eight seconds.
diff --git a/src/re_agent/cli/cmd_monitor.py b/src/re_agent/cli/cmd_monitor.py
index 67fea363..6930b1d3 100644
--- a/src/re_agent/cli/cmd_monitor.py
+++ b/src/re_agent/cli/cmd_monitor.py
@@ -13,7 +13,7 @@ def cmd_monitor(args: argparse.Namespace) -> int:
raise ValueError("Port must be between 0 and 65535")
root = Path(args.work_dir).resolve()
monitor = Monitor(root, root / args.state_dir, args.session_glob or ["re-agent-progress.json"],
- args.log_glob, args.total, args.worker)
+ args.log_glob, args.total, args.worker, args.progress_file, args.stop_file)
with make_server(monitor, args.port) as server:
print(f"Live monitor: http://127.0.0.1:{server.server_port}", flush=True)
print("Closing the monitor does not stop the worker. Use Stop run to terminate it.", flush=True)
diff --git a/src/re_agent/cli/main.py b/src/re_agent/cli/main.py
index f71477ca..37979b16 100644
--- a/src/re_agent/cli/main.py
+++ b/src/re_agent/cli/main.py
@@ -22,6 +22,8 @@ def build_parser() -> argparse.ArgumentParser:
monitor_p.add_argument("--log-glob", help="Relative activity-log pattern; shows most recently modified log")
monitor_p.add_argument("--total", type=int, default=0, help="Planned function count; 0 means unknown")
monitor_p.add_argument("--port", type=int, default=8765)
+ monitor_p.add_argument("--progress-file", help="Relative external batch progress JSON file")
+ monitor_p.add_argument("--stop-file", help="Relative cooperative stop file for the external runner")
monitor_p.add_argument("--worker", nargs=argparse.REMAINDER, help="Optional worker argv; must be the last option")
doctor_p = sub.add_parser("doctor", help="Check configuration and exported evidence without LLM calls")
diff --git a/src/re_agent/monitor/index.html b/src/re_agent/monitor/index.html
index 691b701b..b815fa35 100644
--- a/src/re_agent/monitor/index.html
+++ b/src/re_agent/monitor/index.html
@@ -86,6 +86,8 @@
color: #b9c8dc;
}
.panel {
+ min-width: 0;
+ overflow-x: auto;
background: #121b28;
border: 1px solid #253247;
border-radius: 14px;
@@ -146,6 +148,7 @@
}
td,
th {
+ overflow-wrap: anywhere;
text-align: left;
padding: 10px 6px;
border-bottom: 1px solid #253247;
@@ -195,6 +198,7 @@
gap: 9px;
}
@media (max-width: 800px) {
+ .row { flex-wrap: wrap; }
.grid {
grid-template-columns: 1fr;
}
@@ -209,6 +213,14 @@
padding: 24px 16px;
}
}
+ @media (max-width: 420px) {
+ .stat { padding: 12px; }
+ .panel { padding: 14px; }
+ .number { font-size: 27px; }
+ }
+ @media (prefers-reduced-motion: reduce) {
+ .fill { transition: none; }
+ }
@@ -246,9 +258,9 @@
Reconstruction, live.
Saved final results
-
ACCEPTED
+
ACCEPTED
0
-
Passed the configured acceptance gates
+
Passed the configured acceptance gates
NEEDS ATTENTION
@@ -263,11 +275,16 @@
Reconstruction, live.
Acceptance follows the configured checks.
-
+
Review passes do not establish compilation or binary equivalence.
Inspect the validation outcome separately from model review.
+
+
Run details
+
+
Awaiting collection includes finished children whose results the parent has not collected yet.
+
Function workers
@@ -305,7 +322,7 @@
Worker activity
-
-
Run details
+
+ Run details & throughput
Awaiting collection includes finished children whose results the parent has not collected yet.
@@ -291,6 +306,16 @@
Function workers
Address
Stage
Calls
Elapsed
+
+
Agent workspace
Code & execution trace
+
+
+
Loading agents…
+
Unattributed native tokens stay in the shared stream. Individual agent responses appear when identified by the provider. Recent history is bounded.
+
Candidate code
+
Activity / errors
+ Full response / evidence notes
+
Recent results
@@ -334,19 +359,29 @@
Worker activity
let busy = false;
const el = (id) => document.getElementById(id);
const fmt = (n) => Number(n).toLocaleString();
- let refreshing = false;
- async function refresh() {
- if (refreshing) return;
- refreshing = true;
- const controller = new AbortController();
- const timeout = setTimeout(() => controller.abort(), 8000);
+ let socket;
+ let retryDelay = 1000;
+ let streamWatchdog;
+ let agentRows = [];
+ function render(s) {
try {
- let r = await fetch("/api/status", {signal: controller.signal, cache: "no-store"});
- if (!r.ok) throw Error("Host unavailable");
- let s = await r.json();
let pct = s.targets
? Math.min(100, (100 * s.completed) / s.targets)
: 0;
+ const batchSelection = el("batch-select").value;
+ el("batch-select").replaceChildren(new Option("Live batch", ""));
+ for (const source of s.event_sources || []) el("batch-select").append(new Option(source.split("/")[0], source));
+ el("batch-select").value = batchSelection;
+ if (!batchSelection) agentRows = s.agents || [];
+ const selected = el("agent-select").value;
+ el("agent-select").replaceChildren();
+ for (const agent of agentRows) {
+ const option = document.createElement("option"); option.value = agent.id;
+ option.textContent = `${agent.label} · ${agent.status}`; el("agent-select").append(option);
+ }
+ if (agentRows.some(a => a.id === selected)) el("agent-select").value = selected;
+ else if (agentRows.some(a => a.id.endsWith(":shared"))) el("agent-select").value = agentRows.find(a => a.id.endsWith(":shared")).id;
+ renderAgent();
el("phase").textContent = s.phase.replaceAll("-", " ");
el("passed-label").textContent = s.passed_label || "ACCEPTED";
el("control-note").textContent = s.can_start === false
@@ -366,7 +401,7 @@
Worker activity
: `${fmt(s.completed)} attempted`;
for (let k of ["completed", "passed", "failed", "rounds"])
el(k).textContent = fmt(s[k]);
- el("log").textContent = [s.diagnostics, s.log].filter(Boolean).join("\n\n") || "Waiting for worker output…";
+ el("log").textContent = [s.diagnostics, s.log].filter(Boolean).join("\n\n").replace(/\x1b\[[0-9;]*m/g, "") || "Waiting for worker output…";
el("run-details").hidden = !(s.details || []).length;
el("detail-rows").replaceChildren();
for (const detail of s.details || []) {
@@ -420,7 +455,7 @@