Make agent and benchmark progress crash-safe

README.md

@@ -227,6 +227,7 @@ python -m twine upload dist/*
 1. **评测运行器显式授权**：只有受信任的 SWE-bench runner 实例能调用危险 shell 的授权入口；环境变量不能关闭普通用户的安全检查，forbidden 命令始终拒绝。
 2. **Prompt 收紧**：强制最小改动、禁止安装依赖/修改配置、要求验证后再结束。
 3. **合规修正**：移除 goal description 中的 `FAIL_TO_PASS` 测试名泄露，agent 只看 issue 描述，验收测试由评估 harness 在不可见情况下运行。
+4. **可恢复结果**：每个系统开始和结束时都会原子保存独立状态；环境故障不会计入答错，恢复运行时优先重新验收已保存的 patch，不重复调用模型。
 
 ### 复现
 

agent/atomic_io.py

@@ -0,0 +1,45 @@
+"""Crash-safe helpers for small state and report files."""
+
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+from pathlib import Path
+from typing import Any
+
+
+def atomic_write_text(path: str | Path, content: str, *, mode: int = 0o600) -> None:
+    """Replace *path* atomically after flushing the new contents to disk."""
+    target = Path(path)
+    target.parent.mkdir(parents=True, exist_ok=True)
+    fd, temporary_name = tempfile.mkstemp(prefix=f".{target.name}.", dir=target.parent)
+    temporary = Path(temporary_name)
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as stream:
+            stream.write(content)
+            stream.flush()
+            os.fsync(stream.fileno())
+        temporary.chmod(mode)
+        os.replace(temporary, target)
+        try:
+            directory_fd = os.open(target.parent, os.O_RDONLY)
+            try:
+                os.fsync(directory_fd)
+            finally:
+                os.close(directory_fd)
+        except OSError:
+            # Some filesystems do not support fsync on directories. The file
+            # replacement is still atomic there.
+            pass
+    except BaseException:
+        temporary.unlink(missing_ok=True)
+        raise
+
+
+def atomic_write_json(path: str | Path, value: Any, *, mode: int = 0o600) -> None:
+    atomic_write_text(
+        path,
+        json.dumps(value, indent=2, ensure_ascii=False, default=str) + "\n",
+        mode=mode,
+    )

agent/direct_agent.py

@@ -9,6 +9,8 @@
 
 import json
 import logging
+import os
+import time
 from pathlib import Path
 from typing import Any
 
@@ -143,6 +145,8 @@ def _log_event(self, event: dict[str, Any]) -> None:
         try:
             with self.log_path.open("a", encoding="utf-8") as f:
                 f.write(json.dumps(event, ensure_ascii=False, default=str) + "\n")
+                f.flush()
+                os.fsync(f.fileno())
         except Exception:
             logger.exception("failed to write trace event")
 
@@ -163,6 +167,20 @@ def run(self, goal_description: str, max_steps: int = 50) -> str:
         ]
 
         tools_schema = build_tools_payload(self.tools)
+        started_at = time.monotonic()
+        total_tokens = 0
+
+        def finish(status: str, message: str, step: int) -> str:
+            self._log_event(
+                {
+                    "type": "run_end",
+                    "status": status,
+                    "step": step,
+                    "total_tokens": total_tokens,
+                    "duration_seconds": time.monotonic() - started_at,
+                }
+            )
+            return message
 
         self._log_event(
             {
@@ -174,13 +192,12 @@ def run(self, goal_description: str, max_steps: int = 50) -> str:
             }
         )
 
-        total_tokens = 0
         max_tokens = self.llm.config.max_total_tokens_per_turn
         for step in range(1, max_steps + 1):
             if step > 1 and total_tokens >= max_tokens:
                 message = f"Reached token budget ({max_tokens}) without final answer."
                 self._log_event({"type": "token_budget_reached", "max_tokens": max_tokens})
-                return message
+                return finish("token_budget_reached", message, step - 1)
             messages = self._compact_messages(messages, max_turns=20)
             logger.info("step %d/%d: calling LLM", step, max_steps)
             try:
@@ -194,14 +211,16 @@ def run(self, goal_description: str, max_steps: int = 50) -> str:
                         "error": str(exc),
                     }
                 )
-                return f"LLM error at step {step}: {exc}"
+                return finish("llm_error", f"LLM error at step {step}: {exc}", step)
             total_tokens += response.usage.total_tokens
 
             self._log_event(
                 {
                     "type": "llm_response",
                     "step": step,
                     "content": response.content,
+                    "usage": response.usage.model_dump(),
+                    "cumulative_tokens": total_tokens,
                     "tool_calls": [
                         {
                             "id": c.id,
@@ -231,7 +250,7 @@ def run(self, goal_description: str, max_steps: int = 50) -> str:
                         "content": response.content,
                     }
                 )
-                return response.content or ""
+                return finish("completed", response.content or "", step)
 
             # Execute tool calls in sequence (model may request parallel, we
             # execute sequentially for simplicity — same as Claude Code)
@@ -306,4 +325,8 @@ def run(self, goal_description: str, max_steps: int = 50) -> str:
                 "max_steps": max_steps,
             }
         )
-        return f"Reached maximum steps ({max_steps}) without final answer."
+        return finish(
+            "max_steps_reached",
+            f"Reached maximum steps ({max_steps}) without final answer.",
+            max_steps,
+        )