fix(benchmark): preserve native Goal failure outcomes and scores

benchmark/edgebench/run.py

@@ -71,22 +71,29 @@ def on_interrupt(signum, frame):
 
 
 def _result_status(*, interrupted, started, runtime_seconds, exit_code, timed_out=False,
-                   execution_started=True):
+                   execution_started=True, native_goal_outcome=None):
     if interrupted:
         return "cancelled"
     if not started:
         return "launch_failed"
+    if native_goal_outcome == "cancelled":
+        return "cancelled"
+    if native_goal_outcome in {"failed", "usage_limited", "unknown"}:
+        return "runner_failed"
     if not execution_started or runtime_seconds <= 0 or (not timed_out and exit_code != 0):
         return "runner_failed"
     return "terminal"
 
 
-def _write_native_final_result(trial, result, *, status, agent, task, run_id, model, effort, feedback="native"):
-    """Match the native CLI's visualizer handoff after a completed run only."""
-    if status != "terminal":
+def _write_native_final_result(trial, result, *, status, agent, task, run_id, model, effort,
+                               feedback="native", native_goal_outcome=None):
+    """Preserve native scores separately from the solver's terminal outcome."""
+    if status != "terminal" and not (native_goal_outcome and result.runtime_seconds > 0):
         return
     final = dict(agent=agent, task=task, run_id=run_id, model=model, effort=effort,
                  **result.to_dict())
+    if native_goal_outcome is not None:
+        final.update(runtime_status=status, native_goal_outcome=native_goal_outcome)
     if feedback == "best-only":
         final["evaluation_coverage"] = "online_only"
         final["offline_scoring_complete"] = False
@@ -413,19 +420,26 @@ def main(argv=None):
     # Native cancellation and swallowed Docker failures return runtime=0.
     # Observe the signal independently; never infer cancellation from output prose.
     execution = backend.execution_receipt or {}
+    native = backend.native_goal_receipt or {}
+    if args.worker == "native-goal" and not native.get("native_goal_outcome"):
+        native["native_goal_outcome"] = "unknown"
     status = _result_status(interrupted=was_interrupted,
                             started=(trial / "started_at").is_file(),
                             runtime_seconds=result.runtime_seconds,
                             execution_started=execution.get("execution_started", False),
                             exit_code=execution.get("exit_code"),
+                            native_goal_outcome=native.get("native_goal_outcome"),
                             timed_out=result.timed_out or execution.get("timed_out", False))
     receipt.update(status=status, controller_elapsed_seconds=elapsed,
                    timed_out=result.timed_out, runtime_seconds=result.runtime_seconds,
                    best_score=result.best_score, total_rounds=result.total_rounds)
     receipt["execution"] = execution
+    if native:
+        receipt.update(native)
     _write_native_final_result(trial, result, status=status, agent=agent.name,
                                task=task.task_id, run_id=args.run_id,
-                               model=args.model, effort=args.effort, feedback=args.feedback)
+                               model=args.model, effort=args.effort, feedback=args.feedback,
+                               native_goal_outcome=native.get("native_goal_outcome"))
     receipt_path.write_text(json.dumps(receipt, indent=2))
     print(json.dumps(receipt))
     return 0 if status == "terminal" else 130 if status == "cancelled" else 1

benchmark/runtime/RUNTIME.md

@@ -79,6 +79,29 @@ Native Goal continuation stays with Codex; blocked Goals are not automatically
 unblocked. Plain exec versus Goal app-server also changes transport; it does
 not isolate the continuation effect alone.
 
+Native worker receipts retain both the latest Turn status and Goal status.
+`native_goal_outcome` distinguishes `complete`, `blocked`, `paused`,
+`budget_exhausted`, `usage_limited`, `cancelled`, `failed`, and `unknown`.
+A failed Turn takes precedence over a blocked or complete Goal and exits nonzero;
+intentional blocking/pausing and normal budget exhaustion do not become execution
+failures. `ok` indicates a handled stop, never task acceptance. Only completion
+may enter the explicitly enabled continuation option below.
+
+Terminal errors retain a digest, an allowlisted native error variant and an
+optional structured HTTP status. Unrecognized errors remain unknown; message
+text is not parsed into a category or copied into the receipt. The existing
+`error_event_count` counts error notifications, not requests: a terminal event
+that repeats an error does not increment it. A later native continuation clears
+the terminal-error fields and may complete normally.
+
+The EdgeBench adapter carries this outcome through artifact collection into
+`runtime-receipt.json`. If a native Goal fails after execution starts, its
+`final_result.json` still retains native scores, with explicit `runtime_status`
+and `native_goal_outcome`; the presence of that file alone is not successful
+completion or result qualification. Ordinary non-native profile handling is
+unchanged. Adopt this behavior only for newly pinned attempts; frozen histories
+retain their original receipts and require explicit post-run classification.
+
 The optional `native_goal_continue: true` loop retains the original task's
 requirements and evaluation rules after completion. Its continuation prompt no
 longer requires a candidate to improve on an accepted best before submission.

benchmark/runtime/sforge_backend.py

@@ -24,6 +24,7 @@ def __init__(self, *, log_dir: Path, logger, oauth_proxy: bool = False,
         self.execution_command = None
         self.execution_start_marker = None
         self.execution_receipt = None
+        self.native_goal_receipt = None
         self.auth_ips = [] if oauth_proxy else resolve_hostname("auth.openai.com", logger)
         if not oauth_proxy and not self.auth_ips:
             raise RuntimeError("Cannot resolve Codex OAuth endpoint")
@@ -124,6 +125,23 @@ def cleanup_container(self, handle, logger=None):
                     observations.append({"artifact": name, "collected": False,
                                          "error_kind": type(error).__name__})
             (self.log_dir / "artifact-collection.json").write_text(json.dumps(observations))
+            # A native-Goal worker owns exactly one wake (including opt-in
+            # continuation Goals). Read its retained receipt after collection;
+            # don't infer completion from a shell exit or solver output prose.
+            wakes = self.log_dir / "runtime" / "agent" / "wakes"
+            native = []
+            for path in wakes.glob("*/receipt.json"):
+                try:
+                    receipt = json.loads(path.read_text())
+                    if isinstance(receipt, dict) and receipt.get("mode") in {"native-goal", "loopx-goal"}:
+                        native.append(receipt)
+                except (OSError, ValueError):
+                    continue  # Collection gaps retain the transport failure.
+            if len(native) == 1:
+                self.native_goal_receipt = {key: native[0][key] for key in
+                    ("native_goal_outcome", "native_goal_rounds", "native_goal") if key in native[0]}
+            elif native:
+                self.native_goal_receipt = {"native_goal_outcome": "unknown"}
         super().cleanup_container(handle, logger)
         if self.feedback is not None:
             # Native cleanup logs some failures instead of raising. Verify the

benchmark/runtime/worker.py

@@ -266,8 +266,8 @@ def native_goal_rounds(run_round, *, budget_seconds, continue_after_complete, cl
     """Drive one native Goal and, when opted in, continuation Goals until the budget ends.
 
     ``run_round(objective_suffix, timeout_sec)`` runs one Goal to a terminal status and
-    returns that status. Returns ``(rounds, outcome)`` where outcome is ``complete``,
-    ``budget_exhausted`` (a Goal was still active at the budget edge) or ``stopped``.
+    returns its typed outcome. Only ``complete`` may start an explicitly opted-in
+    continuation. Failure, blocking, cancellation and unknown outcomes stop.
     """
     from loopx.capabilities.benchmark_toolkit.native_codex_goal import NativeGoalDeadlineExceeded
 
@@ -284,7 +284,7 @@ def native_goal_rounds(run_round, *, budget_seconds, continue_after_complete, cl
             return rounds + 1, "budget_exhausted"
         rounds += 1
         if status != "complete" or not continue_after_complete:
-            return rounds, ("complete" if status == "complete" else "stopped")
+            return rounds, status
         remaining = None if deadline is None else deadline - clock()
         if remaining is not None and remaining <= reserve_seconds:
             return rounds, "complete"
@@ -304,6 +304,7 @@ def run_native_goal(
         NativeGoalConfig,
         StdioNativeGoalTransport,
         compact_native_goal_receipt,
+        native_goal_outcome,
         refresh_native_goal_status,
         run_native_goal_until_terminal,
     )
@@ -339,7 +340,9 @@ def run_round(suffix, timeout_sec):
                 transport, NativeGoalConfig(objective=body + suffix, **base),
                 timeout_sec=timeout_sec, on_turn_started=observed.append,
             )
-            return turn.post_goal_status or refresh_native_goal_status(transport, turn)
+            if not turn.post_goal_status:
+                refresh_native_goal_status(transport, turn)
+            return native_goal_outcome(turn)
 
         try:
             rounds, outcome = native_goal_rounds(
@@ -352,10 +355,10 @@ def run_round(suffix, timeout_sec):
                 # Reaching the budget with an active Goal is normal exhaustion for
                 # a native-Goal worker, not a runner failure.
                 receipt["budget_exhausted"] = True
-            receipt["ok"] = True
+            receipt["ok"] = outcome in {"complete", "blocked", "paused", "budget_exhausted"}
         finally:
             if observed:
-                receipt["native_goal"] = compact_native_goal_receipt(observed[0])
+                receipt["native_goal"] = compact_native_goal_receipt(observed[-1])
 
 
 def native_command(env, execution, stage, wake, session_wake):
@@ -656,6 +659,8 @@ def cancelled(signum, frame):
         resource.setrlimit(resource.RLIMIT_CORE, (0, hard))
     receipt = run_once(dict(os.environ))
     print(json.dumps(receipt))
+    if receipt.get("native_goal_outcome") == "cancelled":
+        return 130
     if receipt.get("budget_exhausted") and receipt["mode"] in {"heartbeat", "turn"}:
         from benchmark.runtime.scheduler import BUDGET_EXHAUSTED_EXIT_CODE