Detect SWE-bench harness failures

scripts/compare_three_systems.py

@@ -501,6 +501,7 @@ def run_swe_agent(
 
 
 INFRA_ERROR_PATTERNS = (
+    "infrastructure failure",
     "ModuleNotFoundError",
     "ImportError",
     "No module named",
@@ -729,6 +730,13 @@ def main() -> int:
                     r.direct_error,
                 )
                 infrastructure_failure = True
+            elif not r.direct_resolved and is_infra_error(r.direct_error):
+                logger.error(
+                    "direct evaluation infrastructure failure for %s: %s. Aborting batch.",
+                    task.id,
+                    r.direct_error,
+                )
+                infrastructure_failure = True
 
         if (
             not infrastructure_failure
@@ -758,6 +766,13 @@ def main() -> int:
                     r.claude_error,
                 )
                 infrastructure_failure = True
+            elif not r.claude_resolved and is_infra_error(r.claude_error):
+                logger.error(
+                    "Claude evaluation infrastructure failure for %s: %s. Aborting batch.",
+                    task.id,
+                    r.claude_error,
+                )
+                infrastructure_failure = True
 
         if (
             not infrastructure_failure

swe_bench/docker.py

@@ -37,6 +37,24 @@
 logger = logging.getLogger("swe_bench.docker")
 
 
+_INFRASTRUCTURE_FAILURE_PATTERNS = (
+    "pytest: command not found",
+    "python: can't open file",
+    "could not find a version that satisfies the requirement",
+    "no matching distribution found",
+    "error: failed to install build dependencies",
+)
+
+
+def detect_infrastructure_failure(test_output: str) -> str | None:
+    """Return a concise error when the test harness failed before grading."""
+    lowered = test_output.lower()
+    for pattern in _INFRASTRUCTURE_FAILURE_PATTERNS:
+        if pattern in lowered:
+            return f"SWE-bench evaluation infrastructure failure: {pattern}"
+    return None
+
+
 class DockerEvaluationError(Exception):
     """Raised when Docker-based evaluation cannot be completed."""
 
@@ -208,6 +226,17 @@ def evaluate(self, patch: str, workspace: Path | None = None) -> EvaluationResul
                     error=f"docker evaluation timed out after {self.timeout_seconds}s",
                 )
 
+            infrastructure_error = detect_infrastructure_failure(test_output)
+            if infrastructure_error:
+                return EvaluationResult(
+                    success=False,
+                    resolved=False,
+                    stdout=test_output,
+                    stderr=infrastructure_error,
+                    exit_code=None,
+                    error=infrastructure_error,
+                )
+
             report = get_eval_report(
                 test_spec=spec,
                 prediction=prediction,
@@ -269,6 +298,17 @@ def evaluate_in_container(self, container, spec) -> EvaluationResult:
                 error=f"docker evaluation timed out after {self.timeout_seconds}s",
             )
 
+        infrastructure_error = detect_infrastructure_failure(test_output)
+        if infrastructure_error:
+            return EvaluationResult(
+                success=False,
+                resolved=False,
+                stdout=test_output,
+                stderr=infrastructure_error,
+                exit_code=None,
+                error=infrastructure_error,
+            )
+
         prediction = {
             KEY_INSTANCE_ID: self.task.id,
             KEY_MODEL: "docker-bash",

tests/test_swe_agent_local_runner.py

@@ -44,6 +44,30 @@ def _docker_evaluator(**kwargs):
     return DockerEvaluator(_task(), **kwargs)
 
 
+@pytest.mark.parametrize(
+    "output",
+    [
+        "/eval.sh: line 89: pytest: command not found",
+        "ERROR: Could not find a version that satisfies the requirement setuptools>=40.0",
+        "ERROR: No matching distribution found for setuptools>=40.0",
+    ],
+)
+def test_docker_evaluator_detects_harness_failures(output):
+    pytest.importorskip("docker")
+    pytest.importorskip("swebench")
+    from swe_bench.docker import detect_infrastructure_failure
+
+    assert detect_infrastructure_failure(output)
+
+
+def test_docker_evaluator_does_not_misclassify_test_failure():
+    pytest.importorskip("docker")
+    pytest.importorskip("swebench")
+    from swe_bench.docker import detect_infrastructure_failure
+
+    assert detect_infrastructure_failure("FAILED tests/test_feature.py::test_answer") is None
+
+
 def test_local_env_preserves_command_exit_status(tmp_path):
     env = LocalSWEEnv(tmp_path, _task(), timeout=5)
     try: