fix(verify): report simulators that produced no results, and fail when none did

backend/biosim_server/api/spec/openapi_3_1_0_generated.yaml

@@ -3256,6 +3256,30 @@ components:
       - processingId
       - jobs
       title: SimulationRunResults
+    SimulatorFailure:
+      properties:
+        simulator_version:
+          type: string
+          title: Simulator Version
+        biosimulations_run_id:
+          anyOf:
+          - type: string
+          - type: 'null'
+          title: Biosimulations Run Id
+        error_message:
+          anyOf:
+          - type: string
+          - type: 'null'
+          title: Error Message
+      type: object
+      required:
+      - simulator_version
+      title: SimulatorFailure
+      description: 'A requested simulator that produced no results: its run failed
+        on biosimulations.org, or could not be
+
+        submitted. Reported instead of being dropped silently, so a verification with
+        no usable runs says why.'
     SimulatorSelection:
       properties:
         id:
@@ -3546,6 +3570,12 @@ components:
           - type: string
           - type: 'null'
           title: Workflow Error
+        failed_simulators:
+          items:
+            $ref: '#/components/schemas/SimulatorFailure'
+          type: array
+          title: Failed Simulators
+          default: []
         workflow_results:
           anyOf:
           - $ref: '#/components/schemas/GenerateStatisticsActivityOutput'

backend/biosim_server/biosim_runs/activities.py

@@ -1,7 +1,7 @@
 import asyncio
 import logging
 import os
-from typing import Optional
+from typing import Any, Optional
 
 from aiohttp import ClientResponseError
 from pydantic import BaseModel
@@ -213,6 +213,15 @@ async def poll_biosim_run_activity(input: PollBiosimRunActivityInput) -> Biosimu
             activity.heartbeat("Polling simulation run status")
             simulation_run = await biosim_service.get_sim_run(input.biosimulations_run_id)
 
+        # biosimulations.org leaves a failed run's `error` empty; the reason is in its logs. Best effort: a run that
+        # failed is reported as failed whether or not its logs can be read.
+        if simulation_run.status == BiosimSimulationRunStatus.FAILED and not simulation_run.error_message:
+            try:
+                logs = await biosim_service.get_sim_run_logs(simulation_run.id)
+                simulation_run.error_message = error_from_run_logs(logs)
+            except Exception as e:
+                activity.logger.warning(f"Could not read the logs of failed run {simulation_run.id}: {e}")
+
         # simdata API may lag behind the run status, so retry HDF5 on 404
         hdf5_file: HDF5File | None = None
         if simulation_run.status == BiosimSimulationRunStatus.SUCCEEDED:
@@ -258,3 +267,23 @@ async def poll_biosim_run_activity(input: PollBiosimRunActivityInput) -> Biosimu
     except Exception as e:
         activity.logger.exception(f"Failed to poll biosim simulation run: {str(e)}", exc_info=e)
         raise e
+
+
+def error_from_run_logs(logs: dict[str, Any]) -> str | None:
+    """The reason a biosimulations.org run failed, from its logs (GET /logs/{run_id}): the archive-level exception
+    (an invalid COMBINE archive, for example), else the first SED document's or task's. None when the logs carry none."""
+    def message(node: Any) -> str | None:
+        exception = node.get("exception") if isinstance(node, dict) else None
+        if isinstance(exception, dict) and exception.get("message"):
+            kind = exception.get("type")
+            return f"{kind}: {exception['message']}" if kind else str(exception["message"])
+        return None
+
+    found = message(logs)
+    if found is None:
+        for document in logs.get("sedDocuments") or []:
+            found = message(document) or next(
+                (m for task in (document.get("tasks") or []) if (m := message(task)) is not None), None)
+            if found is not None:
+                break
+    return found[:2000] if found is not None else None

backend/biosim_server/biosim_verify/models.py

@@ -65,6 +65,14 @@ class GenerateStatisticsActivityOutput(BaseModel):
     sim_run_data: Optional[list[RunData]] = None
 
 
+class SimulatorFailure(BaseModel):
+    """A requested simulator that produced no results: its run failed on biosimulations.org, or could not be
+    submitted. Reported instead of being dropped silently, so a verification with no usable runs says why."""
+    simulator_version: str  # <simulator_name>:<version>
+    biosimulations_run_id: Optional[str] = None  # absent when the run was never created
+    error_message: Optional[str] = None
+
+
 class VerifyWorkflowStatus(StrEnum):
     PENDING = "PENDING"
     IN_PROGRESS = "IN_PROGRESS"
@@ -85,6 +93,8 @@ class VerifyWorkflowOutput(BaseModel):
     timestamp: str
     workflow_run_id: Optional[str] = None
     workflow_error: Optional[str] = None
+    # Requested simulators that produced no results (OmexVerifyWorkflow); empty when every one ran
+    failed_simulators: list[SimulatorFailure] = []
     workflow_results: Optional[GenerateStatisticsActivityOutput] = None
     # Auth0 `sub` of the caller who started the workflow. Optional so in-flight
     # Temporal histories that predate this field still deserialize.

backend/biosim_server/biosim_verify/omex_verify_workflow.py

@@ -11,7 +11,7 @@
 from biosim_server.biosim_runs import BiosimulatorVersion, OmexSimWorkflow, OmexSimWorkflowInput, OmexSimWorkflowOutput, \
     BiosimulatorWorkflowRun
 from biosim_server.biosim_verify import CompareSettings
-from biosim_server.biosim_verify.models import VerifyWorkflowStatus, VerifyWorkflowOutput
+from biosim_server.biosim_verify.models import SimulatorFailure, VerifyWorkflowStatus, VerifyWorkflowOutput
 from biosim_server.biosim_verify.runs_verify_workflow import generate_statistics
 
 
@@ -66,21 +66,54 @@ async def run(self, verify_input: OmexVerifyWorkflowInput) -> VerifyWorkflowOutp
         child_results: list[ChildWorkflowHandle[OmexSimWorkflowInput, OmexSimWorkflowOutput]] = await asyncio.gather(
             *child_workflows)
 
-        simulator_workflow_runs: list[BiosimulatorWorkflowRun] = []
+        outputs: list[OmexSimWorkflowOutput] = []
         for child_result in child_results:
             omex_sim_workflow_output = await child_result
             if not child_result.done():
                 raise Exception(
                     "Child workflow did not complete successfully, even after asyncio.gather on all workflows")
-
-            if omex_sim_workflow_output.biosimulator_workflow_run is None:
-                continue
-            simulator_workflow_runs.append(omex_sim_workflow_output.biosimulator_workflow_run)
+            outputs.append(omex_sim_workflow_output)
+        simulator_workflow_runs, failures = split_simulator_outcomes(verify_input.requested_simulators, outputs)
 
         # Generate comparison report within an activity
         workflow.logger.info("calling activity to generate statistics")
         stats = await generate_statistics(sim_workflow_runs=simulator_workflow_runs, compare_settings=self.verify_input.compare_settings)
 
         self.verify_output.workflow_results = stats
-        self.verify_output.workflow_status = VerifyWorkflowStatus.COMPLETED
+        self.verify_output.failed_simulators = failures
+        no_results = no_results_error(verify_input.requested_simulators, simulator_workflow_runs, failures)
+        if no_results is not None:
+            # Nothing was compared: say so, rather than COMPLETED with empty results
+            self.verify_output.workflow_status = VerifyWorkflowStatus.FAILED
+            self.verify_output.workflow_error = no_results
+        else:
+            self.verify_output.workflow_status = VerifyWorkflowStatus.COMPLETED
         return self.verify_output
+
+
+def split_simulator_outcomes(
+        requested: list[BiosimulatorVersion], outputs: list[OmexSimWorkflowOutput],
+) -> tuple[list[BiosimulatorWorkflowRun], list[SimulatorFailure]]:
+    """The runs to compare, and a failure for each requested simulator that produced none (in request order)."""
+    runs: list[BiosimulatorWorkflowRun] = []
+    failures: list[SimulatorFailure] = []
+    for simulator, output in zip(requested, outputs):
+        if output.biosimulator_workflow_run is not None:
+            runs.append(output.biosimulator_workflow_run)
+            continue
+        failures.append(SimulatorFailure(
+            simulator_version=f"{simulator.id}:{simulator.version}",
+            biosimulations_run_id=output.biosimulations_run_id,
+            error_message=output.error_message or f"the run ended {output.workflow_status}",
+        ))
+    return runs, failures
+
+
+def no_results_error(requested: list[BiosimulatorVersion], runs: list[BiosimulatorWorkflowRun],
+                     failures: list[SimulatorFailure]) -> str | None:
+    """Why the verification compared nothing, when no requested simulator produced results; None otherwise."""
+    if not requested or runs:
+        return None
+    reasons = "; ".join(f"{f.simulator_version}: {(f.error_message or 'no error recorded').splitlines()[0]}"
+                        for f in failures)
+    return f"No requested simulator produced results ({reasons})"[:4000]

backend/tests/biosim_verify/test_failed_simulators.py

@@ -0,0 +1,60 @@
+"""A verification reports the simulators that produced no results, and why, instead of dropping them silently.
+
+Before, a verification whose every simulator run failed on biosimulations.org (for example, an archive whose manifest
+lists files it does not contain) answered COMPLETED with empty results and no hint of what went wrong."""
+
+import json
+from pathlib import Path
+
+from biosim_server.biosim_runs import BiosimulatorVersion, OmexSimWorkflowOutput, OmexSimWorkflowStatus
+from biosim_server.biosim_runs.activities import error_from_run_logs
+from biosim_server.biosim_verify.omex_verify_workflow import no_results_error, split_simulator_outcomes
+
+LOCAL_DATA = Path(__file__).parent.parent / "fixtures" / "local_data"
+
+
+def _version(name: str, version: str) -> BiosimulatorVersion:
+    return BiosimulatorVersion(id=name, name=name, version=version, image_url=f"ghcr.io/biosimulators/{name}",
+                               image_digest="sha256:" + "0" * 64, created="2025-01-01", updated="2025-01-01")
+
+
+def _failed(run_id: str | None, error: str | None) -> OmexSimWorkflowOutput:
+    return OmexSimWorkflowOutput(workflow_id="w", workflow_status=OmexSimWorkflowStatus.FAILED,
+                                 biosimulations_run_id=run_id, error_message=error)
+
+
+def test_a_failed_runs_reason_is_read_from_its_logs() -> None:
+    # A real biosimulations.org run (6ada2269041d31202094ca8f, 2026-10-22): COPASI on BioModels' BIOMD0000000012 archive,
+    # whose manifest lists files the archive does not hold. The run's own `error` was empty.
+    logs = json.loads((LOCAL_DATA / "biosim_failed_run_logs.json").read_text())
+    reason = error_from_run_logs(logs)
+    assert reason is not None
+    assert reason.startswith("ValueError: `/root/archive.omex` is not a valid COMBINE/OMEX archive.")
+    assert "BIOMD0000000012-biopax2.owl" in reason
+
+
+def test_a_task_level_exception_is_used_when_the_archive_level_one_is_absent() -> None:
+    logs = {"exception": None, "sedDocuments": [{"exception": None, "tasks": [
+        {"exception": {"type": "RuntimeError", "message": "integration failed at t=3"}}]}]}
+    assert error_from_run_logs(logs) == "RuntimeError: integration failed at t=3"
+    assert error_from_run_logs({"status": "SUCCEEDED", "exception": None}) is None
+
+
+def test_simulators_without_results_are_reported_not_dropped() -> None:
+    requested = [_version("copasi", "4.45.296"), _version("tellurium", "2.2.10")]
+    outputs = [_failed("6ada2269041d31202094ca8f", "ValueError: not a valid COMBINE/OMEX archive.\n  - details"),
+               _failed(None, None)]
+    runs, failures = split_simulator_outcomes(requested, outputs)
+    assert runs == []
+    assert [(f.simulator_version, f.biosimulations_run_id) for f in failures] == [
+        ("copasi:4.45.296", "6ada2269041d31202094ca8f"), ("tellurium:2.2.10", None)]
+    assert failures[1].error_message == "the run ended FAILED"
+    error = no_results_error(requested, runs, failures)
+    assert error is not None
+    assert error.startswith("No requested simulator produced results (copasi:4.45.296: ValueError: not a valid")
+    assert "tellurium:2.2.10: the run ended FAILED" in error
+
+
+def test_a_verification_with_some_results_is_not_a_failure() -> None:
+    assert no_results_error([_version("copasi", "1")], runs=[object()], failures=[]) is None  # type: ignore[list-item]
+    assert no_results_error([], runs=[], failures=[]) is None