diff --git a/backend/biosim_server/api/spec/openapi_3_1_0_generated.yaml b/backend/biosim_server/api/spec/openapi_3_1_0_generated.yaml index 110b803..55c3ab6 100644 --- a/backend/biosim_server/api/spec/openapi_3_1_0_generated.yaml +++ b/backend/biosim_server/api/spec/openapi_3_1_0_generated.yaml @@ -3256,6 +3256,30 @@ components: - processingId - jobs title: SimulationRunResults + SimulatorFailure: + properties: + simulator_version: + type: string + title: Simulator Version + biosimulations_run_id: + anyOf: + - type: string + - type: 'null' + title: Biosimulations Run Id + error_message: + anyOf: + - type: string + - type: 'null' + title: Error Message + type: object + required: + - simulator_version + title: SimulatorFailure + description: 'A requested simulator that produced no results: its run failed + on biosimulations.org, or could not be + + submitted. Reported instead of being dropped silently, so a verification with + no usable runs says why.' SimulatorSelection: properties: id: @@ -3546,6 +3570,12 @@ components: - type: string - type: 'null' title: Workflow Error + failed_simulators: + items: + $ref: '#/components/schemas/SimulatorFailure' + type: array + title: Failed Simulators + default: [] workflow_results: anyOf: - $ref: '#/components/schemas/GenerateStatisticsActivityOutput' diff --git a/backend/biosim_server/biosim_runs/activities.py b/backend/biosim_server/biosim_runs/activities.py index be85c03..f5047da 100644 --- a/backend/biosim_server/biosim_runs/activities.py +++ b/backend/biosim_server/biosim_runs/activities.py @@ -1,7 +1,7 @@ import asyncio import logging import os -from typing import Optional +from typing import Any, Optional from aiohttp import ClientResponseError from pydantic import BaseModel @@ -213,6 +213,15 @@ async def poll_biosim_run_activity(input: PollBiosimRunActivityInput) -> Biosimu activity.heartbeat("Polling simulation run status") simulation_run = await biosim_service.get_sim_run(input.biosimulations_run_id) + # biosimulations.org leaves a failed run's `error` empty; the reason is in its logs. Best effort: a run that + # failed is reported as failed whether or not its logs can be read. + if simulation_run.status == BiosimSimulationRunStatus.FAILED and not simulation_run.error_message: + try: + logs = await biosim_service.get_sim_run_logs(simulation_run.id) + simulation_run.error_message = error_from_run_logs(logs) + except Exception as e: + activity.logger.warning(f"Could not read the logs of failed run {simulation_run.id}: {e}") + # simdata API may lag behind the run status, so retry HDF5 on 404 hdf5_file: HDF5File | None = None if simulation_run.status == BiosimSimulationRunStatus.SUCCEEDED: @@ -258,3 +267,23 @@ async def poll_biosim_run_activity(input: PollBiosimRunActivityInput) -> Biosimu except Exception as e: activity.logger.exception(f"Failed to poll biosim simulation run: {str(e)}", exc_info=e) raise e + + +def error_from_run_logs(logs: dict[str, Any]) -> str | None: + """The reason a biosimulations.org run failed, from its logs (GET /logs/{run_id}): the archive-level exception + (an invalid COMBINE archive, for example), else the first SED document's or task's. None when the logs carry none.""" + def message(node: Any) -> str | None: + exception = node.get("exception") if isinstance(node, dict) else None + if isinstance(exception, dict) and exception.get("message"): + kind = exception.get("type") + return f"{kind}: {exception['message']}" if kind else str(exception["message"]) + return None + + found = message(logs) + if found is None: + for document in logs.get("sedDocuments") or []: + found = message(document) or next( + (m for task in (document.get("tasks") or []) if (m := message(task)) is not None), None) + if found is not None: + break + return found[:2000] if found is not None else None diff --git a/backend/biosim_server/biosim_verify/models.py b/backend/biosim_server/biosim_verify/models.py index 27ba95c..e492c17 100644 --- a/backend/biosim_server/biosim_verify/models.py +++ b/backend/biosim_server/biosim_verify/models.py @@ -65,6 +65,14 @@ class GenerateStatisticsActivityOutput(BaseModel): sim_run_data: Optional[list[RunData]] = None +class SimulatorFailure(BaseModel): + """A requested simulator that produced no results: its run failed on biosimulations.org, or could not be + submitted. Reported instead of being dropped silently, so a verification with no usable runs says why.""" + simulator_version: str # : + biosimulations_run_id: Optional[str] = None # absent when the run was never created + error_message: Optional[str] = None + + class VerifyWorkflowStatus(StrEnum): PENDING = "PENDING" IN_PROGRESS = "IN_PROGRESS" @@ -85,6 +93,8 @@ class VerifyWorkflowOutput(BaseModel): timestamp: str workflow_run_id: Optional[str] = None workflow_error: Optional[str] = None + # Requested simulators that produced no results (OmexVerifyWorkflow); empty when every one ran + failed_simulators: list[SimulatorFailure] = [] workflow_results: Optional[GenerateStatisticsActivityOutput] = None # Auth0 `sub` of the caller who started the workflow. Optional so in-flight # Temporal histories that predate this field still deserialize. diff --git a/backend/biosim_server/biosim_verify/omex_verify_workflow.py b/backend/biosim_server/biosim_verify/omex_verify_workflow.py index a589ed3..9f44673 100644 --- a/backend/biosim_server/biosim_verify/omex_verify_workflow.py +++ b/backend/biosim_server/biosim_verify/omex_verify_workflow.py @@ -11,7 +11,7 @@ from biosim_server.biosim_runs import BiosimulatorVersion, OmexSimWorkflow, OmexSimWorkflowInput, OmexSimWorkflowOutput, \ BiosimulatorWorkflowRun from biosim_server.biosim_verify import CompareSettings -from biosim_server.biosim_verify.models import VerifyWorkflowStatus, VerifyWorkflowOutput +from biosim_server.biosim_verify.models import SimulatorFailure, VerifyWorkflowStatus, VerifyWorkflowOutput from biosim_server.biosim_verify.runs_verify_workflow import generate_statistics @@ -66,21 +66,54 @@ async def run(self, verify_input: OmexVerifyWorkflowInput) -> VerifyWorkflowOutp child_results: list[ChildWorkflowHandle[OmexSimWorkflowInput, OmexSimWorkflowOutput]] = await asyncio.gather( *child_workflows) - simulator_workflow_runs: list[BiosimulatorWorkflowRun] = [] + outputs: list[OmexSimWorkflowOutput] = [] for child_result in child_results: omex_sim_workflow_output = await child_result if not child_result.done(): raise Exception( "Child workflow did not complete successfully, even after asyncio.gather on all workflows") - - if omex_sim_workflow_output.biosimulator_workflow_run is None: - continue - simulator_workflow_runs.append(omex_sim_workflow_output.biosimulator_workflow_run) + outputs.append(omex_sim_workflow_output) + simulator_workflow_runs, failures = split_simulator_outcomes(verify_input.requested_simulators, outputs) # Generate comparison report within an activity workflow.logger.info("calling activity to generate statistics") stats = await generate_statistics(sim_workflow_runs=simulator_workflow_runs, compare_settings=self.verify_input.compare_settings) self.verify_output.workflow_results = stats - self.verify_output.workflow_status = VerifyWorkflowStatus.COMPLETED + self.verify_output.failed_simulators = failures + no_results = no_results_error(verify_input.requested_simulators, simulator_workflow_runs, failures) + if no_results is not None: + # Nothing was compared: say so, rather than COMPLETED with empty results + self.verify_output.workflow_status = VerifyWorkflowStatus.FAILED + self.verify_output.workflow_error = no_results + else: + self.verify_output.workflow_status = VerifyWorkflowStatus.COMPLETED return self.verify_output + + +def split_simulator_outcomes( + requested: list[BiosimulatorVersion], outputs: list[OmexSimWorkflowOutput], +) -> tuple[list[BiosimulatorWorkflowRun], list[SimulatorFailure]]: + """The runs to compare, and a failure for each requested simulator that produced none (in request order).""" + runs: list[BiosimulatorWorkflowRun] = [] + failures: list[SimulatorFailure] = [] + for simulator, output in zip(requested, outputs): + if output.biosimulator_workflow_run is not None: + runs.append(output.biosimulator_workflow_run) + continue + failures.append(SimulatorFailure( + simulator_version=f"{simulator.id}:{simulator.version}", + biosimulations_run_id=output.biosimulations_run_id, + error_message=output.error_message or f"the run ended {output.workflow_status}", + )) + return runs, failures + + +def no_results_error(requested: list[BiosimulatorVersion], runs: list[BiosimulatorWorkflowRun], + failures: list[SimulatorFailure]) -> str | None: + """Why the verification compared nothing, when no requested simulator produced results; None otherwise.""" + if not requested or runs: + return None + reasons = "; ".join(f"{f.simulator_version}: {(f.error_message or 'no error recorded').splitlines()[0]}" + for f in failures) + return f"No requested simulator produced results ({reasons})"[:4000] diff --git a/backend/tests/biosim_verify/test_failed_simulators.py b/backend/tests/biosim_verify/test_failed_simulators.py new file mode 100644 index 0000000..d9bc3b6 --- /dev/null +++ b/backend/tests/biosim_verify/test_failed_simulators.py @@ -0,0 +1,60 @@ +"""A verification reports the simulators that produced no results, and why, instead of dropping them silently. + +Before, a verification whose every simulator run failed on biosimulations.org (for example, an archive whose manifest +lists files it does not contain) answered COMPLETED with empty results and no hint of what went wrong.""" + +import json +from pathlib import Path + +from biosim_server.biosim_runs import BiosimulatorVersion, OmexSimWorkflowOutput, OmexSimWorkflowStatus +from biosim_server.biosim_runs.activities import error_from_run_logs +from biosim_server.biosim_verify.omex_verify_workflow import no_results_error, split_simulator_outcomes + +LOCAL_DATA = Path(__file__).parent.parent / "fixtures" / "local_data" + + +def _version(name: str, version: str) -> BiosimulatorVersion: + return BiosimulatorVersion(id=name, name=name, version=version, image_url=f"ghcr.io/biosimulators/{name}", + image_digest="sha256:" + "0" * 64, created="2025-01-01", updated="2025-01-01") + + +def _failed(run_id: str | None, error: str | None) -> OmexSimWorkflowOutput: + return OmexSimWorkflowOutput(workflow_id="w", workflow_status=OmexSimWorkflowStatus.FAILED, + biosimulations_run_id=run_id, error_message=error) + + +def test_a_failed_runs_reason_is_read_from_its_logs() -> None: + # A real biosimulations.org run (6ada2269041d31202094ca8f, 2026-10-22): COPASI on BioModels' BIOMD0000000012 archive, + # whose manifest lists files the archive does not hold. The run's own `error` was empty. + logs = json.loads((LOCAL_DATA / "biosim_failed_run_logs.json").read_text()) + reason = error_from_run_logs(logs) + assert reason is not None + assert reason.startswith("ValueError: `/root/archive.omex` is not a valid COMBINE/OMEX archive.") + assert "BIOMD0000000012-biopax2.owl" in reason + + +def test_a_task_level_exception_is_used_when_the_archive_level_one_is_absent() -> None: + logs = {"exception": None, "sedDocuments": [{"exception": None, "tasks": [ + {"exception": {"type": "RuntimeError", "message": "integration failed at t=3"}}]}]} + assert error_from_run_logs(logs) == "RuntimeError: integration failed at t=3" + assert error_from_run_logs({"status": "SUCCEEDED", "exception": None}) is None + + +def test_simulators_without_results_are_reported_not_dropped() -> None: + requested = [_version("copasi", "4.45.296"), _version("tellurium", "2.2.10")] + outputs = [_failed("6ada2269041d31202094ca8f", "ValueError: not a valid COMBINE/OMEX archive.\n - details"), + _failed(None, None)] + runs, failures = split_simulator_outcomes(requested, outputs) + assert runs == [] + assert [(f.simulator_version, f.biosimulations_run_id) for f in failures] == [ + ("copasi:4.45.296", "6ada2269041d31202094ca8f"), ("tellurium:2.2.10", None)] + assert failures[1].error_message == "the run ended FAILED" + error = no_results_error(requested, runs, failures) + assert error is not None + assert error.startswith("No requested simulator produced results (copasi:4.45.296: ValueError: not a valid") + assert "tellurium:2.2.10: the run ended FAILED" in error + + +def test_a_verification_with_some_results_is_not_a_failure() -> None: + assert no_results_error([_version("copasi", "1")], runs=[object()], failures=[]) is None # type: ignore[list-item] + assert no_results_error([], runs=[], failures=[]) is None diff --git a/backend/tests/fixtures/local_data/biosim_failed_run_logs.json b/backend/tests/fixtures/local_data/biosim_failed_run_logs.json new file mode 100644 index 0000000..d849abb --- /dev/null +++ b/backend/tests/fixtures/local_data/biosim_failed_run_logs.json @@ -0,0 +1 @@ +{"status":"FAILED","exception":{"type":"ValueError","message":"`/root/archive.omex` is not a valid COMBINE/OMEX archive.\n - Content element `BIOMD0000000012-biopax2.owl` is invalid.\n - Location is not a file.\n - Content element `BIOMD0000000012-biopax3.owl` is invalid.\n - Location is not a file.\n - Content element `BIOMD0000000012-matlab.m` is invalid.\n - Location is not a file.\n - Content element `BIOMD0000000012-octave.m` is invalid.\n - Location is not a file.\n - Content element `BIOMD0000000012.m` is invalid.\n - Location is not a file.\n - Content element `BIOMD0000000012.ode` is invalid.\n - Location is not a file.\n - Content element `BIOMD0000000012.pdf` is invalid.\n - Location is not a file.\n - Content element `BIOMD0000000012.png` is invalid.\n - Location is not a file.\n - Content element `BIOMD0000000012.svg` is invalid.\n - Location is not a file.\n - Content element `BIOMD0000000012.vcml` is invalid.\n - Location is not a file.\n - Content element `curation_image.png` is invalid.\n - Location is not a file.\n - Content element `curation_notes.txt` is invalid.\n - Location is not a file.\n - Content element `metadata.rdf` is invalid.\n - Location is not a file."},"skipReason":null,"output":"\u001b[0;36mThank you for using runBioSimulations!\u001b[0m\n\n\u001b[0;36m================================================ Loading Singularity ================================================\u001b[0m\n/var/spool/slurm/slurmd/job4303327/slurm_script: line 24: /usr/share/Modules/init/bash: No such file or directory\n/var/spool/slurm/slurmd/job4303327/slurm_script: line 26: module: command not found\n\n\u001b[0;36m=================================================== Set up storage ==================================================\u001b[0m\n\n\u001b[0;36m========================================== Downloading COMBINE/OMEX archive =========================================\u001b[0m\n % Total % Received % Xferd Average Speed Time Time Time Current\n Dload Upload Total Spent Left Speed\n\r 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0\r100 139 100 139 0 0 741 0 --:--:-- --:--:-- --:--:-- 743\n\r100 9000 100 9000 0 0 22692 0 --:--:-- --:--:-- --:--:-- 22692\n\n\u001b[0;36m=========================================== Executing COMBINE/OMEX archive ==========================================\u001b[0m\nmktemp: failed to create directory via template ‘/local/tmp.XXXXXXXXXX’: Permission denied\nINFO: Using cached SIF image\n`/root/archive.omex` is not a valid COMBINE/OMEX archive.\n - Content element `BIOMD0000000012-biopax2.owl` is invalid.\n - Location is not a file.\n - Content element `BIOMD0000000012-biopax3.owl` is invalid.\n - Location is not a file.\n - Content element `BIOMD0000000012-matlab.m` is invalid.\n - Location is not a file.\n - Content element `BIOMD0000000012-octave.m` is invalid.\n - Location is not a file.\n - Content element `BIOMD0000000012.m` is invalid.\n - Location is not a file.\n - Content element `BIOMD0000000012.ode` is invalid.\n - Location is not a file.\n - Content element `BIOMD0000000012.pdf` is invalid.\n - Location is not a file.\n - Content element `BIOMD0000000012.png` is invalid.\n - Location is not a file.\n - Content element `BIOMD0000000012.svg` is invalid.\n - Location is not a file.\n - Content element `BIOMD0000000012.vcml` is invalid.\n - Location is not a file.\n - Content element `curation_image.png` is invalid.\n - Location is not a file.\n - Content element `curation_notes.txt` is invalid.\n - Location is not a file.\n - Content element `metadata.rdf` is invalid.\n - Location is not a file.\nsrun: error: mantis-040: task 0: Exited with exit code 1\nsrun: Terminating StepId=4303327.1\n\n\u001b[0;36m===================================================== Saving log ====================================================\u001b[0m\nCompleted 2.6 KiB/2.6 KiB (20.7 KiB/s) with 1 file(s) remaining\rupload: ./rawLog.txt to s3://files.biosimulations.org/simulations/6ada2269041d31202094ca8f/rawLog.txt\n\n\u001b[0;36m===================================================== Saving Structured log ==========================================\u001b[0m\nCompleted 1.4 KiB/1.4 KiB (9.2 KiB/s) with 1 file(s) remaining\rupload: outputs/log.yml to s3://files.biosimulations.org/simulations/6ada2269041d31202094ca8f/outputs/log.yml\n\n\u001b[0;36m================================================== Zipping outputs ==================================================\u001b[0m\n adding: outputs/ (stored 0%)\n adding: outputs/log.yml (deflated 77%)\n adding: rawLog.txt (deflated 74%)\n\n\u001b[0;36m=================================================== Saving HDF5 outputs ==================================================\u001b[0m\n\n\u001b[0;36m=================================================== Saving non-HDF5 outputs ==================================================\u001b[0m\nCompleted 3.6 KiB/5.2 KiB (22.4 KiB/s) with 2 file(s) remaining\rupload: ./rawLog.txt to s3://files.biosimulations.org/simulations/6ada2269041d31202094ca8f/rawLog.txt\nCompleted 3.6 KiB/5.2 KiB (22.4 KiB/s) with 1 file(s) remaining\rCompleted 5.2 KiB/5.2 KiB (23.5 KiB/s) with 1 file(s) remaining\rupload: ./output.zip to s3://files.biosimulations.org/simulations/6ada2269041d31202094ca8f/output.zip\n\n\u001b[0;36m====================================== Unzip contents of COMBINE/OMEX archive ======================================\u001b[0m\nArchive: archive.omex\n inflating: contents/manifest.xml \n inflating: contents/BIOMD0000000012_url.sedml \n inflating: contents/BIOMD0000000012_url.xml \n\n\u001b[0;36m====================================== Save contents of COMBINE/OMEX archive ======================================\u001b[0m\nCompleted 20.1 KiB/64.8 KiB (128.9 KiB/s) with 3 file(s) remaining\rupload: contents/BIOMD0000000012_url.sedml to s3://files.biosimulations.org/simulations/6ada2269041d31202094ca8f/contents/BIOMD0000000012_url.sedml\nCompleted 20.1 KiB/64.8 KiB (128.9 KiB/s) with 2 file(s) remaining\rCompleted 21.9 KiB/64.8 KiB (82.3 KiB/s) with 2 file(s) remaining \rupload: contents/manifest.xml to s3://files.biosimulations.org/simulations/6ada2269041d31202094ca8f/contents/manifest.xml\nCompleted 21.9 KiB/64.8 KiB (82.3 KiB/s) with 1 file(s) remaining\rCompleted 64.8 KiB/64.8 KiB (213.3 KiB/s) with 1 file(s) remaining\rupload: contents/BIOMD0000000012_url.xml to s3://files.biosimulations.org/simulations/6ada2269041d31202094ca8f/contents/BIOMD0000000012_url.xml\n\n\u001b[0;36m==================================================== Updating log (2) ===============================================\u001b[0m\nCompleted 5.2 KiB/5.2 KiB (33.8 KiB/s) with 1 file(s) remaining\rupload: ./rawLog.txt to s3://files.biosimulations.org/simulations/6ada2269041d31202094ca8f/rawLog.txt\n\n\u001b[0;36m================================================== Saving final log =================================================\u001b[0m\n\n\u001b[0;36m=========================================== Post-processing simulation run ==========================================\u001b[0m\n\u001b[0;32mRetrieve the logs of the simulation run ... succeeded.\nPost the logs to the API ... succeeded.\nProcess and upload the outputs of the simulation run ... succeeded.\nProcess the SED-ML file in the COMBINE archive ... succeeded.\nPost the SED-ML file to the API ... succeeded.\nProcess the manifest file in the COMBINE archive ... succeeded.\nPost thumbnails to the API ... succeeded.\u001b[0m\n\u001b[1;33mProcess the metadata in the COMBINE archive ... failed due to: The metadata of the file could not be found:\nThere was an error in processing the metadata file. More information is available at https://docs.biosimulations.org/concepts/conventions/simulation-project-metadata/. A validation tool is available at https://run.biosimulations.org/utils/validate-metadata.\nPost the metadata to the API ... failed due to: Metadata not found due to a dependent step failing.\nCreate thumbnails from the images specified in the manifest ... failed due to: Files not found:\nThere was an error creating thumbnails for the project. Please ensure that the images specified in the manifest are valid. A validation tool is available at https://run.biosimulations.org/utils/validate-project.\u001b[0m\n\u001b[0;31mRead the manifest and post the files to the API ... failed due to: Files could not be posted to API:\nThere was an error uploading file metadata to the API. Please try again and contact us if the problem persists.\u001b[0m\n\u001b[0;36m================================ Run complete. Thank you for using runBioSimulations! ===============================\u001b[0m","duration":0.221892,"sedDocuments":[]} \ No newline at end of file