Integration of Spectrum Meta Data

README.md

@@ -4,11 +4,12 @@ This repository contains unodes, resources, and tests for the following engines:
 - PymzMLToIDXGZ
 - PymzMLToMGF
 - ThermoRawFileParser
+- SpectrumMetaData
 
 
 # Unode Resources Installation Instructions
 
 # Instanovo
 
 In order to avoid dependency conflicts with other parts of urgap, Python must be verion 3.10 and the urgap[cloud] dependency must not be installed. After installation of urgap, Instanovo can be installed with
-"pip install instanovo".
+"pip install instanovo".

pyproject.toml

@@ -8,6 +8,10 @@ version = "0.0.1"
 description = "Urgap proteomics specific extension"
 requires-python = ">=3.10"
 dependencies = [
+    "urgap",
+    "numpy>=1.26,<2.0",
+    "pandas>2.0.0,<2.1.0",
+    "simepy>=0.3.2",
     "urgap",
     "pymzml>=2.5.0",
     "pyiohat>=1.9.0",

tests/integrationtests/wrappers/test_spectrum_meta_data.py

@@ -0,0 +1,46 @@
+import tempfile
+
+import numpy as np
+import pandas as pd
+import pytest
+
+import urgap
+
+
+def test_spectrum_meta_data_wrapper_bsa_mzml():
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        ufiles = urgap.UFileList(
+            [
+                urgap.UFile(
+                    uri=f"https://raw.githubusercontent.com/OpenMS/OpenMS/develop/share/OpenMS/examples/BSA"
+                    f"?uftype={urgap.uftypes.ms.converter.mzml.THERMORAWPARSER_MZML}#BSA1.mzML"
+                )
+            ]
+        )
+        urd = urgap.URunDict(
+            {
+                "parameters": {
+                    "ExtractSpectrumMetaData:1.0.0": {"-tf": "%Y-%m-%d %H:%M:%S"},
+                },
+                "unode_parameters": {
+                    "force": True,
+                    "storage_base_uri": f"file://{tmp_dir}",
+                },
+            }
+        )
+        test_node = urgap.init_unode("ExtractSpectrumMetaData:1.0.0")
+        response = test_node.run(ufiles, urd)
+        expected = {"run_shape": (1, 6), "size": 13.64, "median_mz": 532.68}
+
+        # 1. Assert the run metadata
+        assert response[0].path.exists()
+        df = pd.read_csv(response[0].path)
+        assert df.shape == expected["run_shape"]
+        assert df["file"][0] == ufiles[0].object_name == df["lineage_root"][0]
+        assert pytest.approx(df["file_size_mb"][0], 0.01) == expected["size"]
+        # assert np.isnan(df.stop_time[0])
+
+        # 2. Assert the spectrum metadata
+        assert response[1].path.exists()
+        df2 = pd.read_csv(response[1].path)
+        assert pytest.approx(df2.precursor_mz.median(), 0.01) == expected["median_mz"]

urgap/resources/ms_data_extraction/1_0_0/__init__.py

@@ -0,0 +1 @@
+"""Init SpectrumMetaData Unode."""

urgap/resources/ms_data_extraction/1_0_0/extract_meta_data.py

@@ -0,0 +1,122 @@
+"""Init SpectrumMetaData Unode."""
+
+import argparse
+
+from pathlib import Path
+
+from simepy.extract_meta_data import extract_meta_data
+
+
+def main(
+    input_file: str,
+    run_output_file: str | None,
+    spec_output_file: str | None,
+    noise_output_file: str | None,
+    unit_output_file: str | None,
+    time_format: str,
+    object_name: str,
+    lineage_root: str,
+) -> None:
+    """Extract scan info from mzml file using extract_scan_data().
+
+    Args:
+        input_file (PosixPath): path to input file
+        run_output_file (str/NoneType): path to run metadata output_file
+        spec_output_file (str/NoneType): path to spectrum metadata output_file
+        noise_output_file (str/NoneType): path to spectrum noise output_file
+        unit_output_file (str/NoneType): path to instrument units output_file
+        time_format (str): string defining the time_format to format into
+        object_name (str/NoneType): string to be used as file identifier
+        lineage_root (str/NoneType): name of the root file associated to the one, from which metadata is queried
+    """
+    rmd_df, imu_df, smd_df, sn_df = extract_meta_data(
+        input_file=input_file,
+        time_format=time_format,
+        object_name=object_name,
+        lineage_root=lineage_root,
+    )
+
+    if run_output_file is not None:
+        rmd_df.to_csv(run_output_file, index=False)
+
+    if spec_output_file is not None:
+        smd_df.to_csv(spec_output_file, index=False)
+
+    if unit_output_file is not None:
+        imu_df.to_csv(unit_output_file, index=False)
+
+    if noise_output_file is not None:
+        sn_df.to_csv(noise_output_file, index=False)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-i",
+        "--input_file",
+        dest="input_file",
+        help="input file",
+    )
+    parser.add_argument(
+        "-ro",
+        "--run_output_file",
+        type=lambda x: None if x == "None" else str(x),
+        dest="run_output_file",
+        help="run metadata output file",
+    )
+    parser.add_argument(
+        "-so",
+        "--spec_output_file",
+        type=lambda x: None if x == "None" else str(x),
+        dest="spec_output_file",
+        help="spectrum metadata output file",
+    )
+    parser.add_argument(
+        "-sno",
+        "--spec_noise_output_file",
+        type=lambda x: None if x == "None" else str(x),
+        dest="spec_noise_output_file",
+        help="spectrum noise output file",
+    )
+    parser.add_argument(
+        "-iuo",
+        "--instrument_unit_output_file",
+        type=lambda x: None if x == "None" else str(x),
+        dest="instrument_unit_output_file",
+        help="instrument unit output file",
+    )
+    parser.add_argument(
+        "-tf",
+        "--time_format",
+        default="%Y-%m-%d %H:%M:%S",
+        dest="time_format",
+        help="time format string",
+    )
+    parser.add_argument(
+        "-on",
+        "--object_name",
+        dest="object_name",
+        help="object_name string",
+    )
+    parser.add_argument(
+        "-lr",
+        "--lineage_root",
+        dest="lineage_root",
+        help="lineage_root string",
+    )
+    args = parser.parse_args()
+
+    if args.object_name is None:
+        args.object_name = Path(args.input_file).name
+    if args.lineage_root is None:
+        args.lineage_root = Path(args.input_file).resolve()
+    main(
+        input_file=args.input_file,
+        run_output_file=args.run_output_file,
+        spec_output_file=args.spec_output_file,
+        noise_output_file=args.spec_noise_output_file,
+        unit_output_file=args.instrument_unit_output_file,
+        time_format=args.time_format,
+        object_name=args.object_name,
+        lineage_root=args.lineage_root,
+    )

urgap/resources/ms_data_extraction/__init__.py

@@ -0,0 +1 @@
+"""Init SpectrumMetaData Unode."""

urgap/unodes/ms_data_extraction/__init__.py

@@ -0,0 +1 @@
+"""Init SpectrumMetaData Unode."""

urgap/unodes/ms_data_extraction/spectrum_meta_data/__init__.py

@@ -0,0 +1 @@
+"""Init SpectrumMetaData Unode."""

urgap/unodes/ms_data_extraction/spectrum_meta_data/spectrum_meta_data.py

@@ -0,0 +1,123 @@
+"""Urgap ExtractSpectrumMetaData wrapper."""
+
+import logging
+
+import urgap
+
+
+class ExtractSpectrumMetaData(urgap.unode.UNodeBase):
+    """Urgap wrapper for the ExtractSpectrumMetaData resource.
+
+    This wrapper calls the extract_meta_data resource from simepy package to extract
+    peak information from an input raw or mzml file. Note: for raw_file extraction the
+    cz_xcalibur package is required.
+    """
+
+    META_INFO = {
+        "name": "ExtractSpectrumMetaData",
+        "wrapper_version": {"major": 1, "minor": 0, "patch": 0},
+        "versions": [
+            {"version": "1.0.0", "exe_path": "ms_data_extraction/1_0_0/extract_meta_data.py"},
+        ],
+        "parameters_not_triggering_rerun": [],
+        # ^-- TODO: this does not seem to work. All tests are skipped, even if working
+        #       with mzml files only
+        "input_uftypes": {
+            urgap.uftypes.proteomics.THERMO_RAW: {
+                "min": 0,
+                "max": 1,
+            },
+            urgap.uftypes.any.MZML: {
+                "min": 0,
+                "max": 1,
+            },
+            urgap.uftypes.ms.converter.mzml.ANY: {
+                "min": 0,
+                "max": 1,
+            },
+        },
+        "output_uftypes": {
+            urgap.uftypes.ms.RUN_META_CSV: {"min": 1, "max": 1},
+            urgap.uftypes.ms.SPECTRA_META_CSV: {"min": 1, "max": 1},
+            urgap.uftypes.ms.SPECTRA_NOISE_CSV: {"min": 1, "max": 1},
+            urgap.uftypes.ms.INSTRUMENT_UNIT_CSV: {"min": 1, "max": 1},
+        },
+        "engine": None,
+        "engine_type": ("data_extractor",),
+        "citation": "Urgap team (2021)",
+    }
+
+    def __init__(self, *args: object, **kwargs: object) -> None:
+        """Initialize ExtractSpectrumMetaData class."""
+        super().__init__(*args, **kwargs)
+
+    def preflight(
+        self,
+        utrace: urgap.UTrace,
+    ) -> urgap.UTrace:
+        """Preflight routine for ExtractSpectrumMetaData wrapper.
+
+        Checks that only one file at a time is processed and if so, provides the file
+        path to the main function of the resource to extract metadata from.
+
+        Args:
+            utrace: Combination of urun_dict, ufile_list and unode.meta.
+
+        Returns:
+            UTrace object, combination of urun_dict, ufile_list and unode.meta.
+        """
+        index_dict = utrace.input_files.get_index_groups_by_uftypes()
+        uftype_to_process = None
+        for uftype, index_list in index_dict.items():
+            if len(index_list) > 0:
+                if uftype_to_process is None:
+                    uftype_to_process = uftype
+                else:
+                    logging.warning(
+                        "Meta node received multiple mass spec files, "
+                        "however node can only process one at the time!"
+                        "Therefore will process %s but not %s!",
+                        uftype_to_process,
+                        uftype,
+                    )
+
+        idx_ms = index_dict[uftype_to_process][0]
+        input_file = utrace.input_files[idx_ms]
+
+        run_meta_output_file = utrace.output_files.get_path_objects_by_uftype(
+            urgap.uftypes.ms.RUN_META_CSV,
+        )[0]
+        spec_meta_output_file = utrace.output_files.get_path_objects_by_uftype(
+            urgap.uftypes.ms.SPECTRA_META_CSV,
+        )[0]
+        spec_noise_output_file = utrace.output_files.get_path_objects_by_uftype(
+            urgap.uftypes.ms.SPECTRA_NOISE_CSV,
+        )[0]
+        instrument_unit_output_file = utrace.output_files.get_path_objects_by_uftype(
+            urgap.uftypes.ms.INSTRUMENT_UNIT_CSV,
+        )[0]
+        object_name = input_file.object_name
+        utrace.urun_dict.command_list = [
+            "python",
+            str(self.exe_path),
+            "-i",
+            str(input_file.path),
+            "-ro",
+            str(run_meta_output_file),
+            "-so",
+            str(spec_meta_output_file),
+            "-sno",
+            str(spec_noise_output_file),
+            "-iuo",
+            str(instrument_unit_output_file),
+            "-on",
+            str(object_name),
+            "-lr",
+            str(object_name),
+        ]
+        for parameter_key, parameter_value in utrace.urun_dict.parameters[
+            f"{self.META_INFO['unode_full_identifier']}"
+        ].items():
+            utrace.urun_dict.command_list.extend([parameter_key, parameter_value])
+
+        return utrace