diff --git a/.cache/doc/bat/build_doc.bat b/.cache/doc/bat/build_doc.bat index 57444ef..5d7a3e7 100644 --- a/.cache/doc/bat/build_doc.bat +++ b/.cache/doc/bat/build_doc.bat @@ -1,7 +1,7 @@ - -echo off -SET pth=%~dp0 -SET pth=%pth:~0, -5% -echo on - -bash %pth%/bash/build_doc.sh %* + +echo off +SET pth=%~dp0 +SET pth=%pth:~0, -5% +echo on + +bash %pth%/bash/build_doc.sh %* diff --git a/.cache/doc/bat/generate_doc.bat b/.cache/doc/bat/generate_doc.bat index ceee36c..cf646a8 100644 --- a/.cache/doc/bat/generate_doc.bat +++ b/.cache/doc/bat/generate_doc.bat @@ -1,7 +1,7 @@ - -echo off -SET pth=%~dp0 -SET pth=%pth:~0, -5% -echo on - -bash %pth%/bash/generate_doc.sh %* + +echo off +SET pth=%~dp0 +SET pth=%pth:~0, -5% +echo on + +bash %pth%/bash/generate_doc.sh %* diff --git a/.github/workflows/sync-bitbucket.yml b/.github/workflows/sync-bitbucket.yml deleted file mode 100644 index 0df9264..0000000 --- a/.github/workflows/sync-bitbucket.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Sync voXSim remote repository - -# GitHub App/Action mirror-repository: https://github.com/yesolutions/mirror-action - -on: - push: - branches: - - master - -jobs: - sync_mirror_repo: - runs-on: ubuntu-latest - - environment: - name: MIRRORING - - steps: - - uses: actions/checkout@v3 - with: - ref: "master" - fetch-depth: 0 - - - uses: yesolutions/mirror-action@master - with: - REMOTE: "${{ secrets.VOXSIM_REMOTE }}" - GIT_SSH_PRIVATE_KEY: ${{ secrets.VOXSIM_SSH_PRIVATE_KEY }} - GIT_SSH_KNOWN_HOSTS: ${{ secrets.VOXSIM_SSH_KNOWN_HOSTS }} diff --git a/.github/workflows/sync-linum-mirror-repo.yml b/.github/workflows/sync-linum-mirror-repo.yml new file mode 100644 index 0000000..23ef078 --- /dev/null +++ b/.github/workflows/sync-linum-mirror-repo.yml @@ -0,0 +1,28 @@ +name: Sync LINUM mirror repository + +# GitHub App/Action mirror-repository: https://github.com/yesolutions/mirror-action + +on: + push: + branches: + - master + +jobs: + sync_mirror_repo: + runs-on: ubuntu-22.04 + + environment: + name: production + url: https://github.com/linum-uqam/inm5803-ete2022-benoit-dubreuil + + steps: + - uses: actions/checkout@v3 + with: + ref: 'master' + fetch-depth: 0 + + - uses: yesolutions/mirror-action@master + with: + REMOTE: 'git@github.com:linum-uqam/voxsim.git' + GIT_SSH_PRIVATE_KEY: ${{ secrets.LINUM_DEPLOY_GIT_SSH_PRIVATE_KEY }} # The public key is a deploy key on the remote repository + GIT_SSH_NO_VERIFY_HOST: 'true' # We're pushing from a public repo to a public repo, nothing can really be stolen. \ No newline at end of file diff --git a/config.json b/config.json deleted file mode 100644 index 3d18623..0000000 --- a/config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "singularity_path": ".", - "singularity_name": "voxsim_singularity_latest.sif" -} - diff --git a/config.py b/config.py deleted file mode 100644 index 232a35c..0000000 --- a/config.py +++ /dev/null @@ -1,13 +0,0 @@ -import json -from os.path import dirname, join, realpath - - -def get_config(path=dirname(realpath(__file__))): - with open(join(path, "config.json")) as f: - return json.load(f) - - -def override_global_config(config): - print(config) - with open(join(dirname(realpath(__file__)), "config.json"), "w+") as f: - json.dump(config, f) diff --git a/scripts/geometry_factory.py b/scripts/geometry_factory.py index e589ada..241d4d5 100755 --- a/scripts/geometry_factory.py +++ b/scripts/geometry_factory.py @@ -1,13 +1,12 @@ #!/usr/bin/env python import argparse +import pathlib from math import pi -from os import makedirs from tempfile import mkdtemp from simulator.factory import GeometryFactory, Plane - resolution = [10, 10, 10] spacing = [2, 2, 2] @@ -42,7 +41,7 @@ ] -def get_geometry_parameters(output_folder, output_naming): +def get_geometry_parameters(output_folder: pathlib.Path, output_naming: str): geometry_handler = GeometryFactory.get_geometry_handler(resolution, spacing) bundle1 = GeometryFactory.create_bundle( @@ -76,15 +75,15 @@ def get_geometry_parameters(output_folder, output_naming): if __name__ == "__main__": parser = argparse.ArgumentParser("Geometry Factory Example Script") parser.add_argument( - "--out", type=str, required=False, help="Output directory for the files" + "--out", type=pathlib.Path, help="Output directory for the files" ) args = parser.parse_args() - if "out" in args and args.out: - dest = args.out - makedirs(args.out, exist_ok=True) + if args.out: + dest: pathlib.Path = args.out + dest.mkdir(parents=True, exist_ok=True) else: - dest = mkdtemp(prefix="geo_factory") + dest = pathlib.Path(mkdtemp(prefix="geo_factory")) print("Script execution results are in : {}".format(dest)) get_geometry_parameters(dest, "geometry") diff --git a/scripts/geometry_factory_multi_cluster.py b/scripts/geometry_factory_multi_cluster.py index 5004616..c542e20 100755 --- a/scripts/geometry_factory_multi_cluster.py +++ b/scripts/geometry_factory_multi_cluster.py @@ -1,13 +1,12 @@ #!/usr/bin/env python3 import argparse +import pathlib from math import pi -from os import makedirs from tempfile import mkdtemp from simulator.factory import GeometryFactory, Plane - resolution = [10, 10, 10] spacing = [2, 2, 2] @@ -43,7 +42,7 @@ ] -def run_multi_clusters(output_folder, output_naming): +def run_multi_clusters(output_folder: pathlib.Path, output_naming: str): geometry_handler = GeometryFactory.get_geometry_handler(resolution, spacing) bundle1 = GeometryFactory.create_bundle( @@ -87,15 +86,15 @@ def run_multi_clusters(output_folder, output_naming): if __name__ == "__main__": parser = argparse.ArgumentParser("Geometry Multi Clusters Example Script") parser.add_argument( - "--out", type=str, required=False, help="Output directory for the files" + "--out", type=pathlib.Path, help="Output directory for the files" ) args = parser.parse_args() - if "out" in args and args.out: - dest = args.out - makedirs(args.out, exist_ok=True) + if args.out: + dest: pathlib.Path = args.out + dest.mkdir(parents=True, exist_ok=True) else: - dest = mkdtemp(prefix="geo_factory_mc") + dest = pathlib.Path(mkdtemp(prefix="geo_factory_mc")) print("Script execution results are in : {}".format(dest)) run_multi_clusters(dest, "multi_clusters") diff --git a/scripts/simulation_runner.py b/scripts/legacy_simulation_runner.py old mode 100755 new mode 100644 similarity index 75% rename from scripts/simulation_runner.py rename to scripts/legacy_simulation_runner.py index e00329a..1aeff73 --- a/scripts/simulation_runner.py +++ b/scripts/legacy_simulation_runner.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 import argparse -from os import makedirs -from os.path import join +import pathlib from tempfile import mkdtemp from scripts.geometry_factory import get_geometry_parameters @@ -10,7 +9,7 @@ from simulator.runner.legacy import SimulationRunner -def run_simulation(output_folder): +def run_simulation(output_folder: pathlib.Path): geometry_parameters = get_geometry_parameters( output_folder, "runner_test_geometry" ) @@ -32,7 +31,7 @@ def run_simulation(output_folder): output_folder, "runner_test_simulation_standalone" ) - standalone_output = join(output_folder, "standalone_test") + standalone_output = output_folder / "standalone_test" runner.run_simulation_standalone( standalone_output, output_folder, simulation_parameters, "standalone" @@ -42,15 +41,15 @@ def run_simulation(output_folder): if __name__ == "__main__": parser = argparse.ArgumentParser("Simulation Runner Example Script") parser.add_argument( - "--out", type=str, required=False, help="Output directory for the files" + "--out", type=pathlib.Path, help="Output directory for the files" ) args = parser.parse_args() - if "out" in args and args.out: - dest = args.out - makedirs(args.out, exist_ok=True) + if args.out: + dest: pathlib.Path = args.out + dest.mkdir(parents=True, exist_ok=True) else: - dest = mkdtemp(prefix="sim_runner") + dest = pathlib.Path(mkdtemp(prefix="sim_runner")) print("Script execution results are in : {}".format(dest)) run_simulation(dest) diff --git a/scripts/phantom_generation_runner.py b/scripts/phantom_generation_runner.py new file mode 100644 index 0000000..031f783 --- /dev/null +++ b/scripts/phantom_generation_runner.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 + +import argparse +import pathlib + +from scripts.geometry_factory import get_geometry_parameters +from simulator.runner.simulation_runner import SimulationRunner + + +def run_simulation(output_folder: pathlib.Path): + geometry_parameters = get_geometry_parameters( + output_folder, "runner_test_geometry" + ) + + runner = SimulationRunner() + runner.generate_phantom( + run_name="runner_test_simulation_standalone", + phantom_infos=geometry_parameters, + output_folder=output_folder, + output_nifti=False, + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser("Simulation Runner Example Script") + parser.add_argument( + "--out", type=pathlib.Path, default="./out/", help="Output directory for the files" + ) + + args = parser.parse_args() + dest: pathlib.Path = args.out + dest.mkdir(parents=True, exist_ok=True) + dest = dest.resolve(strict=True) + + print("Script execution results are in : {}".format(dest)) + run_simulation(dest) diff --git a/scripts/simulation_factory.py b/scripts/simulation_factory.py index 433c85f..0d92345 100755 --- a/scripts/simulation_factory.py +++ b/scripts/simulation_factory.py @@ -1,7 +1,7 @@ #!/usr/bin/env python import argparse -from os import makedirs +import pathlib from tempfile import mkdtemp from random import uniform @@ -12,7 +12,7 @@ from simulator.utils.test_helpers import GeometryHelper -def get_simulation_parameters(output_folder, output_naming): +def get_simulation_parameters(output_folder: pathlib.Path, output_naming: str): fiber_compartment = SimulationFactory.generate_fiber_stick_compartment( 0.007, 900, 80, SimulationFactory.CompartmentType.INTRA_AXONAL ) @@ -71,15 +71,15 @@ def get_simulation_parameters(output_folder, output_naming): if __name__ == "__main__": parser = argparse.ArgumentParser("Simulation Factory Example Script") parser.add_argument( - "--out", type=str, required=False, help="Output directory for the files" + "--out", type=pathlib.Path, help="Output directory for the files" ) args = parser.parse_args() - if "out" in args and args.out: - dest = args.out - makedirs(args.out, exist_ok=True) + if args.out: + dest: pathlib.Path = args.out + dest.mkdir(parents=True, exist_ok=True) else: - dest = mkdtemp(prefix="sim_factory") + dest = pathlib.Path(mkdtemp(prefix="sim_factory")) print("Script execution results are in : {}".format(dest)) get_simulation_parameters(dest, "simulation") diff --git a/scripts/wall_effect.py b/scripts/wall_effect.py index f0571b6..eba1ee3 100755 --- a/scripts/wall_effect.py +++ b/scripts/wall_effect.py @@ -1,9 +1,8 @@ #!/usr/bin/env python3 import argparse +import pathlib from math import pi, sqrt -from os import makedirs -from os.path import join from tempfile import mkdtemp from numpy import mean @@ -11,9 +10,8 @@ from simulator.factory import GeometryFactory, Plane, SimulationFactory from simulator.runner.legacy import SimulationRunner - resolution = [40, 40, 40] -spacing = [1, 1, 1] +spacing = [2, 2, 2] point_per_centroid = 30 x_anchors = [0.15, 0.25, 0.5, 0.75, 0.85] @@ -26,7 +24,7 @@ cluster_center = [0.5, 0.5, 0.5] -def create_geometry(output_folder, output_naming, fibers_per_bundle): +def create_geometry(output_folder: pathlib.Path, output_naming: str, fibers_per_bundle): geometry_handler = GeometryFactory.get_geometry_handler(resolution, spacing) bundle1 = GeometryFactory.create_bundle( @@ -54,7 +52,7 @@ def create_geometry(output_folder, output_naming, fibers_per_bundle): def create_split_geometry( - output_folder, output_naming, fibers_per_bundle, rotation=None + output_folder: pathlib.Path, output_naming: str, fibers_per_bundle, rotation=None ): geometry_handler = GeometryFactory.get_geometry_handler(resolution, spacing) @@ -142,7 +140,7 @@ def create_simulation_b1000(geometry_handler, output_folder, output_naming): def create_simulation_multishell( - geometry_handler, output_folder, output_naming + geometry_handler, output_folder, output_naming ): base_multishell_simulation_handler = get_base_simulation_handler( geometry_handler @@ -166,18 +164,18 @@ def create_simulation_multishell( if __name__ == "__main__": parser = argparse.ArgumentParser("Wall Effect Example Script") parser.add_argument( - "--out", type=str, required=False, help="Output directory for the files" + "--out", type=pathlib.Path, help="Output directory for the files" ) args = parser.parse_args() if "out" in args and args.out: - dest = args.out - makedirs(args.out, exist_ok=True) + dest: pathlib.Path = args.out + dest.mkdir(parents=True, exist_ok=True) else: - dest = mkdtemp(prefix="wall_effect") + dest = pathlib.Path(mkdtemp(prefix="wall_effect")) print("Script execution results are in : {}".format(dest)) - makedirs(join(dest, "runner_outputs"), exist_ok=True) + (dest / "runner_outputs").mkdir(parents=True, exist_ok=True) geometry_infos, geometry_handler = create_geometry( dest, "geometry", n_fibers_per_bundle @@ -189,7 +187,7 @@ def create_simulation_multishell( SimulationRunner( "simulation_b1000", geometry_infos, simulation_b1000_infos - ).run(join(dest, "runner_outputs"), relative_fiber_compartment=False) + ).run(dest / "runner_outputs", relative_fiber_compartment=False) geometry_infos, geometry_handler = create_split_geometry( dest, "geometry_split_b1", 3000 @@ -201,7 +199,7 @@ def create_simulation_multishell( SimulationRunner( "simulation_split_b1_b1000", geometry_infos, simulation_b1000_infos - ).run(join(dest, "runner_outputs")) + ).run(dest / "runner_outputs") geometry_infos, geometry_handler = create_split_geometry( dest, "geometry_split_b2", 3000, pi @@ -213,7 +211,7 @@ def create_simulation_multishell( SimulationRunner( "simulation_split_b2_b1000", geometry_infos, simulation_b1000_infos - ).run(join(dest, "runner_outputs")) + ).run(dest / "runner_outputs") simulation_multishell_infos = create_simulation_multishell( geometry_handler, dest, "simulation_multishell" @@ -221,4 +219,4 @@ def create_simulation_multishell( SimulationRunner( "simulation_multishell", geometry_infos, simulation_b1000_infos - ).run(join(dest, "runner_outputs")) + ).run(dest / "runner_outputs") diff --git a/setup.py b/setup.py index 19e4249..2e76f14 100644 --- a/setup.py +++ b/setup.py @@ -3,18 +3,16 @@ from simulator.utils.setup.documentation import DocCommand - if __name__ == "__main__": setup( name="simulation_generator", - version="1.0.0", + version="1.0.9", packages=find_packages(exclude=("tests", "tests.*")), url="", license="", author="avcaron", author_email="", description="", - data_files=[(".", ["config.py", "config.json"])], scripts=list( filter(lambda s: "init" not in s, glob.glob("scripts/*.py")) ), diff --git a/simulator/default.py b/simulator/default.py new file mode 100644 index 0000000..fb8f1d8 --- /dev/null +++ b/simulator/default.py @@ -0,0 +1,7 @@ +import typing +import pathlib + +SINGULARITY_PATH: typing.Final[pathlib.Path] = pathlib.Path() +SINGULARITY_NAME: typing.Final[str] = "voxsim_singularity_latest.sif" +SINGULARITY: typing.Final[pathlib.Path] = SINGULARITY_PATH / SINGULARITY_NAME +SINGULARITY_EXEC: typing.Final[str] = "singularity" diff --git a/simulator/factory/common/__init__.py b/simulator/factory/common/__init__.py index fd9d22f..8e48e44 100755 --- a/simulator/factory/common/__init__.py +++ b/simulator/factory/common/__init__.py @@ -1 +1 @@ -from .common import AttributeAsDictClass +from ._common import AttributeAsDictClass diff --git a/simulator/factory/common/common.py b/simulator/factory/common/_common.py old mode 100755 new mode 100644 similarity index 100% rename from simulator/factory/common/common.py rename to simulator/factory/common/_common.py diff --git a/simulator/factory/geometry_factory/__init__.py b/simulator/factory/geometry_factory/__init__.py index b69e045..6cbe349 100755 --- a/simulator/factory/geometry_factory/__init__.py +++ b/simulator/factory/geometry_factory/__init__.py @@ -1,3 +1,3 @@ -from .geometry_factory import GeometryFactory +from ._geometry_factory import GeometryFactory from simulator.factory.geometry_factory.utils.plane import Plane from simulator.factory.geometry_factory.utils.rotation import Rotation diff --git a/simulator/factory/geometry_factory/geometry_factory.py b/simulator/factory/geometry_factory/_geometry_factory.py old mode 100755 new mode 100644 similarity index 99% rename from simulator/factory/geometry_factory/geometry_factory.py rename to simulator/factory/geometry_factory/_geometry_factory.py index 898bfcc..b68f89b --- a/simulator/factory/geometry_factory/geometry_factory.py +++ b/simulator/factory/geometry_factory/_geometry_factory.py @@ -107,7 +107,7 @@ def create_bundle(radius, symmetry, n_point_per_centroid, anchors=list): n_point_per_centroid : int Number of points sampling points along the spline representing the centroid - anchors : list(list(int)) or list, optional + anchors : list(list(float)) or list, optional List of anchor points defining the centroid of the bundle, default : [] diff --git a/simulator/factory/geometry_factory/handlers/geometry_handler.py b/simulator/factory/geometry_factory/handlers/geometry_handler.py index 899944d..c6bc9f7 100755 --- a/simulator/factory/geometry_factory/handlers/geometry_handler.py +++ b/simulator/factory/geometry_factory/handlers/geometry_handler.py @@ -1,11 +1,16 @@ +import logging +import pathlib + from copy import deepcopy -from os import makedirs, path from ..features.ORM.config_builder import ConfigBuilder from .geometry_infos import GeometryInfos +_logger = logging.getLogger(__name__) + class GeometryHandler: + def __init__(self, resolution, spacing, clusters=None, spheres=None): self._parameters_dict = { "resolution": resolution, @@ -73,15 +78,21 @@ def _get_number_of_clusters(self): return len(self._parameters_dict["clusters"]) def generate_json_configuration_files( - self, output_naming, simulation_path="" - ): - if not path.exists(simulation_path): - makedirs(simulation_path, exist_ok=True) + self, output_naming: str, simulation_path: pathlib.Path = pathlib.Path() + ) -> GeometryInfos: + simulation_path.mkdir(parents=True, exist_ok=True) + + try: + simulation_path = simulation_path.resolve(strict=True) + except FileNotFoundError as exc: + _logger.exception( + f"simulation_path does not exist, even after the creation of its directories: {simulation_path}", + exc_info=exc) + raise exc with open( - path.join(simulation_path, output_naming + "_base.json"), "w+" + simulation_path / (output_naming + "_base.json"), "w+" ) as base_file: - world = ConfigBuilder.create_world( len(self.get_resolution()), self.get_resolution() ) @@ -109,11 +120,8 @@ def generate_json_configuration_files( for cluster_idx in range(len(self._parameters_dict["clusters"])): with open( - path.join( - simulation_path, - output_naming + "_f_{}.vspl".format(cluster_idx), - ), - "w+", + simulation_path / (output_naming + "_f_{}.vspl".format(cluster_idx)), + "w+", ) as f: f.write( self._parameters_dict["clusters"][cluster_idx].serialize() diff --git a/simulator/factory/geometry_factory/handlers/geometry_infos.py b/simulator/factory/geometry_factory/handlers/geometry_infos.py old mode 100755 new mode 100644 index 0f8839c..556a9f1 --- a/simulator/factory/geometry_factory/handlers/geometry_infos.py +++ b/simulator/factory/geometry_factory/handlers/geometry_infos.py @@ -1,9 +1,10 @@ from ...common import AttributeAsDictClass +import pathlib class GeometryInfos(AttributeAsDictClass): def __init__( - self, file_path, base_file, resolution, spacing, n_maps, **kwargs + self, file_path: pathlib.Path, base_file: str, resolution, spacing, n_maps, **kwargs ): super().__init__(**kwargs) self.generate_new_key("file_path", file_path) @@ -12,16 +13,16 @@ def __init__( self.generate_new_key("spacing", spacing) self.generate_new_key("n_maps", n_maps) - def get_file_path(self): + def get_file_path(self) -> pathlib.Path: return self._file_path - def set_file_path(self, path): + def set_file_path(self, path: pathlib.Path): self._file_path = path - def get_base_file_name(self): + def get_base_file_name(self) -> str: return self._base_file - def set_base_file_name(self, name): + def set_base_file_name(self, name: str): self._base_file = name def get_resolution(self): diff --git a/simulator/factory/simulation_factory/__init__.py b/simulator/factory/simulation_factory/__init__.py index 1cb8895..fe0c0a9 100755 --- a/simulator/factory/simulation_factory/__init__.py +++ b/simulator/factory/simulation_factory/__init__.py @@ -1 +1 @@ -from .simulation_factory import SimulationFactory +from ._simulation_factory import SimulationFactory diff --git a/simulator/factory/simulation_factory/simulation_factory.py b/simulator/factory/simulation_factory/_simulation_factory.py old mode 100755 new mode 100644 similarity index 100% rename from simulator/factory/simulation_factory/simulation_factory.py rename to simulator/factory/simulation_factory/_simulation_factory.py diff --git a/simulator/factory/simulation_factory/handlers/simulation_handler.py b/simulator/factory/simulation_factory/handlers/simulation_handler.py index 9affdbf..428486f 100755 --- a/simulator/factory/simulation_factory/handlers/simulation_handler.py +++ b/simulator/factory/simulation_factory/handlers/simulation_handler.py @@ -1,5 +1,6 @@ +import pathlib + from lxml.etree import Element, SubElement, tostring -from os import makedirs, path from simulator.factory.simulation_factory.parameters import ( AcquisitionProfile, @@ -70,10 +71,10 @@ def add_compartment(self, compartment): return self def generate_xml_configuration_file( - self, output_naming, simulation_path="" + self, output_naming: str, simulation_path: pathlib.Path = pathlib.Path() ): - if not path.exists(simulation_path): - makedirs(simulation_path, exist_ok=True) + simulation_path.mkdir(parents=True, exist_ok=True) + simulation_path = simulation_path.resolve(strict=True) data = Element("fiberfox") image_element = SubElement(data, "image") @@ -87,7 +88,7 @@ def generate_xml_configuration_file( xml_string = tostring(data, pretty_print=True).decode("utf-8") with open( - path.join(simulation_path, output_naming + ".ffp"), "w+" + simulation_path / (output_naming + ".ffp"), "w+" ) as f: f.write(xml_string) diff --git a/simulator/factory/simulation_factory/handlers/simulation_infos.py b/simulator/factory/simulation_factory/handlers/simulation_infos.py index 1ba8d57..f7dd354 100755 --- a/simulator/factory/simulation_factory/handlers/simulation_infos.py +++ b/simulator/factory/simulation_factory/handlers/simulation_infos.py @@ -1,23 +1,24 @@ from ...common import AttributeAsDictClass +import pathlib class SimulationInfos(AttributeAsDictClass): - def __init__(self, file_path, simulation_file_name, ids): + def __init__(self, file_path: pathlib.Path, simulation_file_name: str, ids): super().__init__() self.generate_new_key("file_path", file_path) self.generate_new_key("param_file", simulation_file_name) self.generate_new_key("compartment_ids", ids) - def get_file_path(self): + def get_file_path(self) -> pathlib.Path: return self._file_path - def set_file_path(self, file_path): + def set_file_path(self, file_path: pathlib.Path): self._file_path = file_path - def get_simulation_file_name(self): + def get_simulation_file_name(self) -> str: return self._param_file - def set_simulation_file_name(self, name): + def set_simulation_file_name(self, name: str): self._param_file = name @classmethod diff --git a/simulator/runner/__init__.py b/simulator/runner/__init__.py index 119609b..49dc376 100755 --- a/simulator/runner/__init__.py +++ b/simulator/runner/__init__.py @@ -1 +1,2 @@ +from .config import SingularityConfig from .simulation_runner import SimulationRunner diff --git a/simulator/runner/config.py b/simulator/runner/config.py new file mode 100644 index 0000000..303675e --- /dev/null +++ b/simulator/runner/config.py @@ -0,0 +1,10 @@ +import dataclasses +import pathlib + +import simulator.default as default + + +@dataclasses.dataclass +class SingularityConfig: + singularity: pathlib.Path = default.SINGULARITY + singularity_exec: str = default.SINGULARITY_EXEC diff --git a/simulator/runner/datastore.py b/simulator/runner/datastore.py index 47fb18d..3afead8 100644 --- a/simulator/runner/datastore.py +++ b/simulator/runner/datastore.py @@ -1,4 +1,4 @@ -from os.path import basename, exists, join +import pathlib from shutil import copyfile from tempfile import TemporaryDirectory @@ -10,16 +10,16 @@ class Datastore: def __init__( - self, - simulation_path, - fibers, - compartment_ids, - inter_axonal_fraction=None, + self, + simulation_path: pathlib.Path, + fibers: pathlib.Path, + compartment_ids, + inter_axonal_fraction=None, ): - self.fibers = fibers + self.fibers: pathlib.Path = fibers self.compartments = [] self.ids = compartment_ids - self.stage_path = simulation_path + self.stage_path: pathlib.Path = simulation_path self.iaf = inter_axonal_fraction self._temp = TemporaryDirectory() @@ -32,12 +32,9 @@ def get_bind_paths(self, bind_compartments=True): self.compartments if bind_compartments else list() ) - def load_compartments(self, input_folder, run_name, use_nifti=True): + def load_compartments(self, input_folder: pathlib.Path, run_name, use_nifti=True): extension = "nii.gz" if use_nifti else "nrrd" - fiber_fraction = join( - input_folder, - "{}_phantom_mergedBundlesMaps.{}".format(run_name, extension), - ) + fiber_fraction = input_folder / "{}_phantom_mergedBundlesMaps.{}".format(run_name, extension) inter_id = SimulationFactory.CompartmentType.INTER_AXONAL.value extra1_id = SimulationFactory.CompartmentType.EXTRA_AXONAL_1.value @@ -50,11 +47,8 @@ def load_compartments(self, input_folder, run_name, use_nifti=True): self.add_compartment(fiber_fraction) if extra1_id in self.ids or extra2_id in self.ids: - ellipses = join( - input_folder, - "{}_mergedEllipsesMaps.{}".format(run_name, extension), - ) - if exists(ellipses): + ellipses: pathlib.Path = input_folder / "{}_mergedEllipsesMaps.{}".format(run_name, extension) + if ellipses.exists(): self.add_compartment(ellipses) if extra1_id in self.ids and extra2_id in self.ids: self.add_compartment("generate") @@ -62,8 +56,8 @@ def load_compartments(self, input_folder, run_name, use_nifti=True): self.add_compartment("generate") assert ( - len(list(filter(lambda c: c == "generate", self.compartments))) - <= 1 + len(list(filter(lambda c: c == "generate", self.compartments))) + <= 1 ) if "generate" in self.compartments: @@ -73,19 +67,18 @@ def add_compartment(self, filepath): self.compartments.append(filepath) def stage_compartments(self, run_name): - extension = ".".join(basename(self.compartments[0]).split(".")[1:]) + extension = pathlib.Path() / ( + pathlib.PurePath("").joinpath( + *pathlib.PurePath(self.compartments[0]).name.split(".")[1:] + ) + ) for m, cmp_id in zip(self.compartments, self.ids): copyfile( m, - join( - self.stage_path, - "{}_simulation.ffp_VOLUME{}.{}".format( - run_name, cmp_id, extension - ), - ), + self.stage_path / "{}_simulation.ffp_VOLUME{}.{}".format(run_name, cmp_id, extension), ) - def generate_inter_axonal_fraction(self, run_name, fiber_fraction): + def generate_inter_axonal_fraction(self, run_name, fiber_fraction: pathlib.Path): img = nib.load(fiber_fraction) fraction = img.get_fdata() inter_fraction = self.iaf * fraction @@ -93,19 +86,15 @@ def generate_inter_axonal_fraction(self, run_name, fiber_fraction): nib.save( nib.Nifti1Image(inter_fraction, img.affine, img.header), - join(self._get_temp(), "{}_inter.nii.gz".format(run_name)), + self._get_temp_path() / "{}_inter.nii.gz".format(run_name), ) nib.save( nib.Nifti1Image(intra_fraction, img.affine, img.header), - join(self._get_temp(), "{}_intra.nii.gz".format(run_name)), + self._get_temp_path() / "{}_intra.nii.gz".format(run_name), ) - self.add_compartment( - join(self._get_temp(), "{}_intra.nii.gz".format(run_name)) - ) - self.add_compartment( - join(self._get_temp(), "{}_inter.nii.gz".format(run_name)) - ) + self.add_compartment(self._get_temp_path() / "{}_intra.nii.gz".format(run_name)) + self.add_compartment(self._get_temp_path() / "{}_inter.nii.gz".format(run_name)) def generate_extra_axonal_fraction(self, run_name): other_fractions = list( @@ -122,13 +111,16 @@ def generate_extra_axonal_fraction(self, run_name): nib.save( nib.Nifti1Image(extra, ref.affine, ref.header), - join(self._get_temp(), "{}_extra.nii.gz".format(run_name)), + self._get_temp_path() / "{}_extra.nii.gz".format(run_name), ) self.add_compartment( - join(self._get_temp(), "{}_extra.nii.gz".format(run_name)) + self._get_temp_path() / "{}_extra.nii.gz".format(run_name) ) def _get_temp(self): assert self._temp is not None return self._temp + + def _get_temp_path(self) -> pathlib.Path: + return pathlib.Path(self._get_temp().name) diff --git a/simulator/runner/legacy.py b/simulator/runner/legacy.py index 463b321..ec05b53 100644 --- a/simulator/runner/legacy.py +++ b/simulator/runner/legacy.py @@ -1,8 +1,7 @@ from asyncio import get_event_loop, new_event_loop, set_event_loop import logging +import pathlib from multiprocessing import Process -from os import makedirs, path -from os.path import basename, exists from shutil import copyfile from subprocess import PIPE, Popen @@ -10,47 +9,42 @@ from numpy import ones_like, sum import nrrd -from config import get_config +from .config import SingularityConfig from ..exceptions import SimulationRunnerException from ..utils.logging import RTLogging +from ..factory.geometry_factory.handlers import GeometryInfos +from ..factory.simulation_factory.handlers import SimulationInfos - -logger = logging.getLogger(basename(__file__).split(".")[0]) +_logger = logging.getLogger(__name__) class SimulationRunner: + def __init__( - self, - base_naming, - geometry_infos, - simulation_infos=None, - singularity_conf=get_config(), - output_nifti=False, + self, + base_naming, + geometry_infos: GeometryInfos, + simulation_infos: SimulationInfos = None, + singularity_conf=SingularityConfig(), + output_nifti=False, ): - self._geometry_path = geometry_infos["file_path"] - self._geometry_base_file = geometry_infos["base_file"] + self._geometry_path: pathlib.Path = geometry_infos["file_path"].resolve(strict=True) + self._geometry_base_file: str = geometry_infos["base_file"] self._geometry_resolution = geometry_infos["resolution"] self._geometry_spacing = geometry_infos["spacing"] - self._base_naming = base_naming - self._geometry_base_naming = base_naming + self._base_naming: str = base_naming + self._geometry_base_naming: str = base_naming if simulation_infos: self._number_of_maps = len(simulation_infos["compartment_ids"]) - self._simulation_path = simulation_infos["file_path"] - self._simulation_parameters = simulation_infos["param_file"] + self._simulation_path: pathlib.Path = simulation_infos["file_path"].resolve(strict=True) + self._simulation_parameters: str = simulation_infos["param_file"] self._compartment_ids = simulation_infos["compartment_ids"] singularity_conf = ( - singularity_conf if singularity_conf else get_config() - ) - self._singularity = path.join( - singularity_conf["singularity_path"], - singularity_conf["singularity_name"], - ) - self._singularity_exec = ( - singularity_conf["singularity_exec"] - if "singularity_exec" in singularity_conf - else "singularity" + singularity_conf if singularity_conf else SingularityConfig() ) + self._singularity: pathlib.Path = singularity_conf.singularity.resolve(strict=True) + self._singularity_exec: str = singularity_conf.singularity_exec self._run_simulation = True if simulation_infos else False self._extension = "nii.gz" if output_nifti else "nrrd" @@ -60,58 +54,52 @@ def __init__( self._save_image = self._save_nifti if output_nifti else self._save_nrrd self._event_loop = new_event_loop() - def change_base_naming(self, name): + def change_base_naming(self, name: str): self._base_naming = name - def set_geometry_base_naming(self, name): + def set_geometry_base_naming(self, name: str): self._geometry_base_naming = name def run_simulation_dwimage( - self, output_folder, image_file, simulation_infos, test_mode=False + self, + output_folder: pathlib.Path, + image_file: pathlib.Path, + simulation_infos: SimulationInfos, + test_mode=False ): self._start_loop_if_closed() - simulation_output_folder = path.join( - output_folder, "simulation_outputs" - ) - if not path.exists(simulation_output_folder): - makedirs(simulation_output_folder, exist_ok=True) + + simulation_output_folder: pathlib.Path = output_folder / "simulation_outputs" + simulation_output_folder.mkdir(parents=True, exist_ok=True) + simulation_output_folder = simulation_output_folder.resolve(strict=True) + + output_folder = output_folder.resolve(strict=True) simulation_command = ( "{} run -B {} --app launch_mitk {} -p {} -i {} -o {} {}".format( self._singularity_exec, ",".join( - [simulation_infos["file_path"], simulation_output_folder] + [str(simulation_infos["file_path"]), str(simulation_output_folder)] ), self._singularity, - path.join( - simulation_output_folder, - "{}_simulation.ffp".format(self._base_naming), - ), + simulation_output_folder / "{}_simulation.ffp".format(self._base_naming), image_file, - path.join( - simulation_output_folder, - "{}.{}".format(self._base_naming, self._extension), - ), + simulation_output_folder / "{}.{}".format(self._base_naming, self._extension), "-v" if test_mode else "", ) ) copyfile( - path.join( - simulation_infos["file_path"], simulation_infos["param_file"] - ), - path.join( - simulation_output_folder, - "{}_simulation.ffp".format(self._base_naming), - ), + simulation_infos["file_path"] / simulation_infos["param_file"], + simulation_output_folder / "{}_simulation.ffp".format(self._base_naming), ) set_event_loop(self._event_loop) async_loop = get_event_loop() - log_file = path.join(output_folder, "{}.log".format(self._base_naming)) + log_file: pathlib.Path = output_folder / "{}.log".format(self._base_naming) - logger.info("Simulating DWI signal") - return_code, log = async_loop.run_until_complete( + _logger.info("Simulating DWI signal") + return_code, _ = async_loop.run_until_complete( self._launch_command( simulation_command, log_file, "[RUNNING FIBERFOX]" ) @@ -121,10 +109,10 @@ def run_simulation_dwimage( "Simulation ended in error", SimulationRunnerException.ExceptionType.Fiberfox, return_code, - (log,), + str(log_file), ) - logger.debug( + _logger.debug( "Simulation {} ended with code {}".format( self._base_naming, return_code ) @@ -132,63 +120,53 @@ def run_simulation_dwimage( async_loop.close() def run_simulation_standalone( - self, - output_folder, - geometry_folder, - simulation_infos, - base_naming=None, - test_mode=False, + self, + output_folder: pathlib.Path, + geometry_folder: pathlib.Path, + simulation_infos: SimulationInfos, + base_naming=None, + test_mode=False, ): if not base_naming: base_naming = self._base_naming self._start_loop_if_closed() - simulation_output_folder = path.join( - output_folder, "simulation_outputs" - ) - geometry_output_folder = path.join(geometry_folder, "geometry_outputs") - if not path.exists(simulation_output_folder): - makedirs(simulation_output_folder, exist_ok=True) + simulation_output_folder = output_folder / "simulation_outputs" + simulation_output_folder.mkdir(parents=True, exist_ok=True) + simulation_output_folder = simulation_output_folder.resolve(strict=True) + + output_folder = output_folder.resolve(strict=True) + + geometry_output_folder = geometry_folder / "geometry_outputs" + geometry_output_folder = geometry_output_folder.resolve(strict=True) simulation_command = ( "{} run -B {} --app launch_mitk {} -p {} -i {} -o {} {}".format( self._singularity_exec, ",".join( [ - geometry_folder, - simulation_infos["file_path"], - simulation_output_folder, + str(geometry_folder), + str(simulation_infos["file_path"]), + str(simulation_output_folder), ] ), self._singularity, - path.join( - simulation_output_folder, - "{}_simulation.ffp".format(base_naming), - ), - path.join(geometry_output_folder, self._geometry_base_naming) - + "_merged_bundles.fib", - path.join( - simulation_output_folder, - "{}.{}".format(base_naming, self._extension), - ), + simulation_output_folder / "{}_simulation.ffp".format(base_naming), + geometry_output_folder / (self._geometry_base_naming + "_merged_bundles.fib"), + simulation_output_folder / "{}.{}".format(base_naming, self._extension), "-v" if test_mode else "", ) ) copyfile( - path.join( - simulation_infos["file_path"], simulation_infos["param_file"] - ), - path.join( - simulation_output_folder, - "{}_simulation.ffp".format(base_naming), - ), + simulation_infos["file_path"] / simulation_infos["param_file"], + simulation_output_folder / "{}_simulation.ffp".format(base_naming), ) set_event_loop(self._event_loop) async_loop = get_event_loop() - log_file = path.join(output_folder, "{}.log".format(base_naming)) + log_file: pathlib.Path = output_folder / "{}.log".format(base_naming) self._rename_and_copy_compartments_standalone( simulation_infos, @@ -197,8 +175,8 @@ def run_simulation_standalone( base_naming, ) - logger.info("Simulating DWI signal") - return_code, log = async_loop.run_until_complete( + _logger.info("Simulating DWI signal") + return_code, _ = async_loop.run_until_complete( self._launch_command( simulation_command, log_file, "[RUNNING FIBERFOX]" ) @@ -208,10 +186,10 @@ def run_simulation_standalone( "Simulation ended in error", SimulationRunnerException.ExceptionType.Fiberfox, return_code, - (log,), + str(log_file), ) - logger.debug( + _logger.debug( "Simulation {} ended with code {}".format( self._base_naming, return_code ) @@ -219,31 +197,29 @@ def run_simulation_standalone( async_loop.close() def run( - self, output_folder, test_mode=False, relative_fiber_compartment=True + self, output_folder: pathlib.Path, test_mode=False, relative_fiber_compartment=True ): - geometry_output_folder = path.join(output_folder, "geometry_outputs") + geometry_output_folder: pathlib.Path = output_folder / "geometry_outputs" + geometry_output_folder.mkdir(parents=True, exist_ok=True) + geometry_output_folder = geometry_output_folder.resolve(strict=True) - if not path.exists(geometry_output_folder): - makedirs(geometry_output_folder, exist_ok=True) + output_folder = output_folder.resolve(strict=True) if self._run_simulation: - simulation_output_folder = path.join( - output_folder, "simulation_outputs" - ) - - if not path.exists(simulation_output_folder): - makedirs(simulation_output_folder, exist_ok=True) + simulation_output_folder: pathlib.Path = output_folder / "simulation_outputs" + simulation_output_folder.mkdir(parents=True, exist_ok=True) + simulation_output_folder = simulation_output_folder.resolve(strict=True) geometry_command = ( "singularity run -B {} --app launch_voxsim {} -f {} -r {} " "-s {} -o {} --comp-map {} --quiet{}".format( - ",".join([self._geometry_path, geometry_output_folder]), + ",".join([str(self._geometry_path), str(geometry_output_folder)]), self._singularity, - path.join(self._geometry_path, self._geometry_base_file), + self._geometry_path / self._geometry_base_file, ",".join([str(r) for r in self._geometry_resolution]), ",".join([str(s) for s in self._geometry_spacing]), - path.join(geometry_output_folder, self._geometry_base_naming), + geometry_output_folder / self._geometry_base_naming, "rel" if relative_fiber_compartment else "abs", self._fib_extension_arg, ) @@ -255,41 +231,29 @@ def run( "-p {} -i {} -o {} {}".format( ",".join( [ - self._simulation_path, - geometry_output_folder, - simulation_output_folder, + str(self._simulation_path), + str(geometry_output_folder), + str(simulation_output_folder), ] ), self._singularity, - path.join( - simulation_output_folder, - "{}_simulation.ffp".format(self._base_naming), - ), - path.join( - geometry_output_folder, self._geometry_base_naming - ) - + "_merged_bundles.fib", - path.join( - simulation_output_folder, - "{}.{}".format(self._base_naming, self._extension), - ), + simulation_output_folder / "{}_simulation.ffp".format(self._base_naming), + geometry_output_folder / (self._geometry_base_naming + "_merged_bundles.fib"), + simulation_output_folder / "{}.{}".format(self._base_naming, self._extension), "-v" if test_mode else "", ) ) copyfile( - path.join(self._simulation_path, self._simulation_parameters), - path.join( - simulation_output_folder, - "{}_simulation.ffp".format(self._base_naming), - ), + self._simulation_path / self._simulation_parameters, + simulation_output_folder / "{}_simulation.ffp".format(self._base_naming), ) set_event_loop(self._event_loop) async_loop = get_event_loop() - log_file = path.join(output_folder, "{}.log".format(self._base_naming)) + log_file = output_folder / "{}.log".format(self._base_naming) - logger.info("Generating simulation geometry") + _logger.info("Generating simulation geometry") async_loop.run_until_complete( self._launch_command(geometry_command, log_file, "[RUNNING VOXSIM]") ) @@ -297,9 +261,9 @@ def run( self._rename_and_copy_compartments( geometry_output_folder, simulation_output_folder ) - logger.info("Simulating DWI signal") + _logger.info("Simulating DWI signal") if self._run_simulation: - return_code, log = async_loop.run_until_complete( + return_code, _ = async_loop.run_until_complete( self._launch_command( simulation_command, log_file, "[RUNNING FIBERFOX]" ) @@ -309,10 +273,10 @@ def run( "Simulation ended in error", SimulationRunnerException.ExceptionType.Fiberfox, return_code, - (log,), + str(log_file), ) - logger.debug("Simulation ended with code {}".format(return_code)) + _logger.debug("Simulation ended with code {}".format(return_code)) async_loop.close() def _execute_parallel(self, method, args): @@ -321,73 +285,55 @@ def _execute_parallel(self, method, args): p.join() def _rename_and_copy_compartments_standalone( - self, - simulation_infos, - geometry_output_folder, - simulation_output_folder, - base_naming, + self, + simulation_infos, + geometry_output_folder: pathlib.Path, + simulation_output_folder: pathlib.Path, + base_naming, ): + geometry_output_folder = geometry_output_folder.resolve(strict=True) + copyfile( - path.join( - geometry_output_folder, - self._geometry_base_naming - + "_mergedBundlesMaps.{}".format(self._extension), - ), - path.join( - simulation_output_folder, - "{}_simulation.ffp_VOLUME{}.{}".format( - base_naming, - simulation_infos["compartment_ids"][0], - self._extension, - ), + geometry_output_folder / (self._geometry_base_naming + "_mergedBundlesMaps.{}".format(self._extension)), + simulation_output_folder / + "{}_simulation.ffp_VOLUME{}.{}".format( + base_naming, + simulation_infos["compartment_ids"][0], + self._extension, ), ) if len(simulation_infos["compartment_ids"]) > 1: - merged_maps = exists( - path.join( - geometry_output_folder, - self._geometry_base_naming - + "_mergedEllipsesMaps.{}".format(self._extension), - ) - ) - base_map = (not merged_maps) and exists( - path.join( - geometry_output_folder, - self._geometry_base_naming - + "_ellipsoid{}_cmap.{}".format(0, self._extension), - ) - ) + merged_maps = ( + geometry_output_folder / ( + self._geometry_base_naming + "_mergedEllipsesMaps.{}".format(self._extension)) + ).exists() + base_map = (not merged_maps) and ( + geometry_output_folder / ( + self._geometry_base_naming + "_ellipsoid{}_cmap.{}".format(0, self._extension)) + ).exists() if merged_maps: copyfile( - path.join( - geometry_output_folder, - self._geometry_base_naming - + "_mergedEllipsesMaps.{}".format(self._extension), + geometry_output_folder / ( + self._geometry_base_naming + "_mergedEllipsesMaps.{}".format(self._extension) ), - path.join( - simulation_output_folder, - "{}_simulation.ffp_VOLUME{}.{}".format( - base_naming, - simulation_infos["compartment_ids"][1], - self._extension, - ), + simulation_output_folder / + "{}_simulation.ffp_VOLUME{}.{}".format( + base_naming, + simulation_infos["compartment_ids"][1], + self._extension, ), ) elif base_map: copyfile( - path.join( - geometry_output_folder, - self._geometry_base_naming - + "_ellipsoid1_cmap.{}".format(self._extension), + geometry_output_folder / ( + self._geometry_base_naming + "_ellipsoid1_cmap.{}".format(self._extension) ), - path.join( - simulation_output_folder, - "{}_simulation.ffp_VOLUME{}.{}".format( - base_naming, - simulation_infos["compartment_ids"][1], - self._extension, - ), + simulation_output_folder / + "{}_simulation.ffp_VOLUME{}.{}".format( + base_naming, + simulation_infos["compartment_ids"][1], + self._extension, ), ) else: @@ -401,7 +347,7 @@ def _rename_and_copy_compartments_standalone( ) if len(simulation_infos["compartment_ids"]) > 2 and ( - merged_maps or base_map + merged_maps or base_map ): self._generate_background_map( geometry_output_folder, @@ -420,67 +366,47 @@ def _rename_and_copy_compartments_standalone( ) def _rename_and_copy_compartments( - self, geometry_output_folder, simulation_output_folder + self, geometry_output_folder, simulation_output_folder ): + geometry_output_folder = geometry_output_folder.resolve(strict=True) + copyfile( - path.join( - geometry_output_folder, - self._geometry_base_naming - + "_mergedBundlesMaps.{}".format(self._extension), - ), - path.join( - simulation_output_folder, - "{}_simulation.ffp_VOLUME{}.{}".format( - self._base_naming, self._compartment_ids[0], self._extension - ), - ), + geometry_output_folder / (self._geometry_base_naming + "_mergedBundlesMaps.{}".format(self._extension)), + simulation_output_folder / + "{}_simulation.ffp_VOLUME{}.{}".format(self._base_naming, self._compartment_ids[0], self._extension), ) if self._number_of_maps > 1: - merged_maps = exists( - path.join( - geometry_output_folder, - self._geometry_base_naming - + "_mergedEllipsesMaps.{}".format(self._extension), - ) - ) - base_map = (not merged_maps) and exists( - path.join( - geometry_output_folder, - self._geometry_base_naming - + "_ellipsoid1_cmap.{}".format(self._extension), - ) - ) + merged_maps = ( + geometry_output_folder / ( + self._geometry_base_naming + "_mergedEllipsesMaps.{}".format(self._extension)) + ).exists() + base_map = (not merged_maps) and ( + geometry_output_folder / ( + self._geometry_base_naming + "_ellipsoid1_cmap.{}".format(self._extension)) + ).exists() if merged_maps: copyfile( - path.join( - geometry_output_folder, - self._geometry_base_naming - + "_mergedEllipsesMaps.{}".format(self._extension), + geometry_output_folder / ( + self._geometry_base_naming + "_mergedEllipsesMaps.{}".format(self._extension) ), - path.join( - simulation_output_folder, - "{}_simulation.ffp_VOLUME{}.{}".format( - self._base_naming, - self._compartment_ids[1], - self._extension, - ), + simulation_output_folder / + "{}_simulation.ffp_VOLUME{}.{}".format( + self._base_naming, + self._compartment_ids[1], + self._extension, ), ) elif base_map: copyfile( - path.join( - geometry_output_folder, - self._geometry_base_naming - + "_ellipsoid0_cmap.{}".format(self._extension), + geometry_output_folder / ( + self._geometry_base_naming + "_ellipsoid0_cmap.{}".format(self._extension) ), - path.join( - simulation_output_folder, - "{}_simulation.ffp_VOLUME{}.{}".format( - self._base_naming, - self._compartment_ids[1], - self._extension, - ), + simulation_output_folder / + "{}_simulation.ffp_VOLUME{}.{}".format( + self._base_naming, + self._compartment_ids[1], + self._extension, ), ) else: @@ -522,45 +448,35 @@ def _save_nrrd(self, data, header, name): nrrd.write("{}.nrrd".format(name), data, header) def _generate_background_map( - self, - geometry_output_folder, - simulation_output_folder, - compartment_id, - merged_maps=False, - base_map=False, - base_naming=None, + self, + geometry_output_folder: pathlib.Path, + simulation_output_folder: pathlib.Path, + compartment_id, + merged_maps=False, + base_map=False, + base_naming=None, ): + geometry_output_folder = geometry_output_folder.resolve(strict=True) + simulation_output_folder = simulation_output_folder.resolve(strict=True) + if not base_naming: base_naming = self._base_naming img, header = self._load_image( - path.join( - geometry_output_folder, - "{}_mergedBundlesMaps".format(self._geometry_base_naming), - ) + geometry_output_folder / "{}_mergedBundlesMaps".format(self._geometry_base_naming) ) maps = [img] if merged_maps: maps.append( self._load_image( - path.join( - geometry_output_folder, - "{}{}".format( - self._geometry_base_naming, "_mergedEllipsesMaps" - ), - ) + geometry_output_folder / "{}{}".format(self._geometry_base_naming, "_mergedEllipsesMaps") )[0] ) elif base_map: maps.append( self._load_image( - path.join( - geometry_output_folder, - "{}{}".format( - self._geometry_base_naming, "_ellipsoid1_cmap" - ), - ) + geometry_output_folder / "{}{}".format(self._geometry_base_naming, "_ellipsoid1_cmap") )[0] ) @@ -570,23 +486,18 @@ def _generate_background_map( self._save_image( extra_map, header, - path.join( - simulation_output_folder, - "{}_simulation.ffp_VOLUME{}".format( - base_naming, compartment_id - ), - ), + simulation_output_folder / "{}_simulation.ffp_VOLUME{}".format(base_naming, compartment_id), ) def _start_loop_if_closed(self): if self._event_loop.is_closed(): self._event_loop = new_event_loop() - async def _launch_command(self, command, log_file, log_tag): + async def _launch_command(self, command, log_file: pathlib.Path, log_tag): process = Popen(command.split(" "), stdout=PIPE, stderr=PIPE) - logger = RTLogging(process, log_file, log_tag) - logger.start() - logger.join() + _logger = RTLogging(process, log_file, log_tag) + _logger.start() + _logger.join() return process.returncode, log_file diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index 1857cc1..d8eeb17 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -1,19 +1,19 @@ import logging +import pathlib +import typing from asyncio import get_event_loop, new_event_loop, set_event_loop -from os import makedirs, path -from os.path import basename from subprocess import PIPE, Popen -from config import get_config +from .config import SingularityConfig from .datastore import Datastore from ..utils.logging import RTLogging - - -logger = logging.getLogger(basename(__file__).split(".")[0]) +from ..factory.geometry_factory.handlers import GeometryInfos +from ..factory.simulation_factory.handlers import SimulationInfos class AsyncRunner: + def __init__(self): self._event_loop = new_event_loop() @@ -24,43 +24,42 @@ def stop(self): if not self._event_loop.is_closed(): self._event_loop.close() - def _run_command(self, command, log_file, log_tag): + def _run_command(self, command: str, log_file: pathlib.Path, log_tag: str) -> int: set_event_loop(self._event_loop) async_loop = get_event_loop() - async_loop.run_until_complete( + + returncode = async_loop.run_until_complete( self._run_async(command, log_file, log_tag) ) + async_loop.close() + return returncode def _start_loop_if_closed(self): if self._event_loop.is_closed(): self._event_loop = new_event_loop() - async def _run_async(self, command, log_file, log_tag): + @staticmethod + async def _run_async(command: str, log_file, log_tag) -> int: process = Popen(command.split(" "), stdout=PIPE, stderr=PIPE) - logger = RTLogging(process, log_file, log_tag) - logger.start() - logger.join() + _logger = RTLogging(process, log_file, log_tag) + _logger.start() + _logger.join() - return process.returncode, log_file + return process.returncode class SimulationRunner(AsyncRunner): _apps = {"phantom": "launch_voxsim", "diffusion mri": "launch_mitk"} - def __init__(self, singularity_conf=get_config()): - self._singularity = path.join( - singularity_conf["singularity_path"], - singularity_conf["singularity_name"], - ) + def __init__(self, singularity_conf=SingularityConfig()): + self._singularity = singularity_conf.singularity.resolve(strict=True) super().__init__() - self._singularity_exec = "singularity" - if "singularity_exec" in singularity_conf: - self._singularity_exec = singularity_conf["singularity_exec"] + self._singularity_exec = singularity_conf.singularity_exec - def _bind_singularity(self, step, paths, arguments): + def _bind_singularity(self, step, paths, arguments) -> str: return "{} run -B {} --app {} {} {}".format( self._singularity_exec, paths, @@ -69,25 +68,26 @@ def _bind_singularity(self, step, paths, arguments): arguments, ) - def _create_outputs(self, path): - if not path.exists(path): - makedirs(path, exist_ok=True) - - return path + @staticmethod + def _create_outputs(path: pathlib.Path): + path.mkdir(parents=True, exist_ok=True) + return path.resolve(strict=True) def run( self, - run_name, - phantom_infos, - simulation_infos, - output_folder, + run_name: str, + phantom_infos: GeometryInfos, + simulation_infos: SimulationInfos, + output_folder: pathlib.Path, output_nifti=True, relative_fiber_fraction=True, inter_axonal_fraction=None, - ): + ) -> typing.Tuple[int, int]: self.start() - self.generate_phantom( + output_folder = output_folder.resolve(strict=True) + + phantom_returncode = self.generate_phantom( run_name, phantom_infos, output_folder, @@ -97,22 +97,16 @@ def run( ) datastore = Datastore( - path.join(output_folder, "simulation"), - path.join( - output_folder, - "phantom", - "{}_phantom_merged_bundles.fib".format(run_name), - ), + output_folder / "simulation", + output_folder / "phantom" / "{}_phantom_merged_bundles.fib".format(run_name), simulation_infos["compartment_ids"], inter_axonal_fraction, ) - datastore.load_compartments( - path.join(output_folder, "phantom"), run_name, output_nifti - ) + datastore.load_compartments(output_folder / "phantom", run_name, output_nifti) datastore.stage_compartments(run_name) - self.simulate_diffusion_mri( + mri_returncode = self.simulate_diffusion_mri( run_name, simulation_infos, output_folder, @@ -125,34 +119,29 @@ def run( ) self.stop() + return phantom_returncode, mri_returncode def generate_phantom( self, - run_name, - phantom_infos, - output_folder, + run_name: str, + phantom_infos: GeometryInfos, + output_folder: pathlib.Path, relative_fiber_fraction=True, output_nifti=True, loop_managed=False, - ): + ) -> int: loop_managed or self.start() - base_output_folder = output_folder - output_folder = self._create_outputs( - path.join(output_folder, "phantom") - ) + base_output_folder: pathlib.Path = self._create_outputs(output_folder) + output_folder: pathlib.Path = self._create_outputs(output_folder / "phantom") - phantom_def = path.join( - phantom_infos["file_path"], phantom_infos["base_file"] - ) + phantom_def: pathlib.Path = phantom_infos["file_path"] / phantom_infos["base_file"] resolution = ",".join([str(r) for r in phantom_infos["resolution"]]) spacing = ",".join([str(s) for s in phantom_infos["spacing"]]) fiber_fraction = "rel" if relative_fiber_fraction else "abs" - out_name = path.join( - output_folder, "phantom, " "{}_phantom".format(run_name) - ) + out_name: pathlib.Path = output_folder / "{}_phantom".format(run_name) arguments = "-f {} -r {} -s {} -o {} --comp-map {} --quiet".format( phantom_def, resolution, spacing, out_name, fiber_fraction @@ -161,42 +150,41 @@ def generate_phantom( if output_nifti: arguments += " --nii" - bind_paths = ",".join([phantom_infos["file_path"], output_folder]) + bind_paths = ",".join([str(phantom_infos["file_path"]), str(output_folder)]) command = self._bind_singularity("phantom", bind_paths, arguments) - log_file = path.join(base_output_folder, "{}.log".format(run_name)) - self._run_command(command, log_file, "[PHANTOM]") + log_file: pathlib.Path = base_output_folder / "{}.log".format(run_name) + returncode = self._run_command(command, log_file, "[PHANTOM]") loop_managed or self.stop() + return returncode def simulate_diffusion_mri( self, - run_name, - simulation_infos, - output_folder, - fibers_file, + run_name: str, + simulation_infos: SimulationInfos, + output_folder: pathlib.Path, + fibers_file: pathlib.Path, compartment_maps=None, bind_paths=None, output_nifti=True, loop_managed=False, compartments_staged=True, - ): + ) -> int: loop_managed or self.start() bind_paths = [] if bind_paths is None else bind_paths - base_output_folder = output_folder - output_folder = self._create_outputs( - path.join(output_folder, "simulation") - ) + fibers_file = fibers_file.resolve(strict=True) + base_output_folder: pathlib.Path = self._create_outputs(output_folder) + output_folder: pathlib.Path = self._create_outputs(output_folder / "simulation") name = "{}_simulation".format(run_name) - bind_paths += [simulation_infos["file_path"], output_folder] + bind_paths += [str(simulation_infos["file_path"]), str(output_folder)] bind_paths = ",".join(bind_paths) - ffp_file = path.join( - simulation_infos["file_path"], simulation_infos["param_file"] - ) + ffp_file = simulation_infos["file_path"] / simulation_infos["param_file"] + extension = "nii.gz" if output_nifti else "nrrd" - out_name = path.join(output_folder, "{}.{}".format(name, extension)) + out_name = output_folder / "{}.{}".format(name, extension) if not compartments_staged and compartment_maps is not None: datastore = Datastore( @@ -211,7 +199,8 @@ def simulate_diffusion_mri( arguments = "-p {} -i {} -o {}".format(ffp_file, fibers_file, out_name) command = self._bind_singularity("diffusion mri", bind_paths, arguments) - log_file = path.join(base_output_folder, "{}.log".format(run_name)) - self._run_command(command, log_file, "[DIFFUSION MRI]") + log_file: pathlib.Path = base_output_folder / "{}.log".format(run_name) + returncode = self._run_command(command, log_file, "[DIFFUSION MRI]") loop_managed or self.stop() + return returncode diff --git a/simulator/utils/logging.py b/simulator/utils/logging.py index 89f60d3..43c8d0e 100755 --- a/simulator/utils/logging.py +++ b/simulator/utils/logging.py @@ -1,12 +1,13 @@ -from queue import Queue +import queue from threading import Thread +import pathlib import time class RTLogging: - def __init__(self, process, log_file_path, log_tag=""): + def __init__(self, process, log_file_path: pathlib.Path, log_tag=""): self._process = process - self._log = log_file_path + self._log: pathlib.Path = log_file_path self._tag = log_tag self._thread = None @@ -19,8 +20,8 @@ def join(self): self._thread.join() def _read_output(self, poll_timer=4, logging_callback=lambda a: None): - stdout_queue = Queue() - stderr_queue = Queue() + stdout_queue = queue.Queue() + stderr_queue = queue.Queue() t1 = Thread( target=self._enqueue_thread_output, args=(self._process.stdout, stdout_queue), @@ -48,15 +49,15 @@ def _read_output(self, poll_timer=4, logging_callback=lambda a: None): logging_callback(self._log) time.sleep(poll_timer) - def _enqueue_thread_output(self, pipe, queue): + def _enqueue_thread_output(self, pipe, output: queue.Queue): while self._process.poll() is None: ln = pipe.readline() - queue.put(ln) + output.put(ln) - def _dequeue_output(self, log_file, queue, tag): + def _dequeue_output(self, log_file, output: queue.Queue, tag): try: - while not queue.empty(): - ln = queue.get_nowait() + while not output.empty(): + ln = output.get_nowait() if ln: log_file.write( "\n".join( @@ -68,5 +69,5 @@ def _dequeue_output(self, log_file, queue, tag): + "\n" ) log_file.flush() - except: + except queue.Empty: pass diff --git a/simulator/utils/setup/documentation.py b/simulator/utils/setup/documentation.py index 58432b9..ee32973 100755 --- a/simulator/utils/setup/documentation.py +++ b/simulator/utils/setup/documentation.py @@ -1,5 +1,5 @@ import os -from os.path import join +import pathlib from setuptools import Command @@ -9,9 +9,7 @@ class DocCommand(Command): user_options = [ ("out-type=", "o", "Output documentation type, default : html") ] - script_dir = join( - "documentation", "_cache", "bat" if os.name == "nt" else "bash" - ) + script_dir: pathlib.Path = pathlib.Path("documentation") / "_cache" / ("bat" if os.name == "nt" else "bash") script_ext = "bat" if os.name == "nt" else "sh" def initialize_options(self): @@ -22,13 +20,13 @@ def finalize_options(self): def run(self): self._run_command( - [join(self.script_dir, "generate_doc.{}".format(self.script_ext))], + [self.script_dir / "generate_doc.{}".format(self.script_ext)], "Generating automatic documentation", ) self._run_command( [ - join(self.script_dir, "build_doc.{}".format(self.script_ext)), + self.script_dir / "build_doc.{}".format(self.script_ext), self.out_type, ], "Building documentation",