From ba3d2e6dff6795ee1733bbc029c007ce5bead07c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Wed, 29 Jun 2022 13:04:48 -0400 Subject: [PATCH 001/100] Remove data_files entry --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 19e4249..c41deef 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,6 @@ author="avcaron", author_email="", description="", - data_files=[(".", ["config.py", "config.json"])], scripts=list( filter(lambda s: "init" not in s, glob.glob("scripts/*.py")) ), From 3bc144e93ad2c9115f9b978d863645e6852c3cab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Wed, 29 Jun 2022 13:22:04 -0400 Subject: [PATCH 002/100] Added default util --- simulator/default.py | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 simulator/default.py diff --git a/simulator/default.py b/simulator/default.py new file mode 100644 index 0000000..5d7b73e --- /dev/null +++ b/simulator/default.py @@ -0,0 +1,4 @@ +import typing + +SINGULARITY_PATH: typing.Final[str] = "utils" +SINGULARITY_NAME: typing.Final[str] = "voxsim_singularity_latest.sif" From e260acc3e306b7f4949c799bc638f896ecf31d65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Wed, 29 Jun 2022 13:22:12 -0400 Subject: [PATCH 003/100] Added SingularityConfig class --- simulator/runner/config.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 simulator/runner/config.py diff --git a/simulator/runner/config.py b/simulator/runner/config.py new file mode 100644 index 0000000..2d4764a --- /dev/null +++ b/simulator/runner/config.py @@ -0,0 +1,8 @@ +import dataclasses +import simulator.default as default + + +@dataclasses.dataclass +class SingularityConfig: + singularity_path: str = default.SINGULARITY_PATH + singularity_name: str = default.SINGULARITY_NAME From d7d69669772b2634e3af509c6f0c45a1ad47ef34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Wed, 29 Jun 2022 13:25:07 -0400 Subject: [PATCH 004/100] Format --- simulator/runner/legacy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simulator/runner/legacy.py b/simulator/runner/legacy.py index 463b321..fa721e1 100644 --- a/simulator/runner/legacy.py +++ b/simulator/runner/legacy.py @@ -14,11 +14,11 @@ from ..exceptions import SimulationRunnerException from ..utils.logging import RTLogging - logger = logging.getLogger(basename(__file__).split(".")[0]) class SimulationRunner: + def __init__( self, base_naming, From d6cf37fc752e22d1a0bd69e913744a38f2d5f33b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Wed, 29 Jun 2022 13:25:44 -0400 Subject: [PATCH 005/100] Integrated SingularityConfig into legacy runner --- simulator/runner/legacy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/simulator/runner/legacy.py b/simulator/runner/legacy.py index fa721e1..10e79c3 100644 --- a/simulator/runner/legacy.py +++ b/simulator/runner/legacy.py @@ -10,7 +10,7 @@ from numpy import ones_like, sum import nrrd -from config import get_config +from .config import SingularityConfig from ..exceptions import SimulationRunnerException from ..utils.logging import RTLogging @@ -24,7 +24,7 @@ def __init__( base_naming, geometry_infos, simulation_infos=None, - singularity_conf=get_config(), + singularity_conf=SingularityConfig(), output_nifti=False, ): self._geometry_path = geometry_infos["file_path"] @@ -40,7 +40,7 @@ def __init__( self._compartment_ids = simulation_infos["compartment_ids"] singularity_conf = ( - singularity_conf if singularity_conf else get_config() + singularity_conf if singularity_conf else SingularityConfig() ) self._singularity = path.join( singularity_conf["singularity_path"], From 49d6d9f8768d57a362eec983c9af75aa55605046 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Wed, 29 Jun 2022 13:26:06 -0400 Subject: [PATCH 006/100] Format --- simulator/runner/simulation_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index 1857cc1..523e6e5 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -9,11 +9,11 @@ from .datastore import Datastore from ..utils.logging import RTLogging - logger = logging.getLogger(basename(__file__).split(".")[0]) class AsyncRunner: + def __init__(self): self._event_loop = new_event_loop() From 14481bda87adf4b5e23b7e396f12dccf015d0b1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Wed, 29 Jun 2022 13:42:00 -0400 Subject: [PATCH 007/100] Format --- simulator/runner/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/simulator/runner/config.py b/simulator/runner/config.py index 2d4764a..4d17ac3 100644 --- a/simulator/runner/config.py +++ b/simulator/runner/config.py @@ -1,4 +1,5 @@ import dataclasses + import simulator.default as default From 0bde314321638bb3a15af528464a85bfa84269e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Wed, 29 Jun 2022 15:38:01 -0400 Subject: [PATCH 008/100] Added singularity exec attribute to config --- simulator/default.py | 1 + simulator/runner/config.py | 1 + 2 files changed, 2 insertions(+) diff --git a/simulator/default.py b/simulator/default.py index 5d7b73e..45d3c33 100644 --- a/simulator/default.py +++ b/simulator/default.py @@ -2,3 +2,4 @@ SINGULARITY_PATH: typing.Final[str] = "utils" SINGULARITY_NAME: typing.Final[str] = "voxsim_singularity_latest.sif" +SINGULARITY_EXEC: typing.Final[str] = "singularity" diff --git a/simulator/runner/config.py b/simulator/runner/config.py index 4d17ac3..1368938 100644 --- a/simulator/runner/config.py +++ b/simulator/runner/config.py @@ -7,3 +7,4 @@ class SingularityConfig: singularity_path: str = default.SINGULARITY_PATH singularity_name: str = default.SINGULARITY_NAME + singularity_exec: str = default.SINGULARITY_EXEC From 8714a5eec87d1429d9967691078ed5e04304c7b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Wed, 29 Jun 2022 15:43:28 -0400 Subject: [PATCH 009/100] Typo --- simulator/default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simulator/default.py b/simulator/default.py index 45d3c33..da6816d 100644 --- a/simulator/default.py +++ b/simulator/default.py @@ -1,5 +1,5 @@ import typing -SINGULARITY_PATH: typing.Final[str] = "utils" +SINGULARITY_PATH: typing.Final[str] = "." SINGULARITY_NAME: typing.Final[str] = "voxsim_singularity_latest.sif" SINGULARITY_EXEC: typing.Final[str] = "singularity" From 45247c0e1f4efaf7135ab5f160a99102a34d2832 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Wed, 29 Jun 2022 15:56:41 -0400 Subject: [PATCH 010/100] Refactored config to include singularity attr --- simulator/default.py | 4 +++- simulator/runner/config.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/simulator/default.py b/simulator/default.py index da6816d..3a170b7 100644 --- a/simulator/default.py +++ b/simulator/default.py @@ -1,5 +1,7 @@ import typing +import pathlib -SINGULARITY_PATH: typing.Final[str] = "." +SINGULARITY_PATH: typing.Final[pathlib.PurePath] = pathlib.PurePath() SINGULARITY_NAME: typing.Final[str] = "voxsim_singularity_latest.sif" +SINGULARITY: typing.Final[pathlib.PurePath] = SINGULARITY_PATH / SINGULARITY_NAME SINGULARITY_EXEC: typing.Final[str] = "singularity" diff --git a/simulator/runner/config.py b/simulator/runner/config.py index 1368938..1667208 100644 --- a/simulator/runner/config.py +++ b/simulator/runner/config.py @@ -1,10 +1,10 @@ import dataclasses +import pathlib import simulator.default as default @dataclasses.dataclass class SingularityConfig: - singularity_path: str = default.SINGULARITY_PATH - singularity_name: str = default.SINGULARITY_NAME + singularity: pathlib.PurePath = default.SINGULARITY singularity_exec: str = default.SINGULARITY_EXEC From 825e4dc5f4b4ad0d22ee55d9ba4641d7e93b2dce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Wed, 29 Jun 2022 15:59:11 -0400 Subject: [PATCH 011/100] Final integration of config in legacy runner --- simulator/runner/legacy.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/simulator/runner/legacy.py b/simulator/runner/legacy.py index 10e79c3..b73efbb 100644 --- a/simulator/runner/legacy.py +++ b/simulator/runner/legacy.py @@ -42,15 +42,8 @@ def __init__( singularity_conf = ( singularity_conf if singularity_conf else SingularityConfig() ) - self._singularity = path.join( - singularity_conf["singularity_path"], - singularity_conf["singularity_name"], - ) - self._singularity_exec = ( - singularity_conf["singularity_exec"] - if "singularity_exec" in singularity_conf - else "singularity" - ) + self._singularity = singularity_conf.singularity + self._singularity_exec = singularity_conf.singularity_exec self._run_simulation = True if simulation_infos else False self._extension = "nii.gz" if output_nifti else "nrrd" From 2240b4964cb13b28165ef474bcab27303f522568 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Wed, 29 Jun 2022 16:00:28 -0400 Subject: [PATCH 012/100] Final integration of config in new runner --- simulator/runner/simulation_runner.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index 523e6e5..ae13528 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -5,7 +5,7 @@ from os.path import basename from subprocess import PIPE, Popen -from config import get_config +from .config import SingularityConfig from .datastore import Datastore from ..utils.logging import RTLogging @@ -49,16 +49,11 @@ async def _run_async(self, command, log_file, log_tag): class SimulationRunner(AsyncRunner): _apps = {"phantom": "launch_voxsim", "diffusion mri": "launch_mitk"} - def __init__(self, singularity_conf=get_config()): - self._singularity = path.join( - singularity_conf["singularity_path"], - singularity_conf["singularity_name"], - ) + def __init__(self, singularity_conf=SingularityConfig()): + self._singularity = singularity_conf.singularity super().__init__() - self._singularity_exec = "singularity" - if "singularity_exec" in singularity_conf: - self._singularity_exec = singularity_conf["singularity_exec"] + self._singularity_exec = singularity_conf.singularity_exec def _bind_singularity(self, step, paths, arguments): return "{} run -B {} --app {} {} {}".format( From 53b84fa066c8f7b141baa5856e2d5795b9641f5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Wed, 29 Jun 2022 16:02:18 -0400 Subject: [PATCH 013/100] Removed deprecated config files --- config.json | 5 ----- config.py | 13 ------------- 2 files changed, 18 deletions(-) delete mode 100644 config.json delete mode 100644 config.py diff --git a/config.json b/config.json deleted file mode 100644 index 3d18623..0000000 --- a/config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "singularity_path": ".", - "singularity_name": "voxsim_singularity_latest.sif" -} - diff --git a/config.py b/config.py deleted file mode 100644 index 232a35c..0000000 --- a/config.py +++ /dev/null @@ -1,13 +0,0 @@ -import json -from os.path import dirname, join, realpath - - -def get_config(path=dirname(realpath(__file__))): - with open(join(path, "config.json")) as f: - return json.load(f) - - -def override_global_config(config): - print(config) - with open(join(dirname(realpath(__file__)), "config.json"), "w+") as f: - json.dump(config, f) From 19ef8c8a7fd9cf2df73653a35e030f528f16a6df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Wed, 29 Jun 2022 16:02:38 -0400 Subject: [PATCH 014/100] Import SingularityConfig when importing runner pkg --- simulator/runner/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/simulator/runner/__init__.py b/simulator/runner/__init__.py index 119609b..49dc376 100755 --- a/simulator/runner/__init__.py +++ b/simulator/runner/__init__.py @@ -1 +1,2 @@ +from .config import SingularityConfig from .simulation_runner import SimulationRunner From d0bb89a714fdb78fabdd45e9a86afd4069c39f23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Wed, 29 Jun 2022 16:07:00 -0400 Subject: [PATCH 015/100] Bumped version --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index c41deef..c2b3514 100644 --- a/setup.py +++ b/setup.py @@ -3,11 +3,10 @@ from simulator.utils.setup.documentation import DocCommand - if __name__ == "__main__": setup( name="simulation_generator", - version="1.0.0", + version="1.0.1", packages=find_packages(exclude=("tests", "tests.*")), url="", license="", From 5839e7897d827b63698eb1d9267999ef01775e7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Fri, 1 Jul 2022 16:46:01 -0400 Subject: [PATCH 016/100] Prefixed module with `_` so it has a different name than its pkg --- simulator/factory/common/__init__.py | 2 +- simulator/factory/common/{common.py => _common.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename simulator/factory/common/{common.py => _common.py} (100%) mode change 100755 => 100644 diff --git a/simulator/factory/common/__init__.py b/simulator/factory/common/__init__.py index fd9d22f..8e48e44 100755 --- a/simulator/factory/common/__init__.py +++ b/simulator/factory/common/__init__.py @@ -1 +1 @@ -from .common import AttributeAsDictClass +from ._common import AttributeAsDictClass diff --git a/simulator/factory/common/common.py b/simulator/factory/common/_common.py old mode 100755 new mode 100644 similarity index 100% rename from simulator/factory/common/common.py rename to simulator/factory/common/_common.py From 59019ef85b9241fed102360fdaeecb56616acf3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Fri, 1 Jul 2022 16:47:11 -0400 Subject: [PATCH 017/100] Prefixed module with `_` so it has a different name than its pkg --- simulator/factory/geometry_factory/__init__.py | 2 +- .../{geometry_factory.py => _geometry_factory.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename simulator/factory/geometry_factory/{geometry_factory.py => _geometry_factory.py} (100%) mode change 100755 => 100644 diff --git a/simulator/factory/geometry_factory/__init__.py b/simulator/factory/geometry_factory/__init__.py index b69e045..6cbe349 100755 --- a/simulator/factory/geometry_factory/__init__.py +++ b/simulator/factory/geometry_factory/__init__.py @@ -1,3 +1,3 @@ -from .geometry_factory import GeometryFactory +from ._geometry_factory import GeometryFactory from simulator.factory.geometry_factory.utils.plane import Plane from simulator.factory.geometry_factory.utils.rotation import Rotation diff --git a/simulator/factory/geometry_factory/geometry_factory.py b/simulator/factory/geometry_factory/_geometry_factory.py old mode 100755 new mode 100644 similarity index 100% rename from simulator/factory/geometry_factory/geometry_factory.py rename to simulator/factory/geometry_factory/_geometry_factory.py From 73b58f809fff76958431ba68d65584af29348392 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Fri, 1 Jul 2022 16:49:12 -0400 Subject: [PATCH 018/100] Prefixed module with `_` so it has a different name than its pkg --- simulator/factory/simulation_factory/__init__.py | 2 +- .../{simulation_factory.py => _simulation_factory.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename simulator/factory/simulation_factory/{simulation_factory.py => _simulation_factory.py} (100%) mode change 100755 => 100644 diff --git a/simulator/factory/simulation_factory/__init__.py b/simulator/factory/simulation_factory/__init__.py index 1cb8895..fe0c0a9 100755 --- a/simulator/factory/simulation_factory/__init__.py +++ b/simulator/factory/simulation_factory/__init__.py @@ -1 +1 @@ -from .simulation_factory import SimulationFactory +from ._simulation_factory import SimulationFactory diff --git a/simulator/factory/simulation_factory/simulation_factory.py b/simulator/factory/simulation_factory/_simulation_factory.py old mode 100755 new mode 100644 similarity index 100% rename from simulator/factory/simulation_factory/simulation_factory.py rename to simulator/factory/simulation_factory/_simulation_factory.py From bd10859860562c9e0f2948bdf75faa0837b8025b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Fri, 1 Jul 2022 16:55:02 -0400 Subject: [PATCH 019/100] Bumped version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c2b3514..6e94a39 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ if __name__ == "__main__": setup( name="simulation_generator", - version="1.0.1", + version="1.0.2", packages=find_packages(exclude=("tests", "tests.*")), url="", license="", From 109f4b482fa6fb6aa2c4fca15c09e22f97c189fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Sun, 3 Jul 2022 15:44:28 -0400 Subject: [PATCH 020/100] Fragment os imports --- simulator/runner/simulation_runner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index ae13528..27464e9 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -1,7 +1,8 @@ import logging from asyncio import get_event_loop, new_event_loop, set_event_loop -from os import makedirs, path +from os import makedirs +from os import path from os.path import basename from subprocess import PIPE, Popen From ec44af855c3e2a4b7bd5d43cdac7c9de912d7ca7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Sun, 3 Jul 2022 15:46:30 -0400 Subject: [PATCH 021/100] Use import instead of import-from for os.path.basename --- simulator/runner/simulation_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index 27464e9..28a2119 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -3,14 +3,14 @@ from asyncio import get_event_loop, new_event_loop, set_event_loop from os import makedirs from os import path -from os.path import basename +import os.path from subprocess import PIPE, Popen from .config import SingularityConfig from .datastore import Datastore from ..utils.logging import RTLogging -logger = logging.getLogger(basename(__file__).split(".")[0]) +logger = logging.getLogger(os.path.basename(__file__).split(".")[0]) class AsyncRunner: From 60960b16a2d26dcb20917a482afd17444ca2dd97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Sun, 3 Jul 2022 15:48:23 -0400 Subject: [PATCH 022/100] Use import instead of import-from for os.path --- simulator/runner/simulation_runner.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index 28a2119..40a2ecf 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -2,8 +2,7 @@ from asyncio import get_event_loop, new_event_loop, set_event_loop from os import makedirs -from os import path -import os.path +import os from subprocess import PIPE, Popen from .config import SingularityConfig @@ -93,8 +92,8 @@ def run( ) datastore = Datastore( - path.join(output_folder, "simulation"), - path.join( + os.path.join(output_folder, "simulation"), + os.path.join( output_folder, "phantom", "{}_phantom_merged_bundles.fib".format(run_name), @@ -104,7 +103,7 @@ def run( ) datastore.load_compartments( - path.join(output_folder, "phantom"), run_name, output_nifti + os.path.join(output_folder, "phantom"), run_name, output_nifti ) datastore.stage_compartments(run_name) @@ -136,17 +135,17 @@ def generate_phantom( base_output_folder = output_folder output_folder = self._create_outputs( - path.join(output_folder, "phantom") + os.path.join(output_folder, "phantom") ) - phantom_def = path.join( + phantom_def = os.path.join( phantom_infos["file_path"], phantom_infos["base_file"] ) resolution = ",".join([str(r) for r in phantom_infos["resolution"]]) spacing = ",".join([str(s) for s in phantom_infos["spacing"]]) fiber_fraction = "rel" if relative_fiber_fraction else "abs" - out_name = path.join( + out_name = os.path.join( output_folder, "phantom, " "{}_phantom".format(run_name) ) @@ -159,7 +158,7 @@ def generate_phantom( bind_paths = ",".join([phantom_infos["file_path"], output_folder]) command = self._bind_singularity("phantom", bind_paths, arguments) - log_file = path.join(base_output_folder, "{}.log".format(run_name)) + log_file = os.path.join(base_output_folder, "{}.log".format(run_name)) self._run_command(command, log_file, "[PHANTOM]") loop_managed or self.stop() @@ -181,18 +180,18 @@ def simulate_diffusion_mri( bind_paths = [] if bind_paths is None else bind_paths base_output_folder = output_folder output_folder = self._create_outputs( - path.join(output_folder, "simulation") + os.path.join(output_folder, "simulation") ) name = "{}_simulation".format(run_name) bind_paths += [simulation_infos["file_path"], output_folder] bind_paths = ",".join(bind_paths) - ffp_file = path.join( + ffp_file = os.path.join( simulation_infos["file_path"], simulation_infos["param_file"] ) extension = "nii.gz" if output_nifti else "nrrd" - out_name = path.join(output_folder, "{}.{}".format(name, extension)) + out_name = os.path.join(output_folder, "{}.{}".format(name, extension)) if not compartments_staged and compartment_maps is not None: datastore = Datastore( @@ -207,7 +206,7 @@ def simulate_diffusion_mri( arguments = "-p {} -i {} -o {}".format(ffp_file, fibers_file, out_name) command = self._bind_singularity("diffusion mri", bind_paths, arguments) - log_file = path.join(base_output_folder, "{}.log".format(run_name)) + log_file = os.path.join(base_output_folder, "{}.log".format(run_name)) self._run_command(command, log_file, "[DIFFUSION MRI]") loop_managed or self.stop() From ed2d7b6e1a5b50f29eea50c602f4923279365fc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Sun, 3 Jul 2022 15:49:21 -0400 Subject: [PATCH 023/100] Fixed `_create_outputs` name shadowing of `os.path` due to param `path` --- simulator/runner/simulation_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index 40a2ecf..a75f36a 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -65,7 +65,7 @@ def _bind_singularity(self, step, paths, arguments): ) def _create_outputs(self, path): - if not path.exists(path): + if not os.path.exists(path): makedirs(path, exist_ok=True) return path From 9db96aaa7af0046e9a1e712852854e355d846bee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Sun, 3 Jul 2022 15:50:17 -0400 Subject: [PATCH 024/100] Followed recommendation to make `_create_outputs` static --- simulator/runner/simulation_runner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index a75f36a..da0beee 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -64,7 +64,8 @@ def _bind_singularity(self, step, paths, arguments): arguments, ) - def _create_outputs(self, path): + @staticmethod + def _create_outputs(path): if not os.path.exists(path): makedirs(path, exist_ok=True) From bcf88b55f174e991ef6b94cee56aeef13c77001c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Sun, 3 Jul 2022 15:59:42 -0400 Subject: [PATCH 025/100] Bumped version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6e94a39..8825144 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ if __name__ == "__main__": setup( name="simulation_generator", - version="1.0.2", + version="1.0.3", packages=find_packages(exclude=("tests", "tests.*")), url="", license="", From b786c19860564c36bfb4c42afac2e84b700bfa57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 12:24:23 -0400 Subject: [PATCH 026/100] Fixed str.join of Path results in an error --- simulator/runner/simulation_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index da0beee..3b1be6e 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -157,7 +157,7 @@ def generate_phantom( if output_nifti: arguments += " --nii" - bind_paths = ",".join([phantom_infos["file_path"], output_folder]) + bind_paths = ",".join([str(phantom_infos["file_path"]), str(output_folder)]) command = self._bind_singularity("phantom", bind_paths, arguments) log_file = os.path.join(base_output_folder, "{}.log".format(run_name)) self._run_command(command, log_file, "[PHANTOM]") @@ -186,7 +186,7 @@ def simulate_diffusion_mri( name = "{}_simulation".format(run_name) - bind_paths += [simulation_infos["file_path"], output_folder] + bind_paths += [str(simulation_infos["file_path"]), str(output_folder)] bind_paths = ",".join(bind_paths) ffp_file = os.path.join( simulation_infos["file_path"], simulation_infos["param_file"] From 26edbb57154a582f36772825f05617e252df0223 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 12:27:21 -0400 Subject: [PATCH 027/100] Fixed str.join of Path results in an error in legacy runner --- simulator/runner/legacy.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/simulator/runner/legacy.py b/simulator/runner/legacy.py index b73efbb..a56e9ad 100644 --- a/simulator/runner/legacy.py +++ b/simulator/runner/legacy.py @@ -73,7 +73,7 @@ def run_simulation_dwimage( "{} run -B {} --app launch_mitk {} -p {} -i {} -o {} {}".format( self._singularity_exec, ",".join( - [simulation_infos["file_path"], simulation_output_folder] + [str(simulation_infos["file_path"]), str(simulation_output_folder)] ), self._singularity, path.join( @@ -149,9 +149,9 @@ def run_simulation_standalone( self._singularity_exec, ",".join( [ - geometry_folder, - simulation_infos["file_path"], - simulation_output_folder, + str(geometry_folder), + str(simulation_infos["file_path"]), + str(simulation_output_folder), ] ), self._singularity, @@ -231,7 +231,7 @@ def run( geometry_command = ( "singularity run -B {} --app launch_voxsim {} -f {} -r {} " "-s {} -o {} --comp-map {} --quiet{}".format( - ",".join([self._geometry_path, geometry_output_folder]), + ",".join([str(self._geometry_path), str(geometry_output_folder)]), self._singularity, path.join(self._geometry_path, self._geometry_base_file), ",".join([str(r) for r in self._geometry_resolution]), @@ -248,9 +248,9 @@ def run( "-p {} -i {} -o {} {}".format( ",".join( [ - self._simulation_path, - geometry_output_folder, - simulation_output_folder, + str(self._simulation_path), + str(geometry_output_folder), + str(simulation_output_folder), ] ), self._singularity, From f7f73f54b72fd6341a6d6f8d89a429877c4c6882 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 13:04:56 -0400 Subject: [PATCH 028/100] Bumped version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8825144..0bf741d 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ if __name__ == "__main__": setup( name="simulation_generator", - version="1.0.3", + version="1.0.4", packages=find_packages(exclude=("tests", "tests.*")), url="", license="", From a36600cd54bf6742a5b7549ebf96b1408b797229 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 14:47:49 -0400 Subject: [PATCH 029/100] Use Path instead of PurePath --- simulator/default.py | 4 ++-- simulator/runner/config.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/simulator/default.py b/simulator/default.py index 3a170b7..fb8f1d8 100644 --- a/simulator/default.py +++ b/simulator/default.py @@ -1,7 +1,7 @@ import typing import pathlib -SINGULARITY_PATH: typing.Final[pathlib.PurePath] = pathlib.PurePath() +SINGULARITY_PATH: typing.Final[pathlib.Path] = pathlib.Path() SINGULARITY_NAME: typing.Final[str] = "voxsim_singularity_latest.sif" -SINGULARITY: typing.Final[pathlib.PurePath] = SINGULARITY_PATH / SINGULARITY_NAME +SINGULARITY: typing.Final[pathlib.Path] = SINGULARITY_PATH / SINGULARITY_NAME SINGULARITY_EXEC: typing.Final[str] = "singularity" diff --git a/simulator/runner/config.py b/simulator/runner/config.py index 1667208..303675e 100644 --- a/simulator/runner/config.py +++ b/simulator/runner/config.py @@ -6,5 +6,5 @@ @dataclasses.dataclass class SingularityConfig: - singularity: pathlib.PurePath = default.SINGULARITY + singularity: pathlib.Path = default.SINGULARITY singularity_exec: str = default.SINGULARITY_EXEC From d009daee0aed1bf6cb48bbdda1cc1518b4866280 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 15:53:23 -0400 Subject: [PATCH 030/100] Added PathLike typing --- simulator/typing/__init__.py | 1 + simulator/typing/_typing.py | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 simulator/typing/__init__.py create mode 100644 simulator/typing/_typing.py diff --git a/simulator/typing/__init__.py b/simulator/typing/__init__.py new file mode 100644 index 0000000..36f52b9 --- /dev/null +++ b/simulator/typing/__init__.py @@ -0,0 +1 @@ +from ._typing import PathLike diff --git a/simulator/typing/_typing.py b/simulator/typing/_typing.py new file mode 100644 index 0000000..b0cd15f --- /dev/null +++ b/simulator/typing/_typing.py @@ -0,0 +1,4 @@ +import typing +import pathlib + +PathLike = typing.Union[str, pathlib.PurePath] From 1bde0a5f196c56636ebf8d34e735a0f965db8515 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 17:25:01 -0400 Subject: [PATCH 031/100] Time to switch to pathlib only --- simulator/typing/__init__.py | 1 - simulator/typing/_typing.py | 4 ---- 2 files changed, 5 deletions(-) delete mode 100644 simulator/typing/__init__.py delete mode 100644 simulator/typing/_typing.py diff --git a/simulator/typing/__init__.py b/simulator/typing/__init__.py deleted file mode 100644 index 36f52b9..0000000 --- a/simulator/typing/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from ._typing import PathLike diff --git a/simulator/typing/_typing.py b/simulator/typing/_typing.py deleted file mode 100644 index b0cd15f..0000000 --- a/simulator/typing/_typing.py +++ /dev/null @@ -1,4 +0,0 @@ -import typing -import pathlib - -PathLike = typing.Union[str, pathlib.PurePath] From f3f55f97b27e638995fc9b98b9a0fe01c513691e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 17:34:41 -0400 Subject: [PATCH 032/100] Format --- simulator/factory/geometry_factory/handlers/geometry_handler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/simulator/factory/geometry_factory/handlers/geometry_handler.py b/simulator/factory/geometry_factory/handlers/geometry_handler.py index 899944d..83354e9 100755 --- a/simulator/factory/geometry_factory/handlers/geometry_handler.py +++ b/simulator/factory/geometry_factory/handlers/geometry_handler.py @@ -6,6 +6,7 @@ class GeometryHandler: + def __init__(self, resolution, spacing, clusters=None, spheres=None): self._parameters_dict = { "resolution": resolution, From cce35f8f290d07d2ca87c8443a8945f9700234ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 17:34:55 -0400 Subject: [PATCH 033/100] Use pathlib in geomtetry_handler WIP --- .../handlers/geometry_handler.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/simulator/factory/geometry_factory/handlers/geometry_handler.py b/simulator/factory/geometry_factory/handlers/geometry_handler.py index 83354e9..374cc57 100755 --- a/simulator/factory/geometry_factory/handlers/geometry_handler.py +++ b/simulator/factory/geometry_factory/handlers/geometry_handler.py @@ -1,5 +1,5 @@ from copy import deepcopy -from os import makedirs, path +import pathlib from ..features.ORM.config_builder import ConfigBuilder from .geometry_infos import GeometryInfos @@ -74,15 +74,13 @@ def _get_number_of_clusters(self): return len(self._parameters_dict["clusters"]) def generate_json_configuration_files( - self, output_naming, simulation_path="" + self, output_naming, simulation_path: pathlib.Path = pathlib.Path() ): - if not path.exists(simulation_path): - makedirs(simulation_path, exist_ok=True) + simulation_path.mkdir(parents=True, exist_ok=True) with open( - path.join(simulation_path, output_naming + "_base.json"), "w+" + simulation_path / output_naming / "_base.json", "w+" ) as base_file: - world = ConfigBuilder.create_world( len(self.get_resolution()), self.get_resolution() ) @@ -110,11 +108,8 @@ def generate_json_configuration_files( for cluster_idx in range(len(self._parameters_dict["clusters"])): with open( - path.join( - simulation_path, - output_naming + "_f_{}.vspl".format(cluster_idx), - ), - "w+", + simulation_path / (output_naming + "_f_{}.vspl".format(cluster_idx)), + "w+", ) as f: f.write( self._parameters_dict["clusters"][cluster_idx].serialize() From 3de9dbe0fa6337620f9b375daa4fe5cffaa4e6e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 17:38:47 -0400 Subject: [PATCH 034/100] Added typing to generate_json_configuration_files --- .../factory/geometry_factory/handlers/geometry_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/simulator/factory/geometry_factory/handlers/geometry_handler.py b/simulator/factory/geometry_factory/handlers/geometry_handler.py index 374cc57..e3f5e5d 100755 --- a/simulator/factory/geometry_factory/handlers/geometry_handler.py +++ b/simulator/factory/geometry_factory/handlers/geometry_handler.py @@ -74,8 +74,8 @@ def _get_number_of_clusters(self): return len(self._parameters_dict["clusters"]) def generate_json_configuration_files( - self, output_naming, simulation_path: pathlib.Path = pathlib.Path() - ): + self, output_naming: str, simulation_path: pathlib.Path = pathlib.Path() + ) -> GeometryInfos: simulation_path.mkdir(parents=True, exist_ok=True) with open( From d7a10b284e771dd39f82245f05025407bcd23d11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 17:57:19 -0400 Subject: [PATCH 035/100] `__file__` is optional; use `__name__` instead --- simulator/runner/legacy.py | 2 +- simulator/runner/simulation_runner.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/simulator/runner/legacy.py b/simulator/runner/legacy.py index a56e9ad..f8dd304 100644 --- a/simulator/runner/legacy.py +++ b/simulator/runner/legacy.py @@ -14,7 +14,7 @@ from ..exceptions import SimulationRunnerException from ..utils.logging import RTLogging -logger = logging.getLogger(basename(__file__).split(".")[0]) +logger = logging.getLogger(__name__) class SimulationRunner: diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index 3b1be6e..c61593c 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -9,7 +9,7 @@ from .datastore import Datastore from ..utils.logging import RTLogging -logger = logging.getLogger(os.path.basename(__file__).split(".")[0]) +logger = logging.getLogger(__name__) class AsyncRunner: From c982378d26aac91fe59befca0eb6a2605537a124 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 17:57:34 -0400 Subject: [PATCH 036/100] Resolve simulation path --- simulator/factory/geometry_factory/handlers/geometry_handler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/simulator/factory/geometry_factory/handlers/geometry_handler.py b/simulator/factory/geometry_factory/handlers/geometry_handler.py index e3f5e5d..b81b85c 100755 --- a/simulator/factory/geometry_factory/handlers/geometry_handler.py +++ b/simulator/factory/geometry_factory/handlers/geometry_handler.py @@ -77,6 +77,7 @@ def generate_json_configuration_files( self, output_naming: str, simulation_path: pathlib.Path = pathlib.Path() ) -> GeometryInfos: simulation_path.mkdir(parents=True, exist_ok=True) + simulation_path = simulation_path.resolve(strict=True) with open( simulation_path / output_naming / "_base.json", "w+" From bc908250b2468c5953759457f32a6fa87084d2c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 18:00:10 -0400 Subject: [PATCH 037/100] logger var should be private --- simulator/runner/legacy.py | 22 +++++++++++----------- simulator/runner/simulation_runner.py | 8 ++++---- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/simulator/runner/legacy.py b/simulator/runner/legacy.py index f8dd304..d808646 100644 --- a/simulator/runner/legacy.py +++ b/simulator/runner/legacy.py @@ -14,7 +14,7 @@ from ..exceptions import SimulationRunnerException from ..utils.logging import RTLogging -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) class SimulationRunner: @@ -103,7 +103,7 @@ def run_simulation_dwimage( async_loop = get_event_loop() log_file = path.join(output_folder, "{}.log".format(self._base_naming)) - logger.info("Simulating DWI signal") + _logger.info("Simulating DWI signal") return_code, log = async_loop.run_until_complete( self._launch_command( simulation_command, log_file, "[RUNNING FIBERFOX]" @@ -117,7 +117,7 @@ def run_simulation_dwimage( (log,), ) - logger.debug( + _logger.debug( "Simulation {} ended with code {}".format( self._base_naming, return_code ) @@ -190,7 +190,7 @@ def run_simulation_standalone( base_naming, ) - logger.info("Simulating DWI signal") + _logger.info("Simulating DWI signal") return_code, log = async_loop.run_until_complete( self._launch_command( simulation_command, log_file, "[RUNNING FIBERFOX]" @@ -204,7 +204,7 @@ def run_simulation_standalone( (log,), ) - logger.debug( + _logger.debug( "Simulation {} ended with code {}".format( self._base_naming, return_code ) @@ -282,7 +282,7 @@ def run( async_loop = get_event_loop() log_file = path.join(output_folder, "{}.log".format(self._base_naming)) - logger.info("Generating simulation geometry") + _logger.info("Generating simulation geometry") async_loop.run_until_complete( self._launch_command(geometry_command, log_file, "[RUNNING VOXSIM]") ) @@ -290,7 +290,7 @@ def run( self._rename_and_copy_compartments( geometry_output_folder, simulation_output_folder ) - logger.info("Simulating DWI signal") + _logger.info("Simulating DWI signal") if self._run_simulation: return_code, log = async_loop.run_until_complete( self._launch_command( @@ -305,7 +305,7 @@ def run( (log,), ) - logger.debug("Simulation ended with code {}".format(return_code)) + _logger.debug("Simulation ended with code {}".format(return_code)) async_loop.close() def _execute_parallel(self, method, args): @@ -578,8 +578,8 @@ def _start_loop_if_closed(self): async def _launch_command(self, command, log_file, log_tag): process = Popen(command.split(" "), stdout=PIPE, stderr=PIPE) - logger = RTLogging(process, log_file, log_tag) - logger.start() - logger.join() + _logger = RTLogging(process, log_file, log_tag) + _logger.start() + _logger.join() return process.returncode, log_file diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index c61593c..94b9e6a 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -9,7 +9,7 @@ from .datastore import Datastore from ..utils.logging import RTLogging -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) class AsyncRunner: @@ -39,9 +39,9 @@ def _start_loop_if_closed(self): async def _run_async(self, command, log_file, log_tag): process = Popen(command.split(" "), stdout=PIPE, stderr=PIPE) - logger = RTLogging(process, log_file, log_tag) - logger.start() - logger.join() + _logger = RTLogging(process, log_file, log_tag) + _logger.start() + _logger.join() return process.returncode, log_file From 7dfa1f62cc5755729db830b643e19d7febea25a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 18:12:35 -0400 Subject: [PATCH 038/100] Catch resolve exception --- .../geometry_factory/handlers/geometry_handler.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/simulator/factory/geometry_factory/handlers/geometry_handler.py b/simulator/factory/geometry_factory/handlers/geometry_handler.py index b81b85c..fd02773 100755 --- a/simulator/factory/geometry_factory/handlers/geometry_handler.py +++ b/simulator/factory/geometry_factory/handlers/geometry_handler.py @@ -1,9 +1,13 @@ -from copy import deepcopy +import logging import pathlib +from copy import deepcopy + from ..features.ORM.config_builder import ConfigBuilder from .geometry_infos import GeometryInfos +_logger = logging.getLogger(__name__) + class GeometryHandler: @@ -77,7 +81,14 @@ def generate_json_configuration_files( self, output_naming: str, simulation_path: pathlib.Path = pathlib.Path() ) -> GeometryInfos: simulation_path.mkdir(parents=True, exist_ok=True) - simulation_path = simulation_path.resolve(strict=True) + + try: + simulation_path = simulation_path.resolve(strict=True) + except FileNotFoundError as exc: + _logger.exception( + f"simulation_path does not exist, even after the creation of its directories: {simulation_path}", + exc_info=exc) + raise exc with open( simulation_path / output_naming / "_base.json", "w+" From 0c7b2c8772fcb9153813ef59c5850eb0ac99320d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 18:20:04 -0400 Subject: [PATCH 039/100] Use pathlib in `logging` --- simulator/utils/logging.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/simulator/utils/logging.py b/simulator/utils/logging.py index 89f60d3..a75803b 100755 --- a/simulator/utils/logging.py +++ b/simulator/utils/logging.py @@ -1,12 +1,13 @@ from queue import Queue from threading import Thread +import pathlib import time class RTLogging: - def __init__(self, process, log_file_path, log_tag=""): + def __init__(self, process, log_file_path: pathlib.Path, log_tag=""): self._process = process - self._log = log_file_path + self._log: pathlib.Path = log_file_path self._tag = log_tag self._thread = None From 09824b187a7eae76baaa248c593e07ad6f8f2534 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 18:27:46 -0400 Subject: [PATCH 040/100] Replaced `file_path` and `base_file` for typed `file` --- .../handlers/geometry_infos.py | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/simulator/factory/geometry_factory/handlers/geometry_infos.py b/simulator/factory/geometry_factory/handlers/geometry_infos.py index 0f8839c..a5539e5 100755 --- a/simulator/factory/geometry_factory/handlers/geometry_infos.py +++ b/simulator/factory/geometry_factory/handlers/geometry_infos.py @@ -1,28 +1,24 @@ from ...common import AttributeAsDictClass +import pathlib class GeometryInfos(AttributeAsDictClass): def __init__( - self, file_path, base_file, resolution, spacing, n_maps, **kwargs + self, file: pathlib.Path, resolution, spacing, n_maps, **kwargs ): super().__init__(**kwargs) - self.generate_new_key("file_path", file_path) - self.generate_new_key("base_file", base_file) + self.generate_new_key("file", file) self.generate_new_key("resolution", resolution) self.generate_new_key("spacing", spacing) self.generate_new_key("n_maps", n_maps) - def get_file_path(self): - return self._file_path + @property + def file(self) -> pathlib.Path: + return self._file - def set_file_path(self, path): - self._file_path = path - - def get_base_file_name(self): - return self._base_file - - def set_base_file_name(self, name): - self._base_file = name + @file.setter + def file(self, file: pathlib.Path): + self._file = file def get_resolution(self): return self._resolution From 20a6322a6b0272c052c8866c3b071020a730c6f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 18:38:55 -0400 Subject: [PATCH 041/100] Finally, decided to keep API --- .../handlers/geometry_infos.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) mode change 100755 => 100644 simulator/factory/geometry_factory/handlers/geometry_infos.py diff --git a/simulator/factory/geometry_factory/handlers/geometry_infos.py b/simulator/factory/geometry_factory/handlers/geometry_infos.py old mode 100755 new mode 100644 index a5539e5..06a0274 --- a/simulator/factory/geometry_factory/handlers/geometry_infos.py +++ b/simulator/factory/geometry_factory/handlers/geometry_infos.py @@ -4,21 +4,26 @@ class GeometryInfos(AttributeAsDictClass): def __init__( - self, file: pathlib.Path, resolution, spacing, n_maps, **kwargs + self, file_path: pathlib.Path, base_file: pathlib.Path, resolution, spacing, n_maps, **kwargs ): super().__init__(**kwargs) - self.generate_new_key("file", file) + self.generate_new_key("file_path", file_path) + self.generate_new_key("base_file", base_file) self.generate_new_key("resolution", resolution) self.generate_new_key("spacing", spacing) self.generate_new_key("n_maps", n_maps) - @property - def file(self) -> pathlib.Path: - return self._file + def get_file_path(self) -> pathlib.Path: + return self._file_path - @file.setter - def file(self, file: pathlib.Path): - self._file = file + def set_file_path(self, path: pathlib.Path): + self._file_path = path + + def get_base_file_name(self) -> pathlib.Path: + return self._base_file + + def set_base_file_name(self, name: pathlib.Path): + self._base_file = name def get_resolution(self): return self._resolution From d26f36764369290bfdfed566c39446d6c69970e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 18:40:59 -0400 Subject: [PATCH 042/100] Type annotate paths in SimulationInfos --- .../simulation_factory/handlers/simulation_infos.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/simulator/factory/simulation_factory/handlers/simulation_infos.py b/simulator/factory/simulation_factory/handlers/simulation_infos.py index 1ba8d57..f9d2089 100755 --- a/simulator/factory/simulation_factory/handlers/simulation_infos.py +++ b/simulator/factory/simulation_factory/handlers/simulation_infos.py @@ -1,23 +1,24 @@ from ...common import AttributeAsDictClass +import pathlib class SimulationInfos(AttributeAsDictClass): - def __init__(self, file_path, simulation_file_name, ids): + def __init__(self, file_path: pathlib.Path, simulation_file_name: pathlib.Path, ids): super().__init__() self.generate_new_key("file_path", file_path) self.generate_new_key("param_file", simulation_file_name) self.generate_new_key("compartment_ids", ids) - def get_file_path(self): + def get_file_path(self) -> pathlib.Path: return self._file_path - def set_file_path(self, file_path): + def set_file_path(self, file_path: pathlib.Path): self._file_path = file_path - def get_simulation_file_name(self): + def get_simulation_file_name(self) -> pathlib.Path: return self._param_file - def set_simulation_file_name(self, name): + def set_simulation_file_name(self, name: pathlib.Path): self._param_file = name @classmethod From 6aeaab4f42b3e1889ad666fe436971ddea686647 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 18:42:06 -0400 Subject: [PATCH 043/100] Format --- simulator/runner/legacy.py | 56 +++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/simulator/runner/legacy.py b/simulator/runner/legacy.py index d808646..56a1af3 100644 --- a/simulator/runner/legacy.py +++ b/simulator/runner/legacy.py @@ -20,12 +20,12 @@ class SimulationRunner: def __init__( - self, - base_naming, - geometry_infos, - simulation_infos=None, - singularity_conf=SingularityConfig(), - output_nifti=False, + self, + base_naming, + geometry_infos, + simulation_infos=None, + singularity_conf=SingularityConfig(), + output_nifti=False, ): self._geometry_path = geometry_infos["file_path"] self._geometry_base_file = geometry_infos["base_file"] @@ -60,7 +60,7 @@ def set_geometry_base_naming(self, name): self._geometry_base_naming = name def run_simulation_dwimage( - self, output_folder, image_file, simulation_infos, test_mode=False + self, output_folder, image_file, simulation_infos, test_mode=False ): self._start_loop_if_closed() simulation_output_folder = path.join( @@ -125,12 +125,12 @@ def run_simulation_dwimage( async_loop.close() def run_simulation_standalone( - self, - output_folder, - geometry_folder, - simulation_infos, - base_naming=None, - test_mode=False, + self, + output_folder, + geometry_folder, + simulation_infos, + base_naming=None, + test_mode=False, ): if not base_naming: base_naming = self._base_naming @@ -212,7 +212,7 @@ def run_simulation_standalone( async_loop.close() def run( - self, output_folder, test_mode=False, relative_fiber_compartment=True + self, output_folder, test_mode=False, relative_fiber_compartment=True ): geometry_output_folder = path.join(output_folder, "geometry_outputs") @@ -314,11 +314,11 @@ def _execute_parallel(self, method, args): p.join() def _rename_and_copy_compartments_standalone( - self, - simulation_infos, - geometry_output_folder, - simulation_output_folder, - base_naming, + self, + simulation_infos, + geometry_output_folder, + simulation_output_folder, + base_naming, ): copyfile( path.join( @@ -394,7 +394,7 @@ def _rename_and_copy_compartments_standalone( ) if len(simulation_infos["compartment_ids"]) > 2 and ( - merged_maps or base_map + merged_maps or base_map ): self._generate_background_map( geometry_output_folder, @@ -413,7 +413,7 @@ def _rename_and_copy_compartments_standalone( ) def _rename_and_copy_compartments( - self, geometry_output_folder, simulation_output_folder + self, geometry_output_folder, simulation_output_folder ): copyfile( path.join( @@ -515,13 +515,13 @@ def _save_nrrd(self, data, header, name): nrrd.write("{}.nrrd".format(name), data, header) def _generate_background_map( - self, - geometry_output_folder, - simulation_output_folder, - compartment_id, - merged_maps=False, - base_map=False, - base_naming=None, + self, + geometry_output_folder, + simulation_output_folder, + compartment_id, + merged_maps=False, + base_map=False, + base_naming=None, ): if not base_naming: base_naming = self._base_naming From 69b40aea04f08fd2d6d51e18bb277585e20ca38f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 18:43:13 -0400 Subject: [PATCH 044/100] Fixed filename type hints --- .../factory/geometry_factory/handlers/geometry_infos.py | 6 +++--- .../factory/simulation_factory/handlers/simulation_infos.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/simulator/factory/geometry_factory/handlers/geometry_infos.py b/simulator/factory/geometry_factory/handlers/geometry_infos.py index 06a0274..556a9f1 100644 --- a/simulator/factory/geometry_factory/handlers/geometry_infos.py +++ b/simulator/factory/geometry_factory/handlers/geometry_infos.py @@ -4,7 +4,7 @@ class GeometryInfos(AttributeAsDictClass): def __init__( - self, file_path: pathlib.Path, base_file: pathlib.Path, resolution, spacing, n_maps, **kwargs + self, file_path: pathlib.Path, base_file: str, resolution, spacing, n_maps, **kwargs ): super().__init__(**kwargs) self.generate_new_key("file_path", file_path) @@ -19,10 +19,10 @@ def get_file_path(self) -> pathlib.Path: def set_file_path(self, path: pathlib.Path): self._file_path = path - def get_base_file_name(self) -> pathlib.Path: + def get_base_file_name(self) -> str: return self._base_file - def set_base_file_name(self, name: pathlib.Path): + def set_base_file_name(self, name: str): self._base_file = name def get_resolution(self): diff --git a/simulator/factory/simulation_factory/handlers/simulation_infos.py b/simulator/factory/simulation_factory/handlers/simulation_infos.py index f9d2089..f7dd354 100755 --- a/simulator/factory/simulation_factory/handlers/simulation_infos.py +++ b/simulator/factory/simulation_factory/handlers/simulation_infos.py @@ -3,7 +3,7 @@ class SimulationInfos(AttributeAsDictClass): - def __init__(self, file_path: pathlib.Path, simulation_file_name: pathlib.Path, ids): + def __init__(self, file_path: pathlib.Path, simulation_file_name: str, ids): super().__init__() self.generate_new_key("file_path", file_path) self.generate_new_key("param_file", simulation_file_name) @@ -15,10 +15,10 @@ def get_file_path(self) -> pathlib.Path: def set_file_path(self, file_path: pathlib.Path): self._file_path = file_path - def get_simulation_file_name(self) -> pathlib.Path: + def get_simulation_file_name(self) -> str: return self._param_file - def set_simulation_file_name(self, name: pathlib.Path): + def set_simulation_file_name(self, name: str): self._param_file = name @classmethod From 7d826aa90a6d244a562fcd635127f4d595ca5d84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 18:45:13 -0400 Subject: [PATCH 045/100] Integrated pathlib into legacy runner --- simulator/runner/legacy.py | 165 ++++++++++++------------------------- 1 file changed, 51 insertions(+), 114 deletions(-) diff --git a/simulator/runner/legacy.py b/simulator/runner/legacy.py index 56a1af3..2d15649 100644 --- a/simulator/runner/legacy.py +++ b/simulator/runner/legacy.py @@ -1,5 +1,6 @@ from asyncio import get_event_loop, new_event_loop, set_event_loop import logging +import pathlib from multiprocessing import Process from os import makedirs, path from os.path import basename, exists @@ -27,23 +28,23 @@ def __init__( singularity_conf=SingularityConfig(), output_nifti=False, ): - self._geometry_path = geometry_infos["file_path"] - self._geometry_base_file = geometry_infos["base_file"] + self._geometry_path: pathlib.Path = geometry_infos["file_path"] + self._geometry_base_file: str = geometry_infos["base_file"] self._geometry_resolution = geometry_infos["resolution"] self._geometry_spacing = geometry_infos["spacing"] - self._base_naming = base_naming - self._geometry_base_naming = base_naming + self._base_naming: str = base_naming + self._geometry_base_naming: str = base_naming if simulation_infos: self._number_of_maps = len(simulation_infos["compartment_ids"]) - self._simulation_path = simulation_infos["file_path"] - self._simulation_parameters = simulation_infos["param_file"] + self._simulation_path: pathlib.Path = simulation_infos["file_path"] + self._simulation_parameters: str = simulation_infos["param_file"] self._compartment_ids = simulation_infos["compartment_ids"] singularity_conf = ( singularity_conf if singularity_conf else SingularityConfig() ) - self._singularity = singularity_conf.singularity - self._singularity_exec = singularity_conf.singularity_exec + self._singularity: pathlib.Path = singularity_conf.singularity + self._singularity_exec: str = singularity_conf.singularity_exec self._run_simulation = True if simulation_infos else False self._extension = "nii.gz" if output_nifti else "nrrd" @@ -53,21 +54,19 @@ def __init__( self._save_image = self._save_nifti if output_nifti else self._save_nrrd self._event_loop = new_event_loop() - def change_base_naming(self, name): + def change_base_naming(self, name: str): self._base_naming = name - def set_geometry_base_naming(self, name): + def set_geometry_base_naming(self, name: str): self._geometry_base_naming = name def run_simulation_dwimage( - self, output_folder, image_file, simulation_infos, test_mode=False + self, output_folder: pathlib.Path, image_file: pathlib.Path, simulation_infos, test_mode=False ): self._start_loop_if_closed() - simulation_output_folder = path.join( - output_folder, "simulation_outputs" - ) - if not path.exists(simulation_output_folder): - makedirs(simulation_output_folder, exist_ok=True) + + simulation_output_folder: pathlib.Path = output_folder / "simulation_outputs" + simulation_output_folder.mkdir(parents=True, exist_ok=True) simulation_command = ( "{} run -B {} --app launch_mitk {} -p {} -i {} -o {} {}".format( @@ -76,32 +75,21 @@ def run_simulation_dwimage( [str(simulation_infos["file_path"]), str(simulation_output_folder)] ), self._singularity, - path.join( - simulation_output_folder, - "{}_simulation.ffp".format(self._base_naming), - ), + simulation_output_folder / "{}_simulation.ffp".format(self._base_naming), image_file, - path.join( - simulation_output_folder, - "{}.{}".format(self._base_naming, self._extension), - ), + simulation_output_folder / "{}.{}".format(self._base_naming, self._extension), "-v" if test_mode else "", ) ) copyfile( - path.join( - simulation_infos["file_path"], simulation_infos["param_file"] - ), - path.join( - simulation_output_folder, - "{}_simulation.ffp".format(self._base_naming), - ), + simulation_infos["file_path"] / simulation_infos["param_file"], + simulation_output_folder / "{}_simulation.ffp".format(self._base_naming), ) set_event_loop(self._event_loop) async_loop = get_event_loop() - log_file = path.join(output_folder, "{}.log".format(self._base_naming)) + log_file: pathlib.Path = output_folder / "{}.log".format(self._base_naming) _logger.info("Simulating DWI signal") return_code, log = async_loop.run_until_complete( @@ -126,8 +114,8 @@ def run_simulation_dwimage( def run_simulation_standalone( self, - output_folder, - geometry_folder, + output_folder: pathlib.Path, + geometry_folder: pathlib.Path, simulation_infos, base_naming=None, test_mode=False, @@ -136,13 +124,10 @@ def run_simulation_standalone( base_naming = self._base_naming self._start_loop_if_closed() - simulation_output_folder = path.join( - output_folder, "simulation_outputs" - ) - geometry_output_folder = path.join(geometry_folder, "geometry_outputs") - if not path.exists(simulation_output_folder): - makedirs(simulation_output_folder, exist_ok=True) + simulation_output_folder = output_folder / "simulation_outputs" + geometry_output_folder = geometry_folder / "geometry_outputs" + simulation_output_folder.mkdir(parents=True, exist_ok=True) simulation_command = ( "{} run -B {} --app launch_mitk {} -p {} -i {} -o {} {}".format( @@ -155,33 +140,21 @@ def run_simulation_standalone( ] ), self._singularity, - path.join( - simulation_output_folder, - "{}_simulation.ffp".format(base_naming), - ), - path.join(geometry_output_folder, self._geometry_base_naming) - + "_merged_bundles.fib", - path.join( - simulation_output_folder, - "{}.{}".format(base_naming, self._extension), - ), + simulation_output_folder / "{}_simulation.ffp".format(base_naming), + geometry_output_folder / (self._geometry_base_naming + "_merged_bundles.fib"), + simulation_output_folder / "{}.{}".format(base_naming, self._extension), "-v" if test_mode else "", ) ) copyfile( - path.join( - simulation_infos["file_path"], simulation_infos["param_file"] - ), - path.join( - simulation_output_folder, - "{}_simulation.ffp".format(base_naming), - ), + simulation_infos["file_path"] / simulation_infos["param_file"], + simulation_output_folder / "{}_simulation.ffp".format(base_naming), ) set_event_loop(self._event_loop) async_loop = get_event_loop() - log_file = path.join(output_folder, "{}.log".format(base_naming)) + log_file: pathlib.Path = output_folder / "{}.log".format(base_naming) self._rename_and_copy_compartments_standalone( simulation_infos, @@ -212,31 +185,25 @@ def run_simulation_standalone( async_loop.close() def run( - self, output_folder, test_mode=False, relative_fiber_compartment=True + self, output_folder: pathlib.Path, test_mode=False, relative_fiber_compartment=True ): - geometry_output_folder = path.join(output_folder, "geometry_outputs") - - if not path.exists(geometry_output_folder): - makedirs(geometry_output_folder, exist_ok=True) + geometry_output_folder: pathlib.Path = output_folder / "geometry_outputs" + geometry_output_folder.mkdir(parents=True, exist_ok=True) if self._run_simulation: - simulation_output_folder = path.join( - output_folder, "simulation_outputs" - ) - - if not path.exists(simulation_output_folder): - makedirs(simulation_output_folder, exist_ok=True) + simulation_output_folder: pathlib.Path = output_folder / "simulation_outputs" + simulation_output_folder.mkdir(parents=True, exist_ok=True) geometry_command = ( "singularity run -B {} --app launch_voxsim {} -f {} -r {} " "-s {} -o {} --comp-map {} --quiet{}".format( ",".join([str(self._geometry_path), str(geometry_output_folder)]), self._singularity, - path.join(self._geometry_path, self._geometry_base_file), + self._geometry_path / self._geometry_base_file, ",".join([str(r) for r in self._geometry_resolution]), ",".join([str(s) for s in self._geometry_spacing]), - path.join(geometry_output_folder, self._geometry_base_naming), + geometry_output_folder / self._geometry_base_naming, "rel" if relative_fiber_compartment else "abs", self._fib_extension_arg, ) @@ -254,33 +221,21 @@ def run( ] ), self._singularity, - path.join( - simulation_output_folder, - "{}_simulation.ffp".format(self._base_naming), - ), - path.join( - geometry_output_folder, self._geometry_base_naming - ) - + "_merged_bundles.fib", - path.join( - simulation_output_folder, - "{}.{}".format(self._base_naming, self._extension), - ), + simulation_output_folder / "{}_simulation.ffp".format(self._base_naming), + geometry_output_folder / (self._geometry_base_naming + "_merged_bundles.fib"), + simulation_output_folder / "{}.{}".format(self._base_naming, self._extension), "-v" if test_mode else "", ) ) copyfile( - path.join(self._simulation_path, self._simulation_parameters), - path.join( - simulation_output_folder, - "{}_simulation.ffp".format(self._base_naming), - ), + self._simulation_path / self._simulation_parameters, + simulation_output_folder / "{}_simulation.ffp".format(self._base_naming), ) set_event_loop(self._event_loop) async_loop = get_event_loop() - log_file = path.join(output_folder, "{}.log".format(self._base_naming)) + log_file = output_folder / "{}.log".format(self._base_naming) _logger.info("Generating simulation geometry") async_loop.run_until_complete( @@ -316,8 +271,8 @@ def _execute_parallel(self, method, args): def _rename_and_copy_compartments_standalone( self, simulation_infos, - geometry_output_folder, - simulation_output_folder, + geometry_output_folder: pathlib.Path, + simulation_output_folder: pathlib.Path, base_naming, ): copyfile( @@ -527,33 +482,20 @@ def _generate_background_map( base_naming = self._base_naming img, header = self._load_image( - path.join( - geometry_output_folder, - "{}_mergedBundlesMaps".format(self._geometry_base_naming), - ) + geometry_output_folder / "{}_mergedBundlesMaps".format(self._geometry_base_naming) ) maps = [img] if merged_maps: maps.append( self._load_image( - path.join( - geometry_output_folder, - "{}{}".format( - self._geometry_base_naming, "_mergedEllipsesMaps" - ), - ) + geometry_output_folder / "{}{}".format(self._geometry_base_naming, "_mergedEllipsesMaps") )[0] ) elif base_map: maps.append( self._load_image( - path.join( - geometry_output_folder, - "{}{}".format( - self._geometry_base_naming, "_ellipsoid1_cmap" - ), - ) + geometry_output_folder / "{}{}".format(self._geometry_base_naming, "_ellipsoid1_cmap") )[0] ) @@ -563,19 +505,14 @@ def _generate_background_map( self._save_image( extra_map, header, - path.join( - simulation_output_folder, - "{}_simulation.ffp_VOLUME{}".format( - base_naming, compartment_id - ), - ), + simulation_output_folder / "{}_simulation.ffp_VOLUME{}".format(base_naming, compartment_id), ) def _start_loop_if_closed(self): if self._event_loop.is_closed(): self._event_loop = new_event_loop() - async def _launch_command(self, command, log_file, log_tag): + async def _launch_command(self, command, log_file: pathlib.Path, log_tag): process = Popen(command.split(" "), stdout=PIPE, stderr=PIPE) _logger = RTLogging(process, log_file, log_tag) From a7ae7a18c9c7d8cb20e6c75d97cc16f8d16d6717 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 20:17:54 -0400 Subject: [PATCH 046/100] Format --- simulator/runner/simulation_runner.py | 50 +++++++++++++-------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index 94b9e6a..68a25a4 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -72,14 +72,14 @@ def _create_outputs(path): return path def run( - self, - run_name, - phantom_infos, - simulation_infos, - output_folder, - output_nifti=True, - relative_fiber_fraction=True, - inter_axonal_fraction=None, + self, + run_name, + phantom_infos, + simulation_infos, + output_folder, + output_nifti=True, + relative_fiber_fraction=True, + inter_axonal_fraction=None, ): self.start() @@ -123,13 +123,13 @@ def run( self.stop() def generate_phantom( - self, - run_name, - phantom_infos, - output_folder, - relative_fiber_fraction=True, - output_nifti=True, - loop_managed=False, + self, + run_name, + phantom_infos, + output_folder, + relative_fiber_fraction=True, + output_nifti=True, + loop_managed=False, ): loop_managed or self.start() @@ -165,16 +165,16 @@ def generate_phantom( loop_managed or self.stop() def simulate_diffusion_mri( - self, - run_name, - simulation_infos, - output_folder, - fibers_file, - compartment_maps=None, - bind_paths=None, - output_nifti=True, - loop_managed=False, - compartments_staged=True, + self, + run_name, + simulation_infos, + output_folder, + fibers_file, + compartment_maps=None, + bind_paths=None, + output_nifti=True, + loop_managed=False, + compartments_staged=True, ): loop_managed or self.start() From ec4100ede56d87f402fe63cedc8c032f87df9a1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 20:18:36 -0400 Subject: [PATCH 047/100] Replaced `os.path.join` for `pathlib`'s `/` operator --- simulator/runner/simulation_runner.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index 68a25a4..543250d 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -93,18 +93,14 @@ def run( ) datastore = Datastore( - os.path.join(output_folder, "simulation"), - os.path.join( - output_folder, - "phantom", - "{}_phantom_merged_bundles.fib".format(run_name), - ), + output_folder / "simulation", + output_folder / "phantom" / "{}_phantom_merged_bundles.fib".format(run_name), simulation_infos["compartment_ids"], inter_axonal_fraction, ) datastore.load_compartments( - os.path.join(output_folder, "phantom"), run_name, output_nifti + output_folder / "phantom", run_name, output_nifti ) datastore.stage_compartments(run_name) @@ -136,7 +132,7 @@ def generate_phantom( base_output_folder = output_folder output_folder = self._create_outputs( - os.path.join(output_folder, "phantom") + output_folder / "phantom" ) phantom_def = os.path.join( @@ -159,7 +155,7 @@ def generate_phantom( bind_paths = ",".join([str(phantom_infos["file_path"]), str(output_folder)]) command = self._bind_singularity("phantom", bind_paths, arguments) - log_file = os.path.join(base_output_folder, "{}.log".format(run_name)) + log_file = base_output_folder / "{}.log".format(run_name) self._run_command(command, log_file, "[PHANTOM]") loop_managed or self.stop() @@ -181,7 +177,7 @@ def simulate_diffusion_mri( bind_paths = [] if bind_paths is None else bind_paths base_output_folder = output_folder output_folder = self._create_outputs( - os.path.join(output_folder, "simulation") + output_folder / "simulation" ) name = "{}_simulation".format(run_name) @@ -207,7 +203,7 @@ def simulate_diffusion_mri( arguments = "-p {} -i {} -o {}".format(ffp_file, fibers_file, out_name) command = self._bind_singularity("diffusion mri", bind_paths, arguments) - log_file = os.path.join(base_output_folder, "{}.log".format(run_name)) + log_file = base_output_folder / "{}.log".format(run_name) self._run_command(command, log_file, "[DIFFUSION MRI]") loop_managed or self.stop() From 730a6072eac4f8a832afbd9817452262d9d2366b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 20:27:44 -0400 Subject: [PATCH 048/100] Replaced `os.path.join` for `pathlib`'s `/` operator --- .../handlers/simulation_handler.py | 2 +- simulator/runner/legacy.py | 129 ++++++------------ simulator/runner/simulation_runner.py | 15 +- 3 files changed, 48 insertions(+), 98 deletions(-) diff --git a/simulator/factory/simulation_factory/handlers/simulation_handler.py b/simulator/factory/simulation_factory/handlers/simulation_handler.py index 9affdbf..36dc582 100755 --- a/simulator/factory/simulation_factory/handlers/simulation_handler.py +++ b/simulator/factory/simulation_factory/handlers/simulation_handler.py @@ -87,7 +87,7 @@ def generate_xml_configuration_file( xml_string = tostring(data, pretty_print=True).decode("utf-8") with open( - path.join(simulation_path, output_naming + ".ffp"), "w+" + simulation_path / (output_naming + ".ffp"), "w+" ) as f: f.write(xml_string) diff --git a/simulator/runner/legacy.py b/simulator/runner/legacy.py index 2d15649..25bd526 100644 --- a/simulator/runner/legacy.py +++ b/simulator/runner/legacy.py @@ -2,7 +2,6 @@ import logging import pathlib from multiprocessing import Process -from os import makedirs, path from os.path import basename, exists from shutil import copyfile from subprocess import PIPE, Popen @@ -276,66 +275,46 @@ def _rename_and_copy_compartments_standalone( base_naming, ): copyfile( - path.join( - geometry_output_folder, - self._geometry_base_naming - + "_mergedBundlesMaps.{}".format(self._extension), - ), - path.join( - simulation_output_folder, - "{}_simulation.ffp_VOLUME{}.{}".format( - base_naming, - simulation_infos["compartment_ids"][0], - self._extension, - ), + geometry_output_folder / (self._geometry_base_naming + "_mergedBundlesMaps.{}".format(self._extension)), + simulation_output_folder / + "{}_simulation.ffp_VOLUME{}.{}".format( + base_naming, + simulation_infos["compartment_ids"][0], + self._extension, ), ) if len(simulation_infos["compartment_ids"]) > 1: merged_maps = exists( - path.join( - geometry_output_folder, - self._geometry_base_naming - + "_mergedEllipsesMaps.{}".format(self._extension), - ) + geometry_output_folder / (self._geometry_base_naming + "_mergedEllipsesMaps.{}".format(self._extension)) ) base_map = (not merged_maps) and exists( - path.join( - geometry_output_folder, - self._geometry_base_naming - + "_ellipsoid{}_cmap.{}".format(0, self._extension), + geometry_output_folder / ( + self._geometry_base_naming + "_ellipsoid{}_cmap.{}".format(0, self._extension) ) ) if merged_maps: copyfile( - path.join( - geometry_output_folder, - self._geometry_base_naming - + "_mergedEllipsesMaps.{}".format(self._extension), + geometry_output_folder / ( + self._geometry_base_naming + "_mergedEllipsesMaps.{}".format(self._extension) ), - path.join( - simulation_output_folder, - "{}_simulation.ffp_VOLUME{}.{}".format( - base_naming, - simulation_infos["compartment_ids"][1], - self._extension, - ), + simulation_output_folder / + "{}_simulation.ffp_VOLUME{}.{}".format( + base_naming, + simulation_infos["compartment_ids"][1], + self._extension, ), ) elif base_map: copyfile( - path.join( - geometry_output_folder, - self._geometry_base_naming - + "_ellipsoid1_cmap.{}".format(self._extension), + geometry_output_folder / ( + self._geometry_base_naming + "_ellipsoid1_cmap.{}".format(self._extension) ), - path.join( - simulation_output_folder, - "{}_simulation.ffp_VOLUME{}.{}".format( - base_naming, - simulation_infos["compartment_ids"][1], - self._extension, - ), + simulation_output_folder / + "{}_simulation.ffp_VOLUME{}.{}".format( + base_naming, + simulation_infos["compartment_ids"][1], + self._extension, ), ) else: @@ -371,64 +350,40 @@ def _rename_and_copy_compartments( self, geometry_output_folder, simulation_output_folder ): copyfile( - path.join( - geometry_output_folder, - self._geometry_base_naming - + "_mergedBundlesMaps.{}".format(self._extension), - ), - path.join( - simulation_output_folder, - "{}_simulation.ffp_VOLUME{}.{}".format( - self._base_naming, self._compartment_ids[0], self._extension - ), - ), + geometry_output_folder / (self._geometry_base_naming + "_mergedBundlesMaps.{}".format(self._extension)), + simulation_output_folder / + "{}_simulation.ffp_VOLUME{}.{}".format(self._base_naming, self._compartment_ids[0], self._extension), ) if self._number_of_maps > 1: merged_maps = exists( - path.join( - geometry_output_folder, - self._geometry_base_naming - + "_mergedEllipsesMaps.{}".format(self._extension), - ) + geometry_output_folder / (self._geometry_base_naming + "_mergedEllipsesMaps.{}".format(self._extension)) ) base_map = (not merged_maps) and exists( - path.join( - geometry_output_folder, - self._geometry_base_naming - + "_ellipsoid1_cmap.{}".format(self._extension), - ) + geometry_output_folder / (self._geometry_base_naming + "_ellipsoid1_cmap.{}".format(self._extension)) ) if merged_maps: copyfile( - path.join( - geometry_output_folder, - self._geometry_base_naming - + "_mergedEllipsesMaps.{}".format(self._extension), + geometry_output_folder / ( + self._geometry_base_naming + "_mergedEllipsesMaps.{}".format(self._extension) ), - path.join( - simulation_output_folder, - "{}_simulation.ffp_VOLUME{}.{}".format( - self._base_naming, - self._compartment_ids[1], - self._extension, - ), + simulation_output_folder / + "{}_simulation.ffp_VOLUME{}.{}".format( + self._base_naming, + self._compartment_ids[1], + self._extension, ), ) elif base_map: copyfile( - path.join( - geometry_output_folder, - self._geometry_base_naming - + "_ellipsoid0_cmap.{}".format(self._extension), + geometry_output_folder / ( + self._geometry_base_naming + "_ellipsoid0_cmap.{}".format(self._extension) ), - path.join( - simulation_output_folder, - "{}_simulation.ffp_VOLUME{}.{}".format( - self._base_naming, - self._compartment_ids[1], - self._extension, - ), + simulation_output_folder / + "{}_simulation.ffp_VOLUME{}.{}".format( + self._base_naming, + self._compartment_ids[1], + self._extension, ), ) else: diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index 543250d..6974f8a 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -135,16 +135,12 @@ def generate_phantom( output_folder / "phantom" ) - phantom_def = os.path.join( - phantom_infos["file_path"], phantom_infos["base_file"] - ) + phantom_def = phantom_infos["file_path"] / phantom_infos["base_file"] resolution = ",".join([str(r) for r in phantom_infos["resolution"]]) spacing = ",".join([str(s) for s in phantom_infos["spacing"]]) fiber_fraction = "rel" if relative_fiber_fraction else "abs" - out_name = os.path.join( - output_folder, "phantom, " "{}_phantom".format(run_name) - ) + out_name = output_folder / "phantom, " / "{}_phantom".format(run_name) arguments = "-f {} -r {} -s {} -o {} --comp-map {} --quiet".format( phantom_def, resolution, spacing, out_name, fiber_fraction @@ -184,11 +180,10 @@ def simulate_diffusion_mri( bind_paths += [str(simulation_infos["file_path"]), str(output_folder)] bind_paths = ",".join(bind_paths) - ffp_file = os.path.join( - simulation_infos["file_path"], simulation_infos["param_file"] - ) + ffp_file = simulation_infos["file_path"] / simulation_infos["param_file"] + extension = "nii.gz" if output_nifti else "nrrd" - out_name = os.path.join(output_folder, "{}.{}".format(name, extension)) + out_name = output_folder / "{}.{}".format(name, extension) if not compartments_staged and compartment_maps is not None: datastore = Datastore( From b34077285252486d46c931171ec270e47a98dd75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 20:32:51 -0400 Subject: [PATCH 049/100] Removed all usage of `os` in simulation runner --- simulator/runner/simulation_runner.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index 6974f8a..2408110 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -1,8 +1,7 @@ import logging +import pathlib from asyncio import get_event_loop, new_event_loop, set_event_loop -from os import makedirs -import os from subprocess import PIPE, Popen from .config import SingularityConfig @@ -65,11 +64,9 @@ def _bind_singularity(self, step, paths, arguments): ) @staticmethod - def _create_outputs(path): - if not os.path.exists(path): - makedirs(path, exist_ok=True) - - return path + def _create_outputs(path: pathlib.Path): + path.mkdir(parents=True, exist_ok=True) + return path.resolve(strict=True) def run( self, From a1b1508d4fb587546d1988c72f8c4042fac7d2ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 20:35:20 -0400 Subject: [PATCH 050/100] Integrated pathlib into documentation --- simulator/utils/setup/documentation.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/simulator/utils/setup/documentation.py b/simulator/utils/setup/documentation.py index 58432b9..ee32973 100755 --- a/simulator/utils/setup/documentation.py +++ b/simulator/utils/setup/documentation.py @@ -1,5 +1,5 @@ import os -from os.path import join +import pathlib from setuptools import Command @@ -9,9 +9,7 @@ class DocCommand(Command): user_options = [ ("out-type=", "o", "Output documentation type, default : html") ] - script_dir = join( - "documentation", "_cache", "bat" if os.name == "nt" else "bash" - ) + script_dir: pathlib.Path = pathlib.Path("documentation") / "_cache" / ("bat" if os.name == "nt" else "bash") script_ext = "bat" if os.name == "nt" else "sh" def initialize_options(self): @@ -22,13 +20,13 @@ def finalize_options(self): def run(self): self._run_command( - [join(self.script_dir, "generate_doc.{}".format(self.script_ext))], + [self.script_dir / "generate_doc.{}".format(self.script_ext)], "Generating automatic documentation", ) self._run_command( [ - join(self.script_dir, "build_doc.{}".format(self.script_ext)), + self.script_dir / "build_doc.{}".format(self.script_ext), self.out_type, ], "Building documentation", From aafb90e33d06f878d3473b9e61d40d87a123f33d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 20:40:24 -0400 Subject: [PATCH 051/100] Replaced `exists` usage in legacy runner --- simulator/runner/legacy.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/simulator/runner/legacy.py b/simulator/runner/legacy.py index 25bd526..455a299 100644 --- a/simulator/runner/legacy.py +++ b/simulator/runner/legacy.py @@ -2,7 +2,6 @@ import logging import pathlib from multiprocessing import Process -from os.path import basename, exists from shutil import copyfile from subprocess import PIPE, Popen @@ -285,14 +284,14 @@ def _rename_and_copy_compartments_standalone( ) if len(simulation_infos["compartment_ids"]) > 1: - merged_maps = exists( - geometry_output_folder / (self._geometry_base_naming + "_mergedEllipsesMaps.{}".format(self._extension)) - ) - base_map = (not merged_maps) and exists( - geometry_output_folder / ( - self._geometry_base_naming + "_ellipsoid{}_cmap.{}".format(0, self._extension) - ) - ) + merged_maps = ( + geometry_output_folder / ( + self._geometry_base_naming + "_mergedEllipsesMaps.{}".format(self._extension)) + ).exists() + base_map = (not merged_maps) and ( + geometry_output_folder / ( + self._geometry_base_naming + "_ellipsoid{}_cmap.{}".format(0, self._extension)) + ).exists() if merged_maps: copyfile( geometry_output_folder / ( @@ -356,12 +355,14 @@ def _rename_and_copy_compartments( ) if self._number_of_maps > 1: - merged_maps = exists( - geometry_output_folder / (self._geometry_base_naming + "_mergedEllipsesMaps.{}".format(self._extension)) - ) - base_map = (not merged_maps) and exists( - geometry_output_folder / (self._geometry_base_naming + "_ellipsoid1_cmap.{}".format(self._extension)) - ) + merged_maps = ( + geometry_output_folder / ( + self._geometry_base_naming + "_mergedEllipsesMaps.{}".format(self._extension)) + ).exists() + base_map = (not merged_maps) and ( + geometry_output_folder / ( + self._geometry_base_naming + "_ellipsoid1_cmap.{}".format(self._extension)) + ).exists() if merged_maps: copyfile( geometry_output_folder / ( From 0f2f49d1fed462ddfdf806ac509248a919d1a975 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 20:41:39 -0400 Subject: [PATCH 052/100] Format --- simulator/runner/datastore.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/simulator/runner/datastore.py b/simulator/runner/datastore.py index 47fb18d..1ecefac 100644 --- a/simulator/runner/datastore.py +++ b/simulator/runner/datastore.py @@ -10,11 +10,11 @@ class Datastore: def __init__( - self, - simulation_path, - fibers, - compartment_ids, - inter_axonal_fraction=None, + self, + simulation_path, + fibers, + compartment_ids, + inter_axonal_fraction=None, ): self.fibers = fibers self.compartments = [] @@ -62,8 +62,8 @@ def load_compartments(self, input_folder, run_name, use_nifti=True): self.add_compartment("generate") assert ( - len(list(filter(lambda c: c == "generate", self.compartments))) - <= 1 + len(list(filter(lambda c: c == "generate", self.compartments))) + <= 1 ) if "generate" in self.compartments: From fde952c548745136021b1881277bd50caf8fd7f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 21:07:38 -0400 Subject: [PATCH 053/100] Fixed bad join due to type! --- simulator/runner/datastore.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/simulator/runner/datastore.py b/simulator/runner/datastore.py index 1ecefac..398abfe 100644 --- a/simulator/runner/datastore.py +++ b/simulator/runner/datastore.py @@ -1,4 +1,5 @@ from os.path import basename, exists, join +import pathlib from shutil import copyfile from tempfile import TemporaryDirectory @@ -101,10 +102,10 @@ def generate_inter_axonal_fraction(self, run_name, fiber_fraction): ) self.add_compartment( - join(self._get_temp(), "{}_intra.nii.gz".format(run_name)) + join(self._get_temp_path(), "{}_intra.nii.gz".format(run_name)) ) self.add_compartment( - join(self._get_temp(), "{}_inter.nii.gz".format(run_name)) + join(self._get_temp_path(), "{}_inter.nii.gz".format(run_name)) ) def generate_extra_axonal_fraction(self, run_name): @@ -122,13 +123,16 @@ def generate_extra_axonal_fraction(self, run_name): nib.save( nib.Nifti1Image(extra, ref.affine, ref.header), - join(self._get_temp(), "{}_extra.nii.gz".format(run_name)), + join(self._get_temp_path(), "{}_extra.nii.gz".format(run_name)), ) self.add_compartment( - join(self._get_temp(), "{}_extra.nii.gz".format(run_name)) + join(self._get_temp_path(), "{}_extra.nii.gz".format(run_name)) ) def _get_temp(self): assert self._temp is not None return self._temp + + def _get_temp_path(self) -> pathlib.Path: + return pathlib.Path(self._get_temp().name) \ No newline at end of file From 05c2c5a49221abec7ae9886cda4e3b9995469463 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 21:07:38 -0400 Subject: [PATCH 054/100] Fixed bad join due to type! --- simulator/runner/datastore.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/simulator/runner/datastore.py b/simulator/runner/datastore.py index 47fb18d..f063120 100644 --- a/simulator/runner/datastore.py +++ b/simulator/runner/datastore.py @@ -1,4 +1,5 @@ from os.path import basename, exists, join +import pathlib from shutil import copyfile from tempfile import TemporaryDirectory @@ -101,10 +102,10 @@ def generate_inter_axonal_fraction(self, run_name, fiber_fraction): ) self.add_compartment( - join(self._get_temp(), "{}_intra.nii.gz".format(run_name)) + join(self._get_temp_path(), "{}_intra.nii.gz".format(run_name)) ) self.add_compartment( - join(self._get_temp(), "{}_inter.nii.gz".format(run_name)) + join(self._get_temp_path(), "{}_inter.nii.gz".format(run_name)) ) def generate_extra_axonal_fraction(self, run_name): @@ -122,13 +123,16 @@ def generate_extra_axonal_fraction(self, run_name): nib.save( nib.Nifti1Image(extra, ref.affine, ref.header), - join(self._get_temp(), "{}_extra.nii.gz".format(run_name)), + join(self._get_temp_path(), "{}_extra.nii.gz".format(run_name)), ) self.add_compartment( - join(self._get_temp(), "{}_extra.nii.gz".format(run_name)) + join(self._get_temp_path(), "{}_extra.nii.gz".format(run_name)) ) def _get_temp(self): assert self._temp is not None return self._temp + + def _get_temp_path(self) -> pathlib.Path: + return pathlib.Path(self._get_temp().name) \ No newline at end of file From 671abb5f40b24f1ec9ce19e26530e41112486c3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 21:08:42 -0400 Subject: [PATCH 055/100] Bumped version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0bf741d..f12a4d9 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ if __name__ == "__main__": setup( name="simulation_generator", - version="1.0.4", + version="1.0.5", packages=find_packages(exclude=("tests", "tests.*")), url="", license="", From 556618d38fc0e51ea829c6c1102187b14fadcdd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 21:10:45 -0400 Subject: [PATCH 056/100] Replaced `os` usage by `pathlib` in `datastore` --- simulator/runner/datastore.py | 48 ++++++++++----------------- simulator/runner/simulation_runner.py | 4 +-- 2 files changed, 19 insertions(+), 33 deletions(-) diff --git a/simulator/runner/datastore.py b/simulator/runner/datastore.py index 398abfe..b381811 100644 --- a/simulator/runner/datastore.py +++ b/simulator/runner/datastore.py @@ -1,4 +1,3 @@ -from os.path import basename, exists, join import pathlib from shutil import copyfile from tempfile import TemporaryDirectory @@ -33,12 +32,9 @@ def get_bind_paths(self, bind_compartments=True): self.compartments if bind_compartments else list() ) - def load_compartments(self, input_folder, run_name, use_nifti=True): + def load_compartments(self, input_folder: pathlib.Path, run_name, use_nifti=True): extension = "nii.gz" if use_nifti else "nrrd" - fiber_fraction = join( - input_folder, - "{}_phantom_mergedBundlesMaps.{}".format(run_name, extension), - ) + fiber_fraction = input_folder / "{}_phantom_mergedBundlesMaps.{}".format(run_name, extension) inter_id = SimulationFactory.CompartmentType.INTER_AXONAL.value extra1_id = SimulationFactory.CompartmentType.EXTRA_AXONAL_1.value @@ -51,11 +47,8 @@ def load_compartments(self, input_folder, run_name, use_nifti=True): self.add_compartment(fiber_fraction) if extra1_id in self.ids or extra2_id in self.ids: - ellipses = join( - input_folder, - "{}_mergedEllipsesMaps.{}".format(run_name, extension), - ) - if exists(ellipses): + ellipses: pathlib.Path = input_folder / "{}_mergedEllipsesMaps.{}".format(run_name, extension) + if ellipses.exists(): self.add_compartment(ellipses) if extra1_id in self.ids and extra2_id in self.ids: self.add_compartment("generate") @@ -74,19 +67,18 @@ def add_compartment(self, filepath): self.compartments.append(filepath) def stage_compartments(self, run_name): - extension = ".".join(basename(self.compartments[0]).split(".")[1:]) + extension = pathlib.Path() / ( + pathlib.PurePath("").joinpath( + *pathlib.PurePath(self.compartments[0]).name.split(".")[1:] + ) + ) for m, cmp_id in zip(self.compartments, self.ids): copyfile( m, - join( - self.stage_path, - "{}_simulation.ffp_VOLUME{}.{}".format( - run_name, cmp_id, extension - ), - ), + self.stage_path / "{}_simulation.ffp_VOLUME{}.{}".format(run_name, cmp_id, extension), ) - def generate_inter_axonal_fraction(self, run_name, fiber_fraction): + def generate_inter_axonal_fraction(self, run_name, fiber_fraction: pathlib.Path): img = nib.load(fiber_fraction) fraction = img.get_fdata() inter_fraction = self.iaf * fraction @@ -94,19 +86,15 @@ def generate_inter_axonal_fraction(self, run_name, fiber_fraction): nib.save( nib.Nifti1Image(inter_fraction, img.affine, img.header), - join(self._get_temp(), "{}_inter.nii.gz".format(run_name)), + self._get_temp_path() / "{}_inter.nii.gz".format(run_name), ) nib.save( nib.Nifti1Image(intra_fraction, img.affine, img.header), - join(self._get_temp(), "{}_intra.nii.gz".format(run_name)), + self._get_temp_path() / "{}_intra.nii.gz".format(run_name), ) - self.add_compartment( - join(self._get_temp_path(), "{}_intra.nii.gz".format(run_name)) - ) - self.add_compartment( - join(self._get_temp_path(), "{}_inter.nii.gz".format(run_name)) - ) + self.add_compartment(self._get_temp_path() / "{}_intra.nii.gz".format(run_name)) + self.add_compartment(self._get_temp_path() / "{}_inter.nii.gz".format(run_name)) def generate_extra_axonal_fraction(self, run_name): other_fractions = list( @@ -123,11 +111,11 @@ def generate_extra_axonal_fraction(self, run_name): nib.save( nib.Nifti1Image(extra, ref.affine, ref.header), - join(self._get_temp_path(), "{}_extra.nii.gz".format(run_name)), + self._get_temp_path() / "{}_extra.nii.gz".format(run_name), ) self.add_compartment( - join(self._get_temp_path(), "{}_extra.nii.gz".format(run_name)) + self._get_temp_path() / "{}_extra.nii.gz".format(run_name) ) def _get_temp(self): @@ -135,4 +123,4 @@ def _get_temp(self): return self._temp def _get_temp_path(self) -> pathlib.Path: - return pathlib.Path(self._get_temp().name) \ No newline at end of file + return pathlib.Path(self._get_temp().name) diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index 2408110..641391a 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -96,9 +96,7 @@ def run( inter_axonal_fraction, ) - datastore.load_compartments( - output_folder / "phantom", run_name, output_nifti - ) + datastore.load_compartments(output_folder / "phantom", run_name, output_nifti) datastore.stage_compartments(run_name) self.simulate_diffusion_mri( From 3d4cb672b69d70f8fef71925d3e34025e6de7df1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 21:15:40 -0400 Subject: [PATCH 057/100] Replaced `os` usage with`path` in simulation handler --- .../simulation_factory/handlers/simulation_handler.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/simulator/factory/simulation_factory/handlers/simulation_handler.py b/simulator/factory/simulation_factory/handlers/simulation_handler.py index 36dc582..5e9d135 100755 --- a/simulator/factory/simulation_factory/handlers/simulation_handler.py +++ b/simulator/factory/simulation_factory/handlers/simulation_handler.py @@ -1,5 +1,6 @@ +import pathlib + from lxml.etree import Element, SubElement, tostring -from os import makedirs, path from simulator.factory.simulation_factory.parameters import ( AcquisitionProfile, @@ -70,10 +71,9 @@ def add_compartment(self, compartment): return self def generate_xml_configuration_file( - self, output_naming, simulation_path="" + self, output_naming: str, simulation_path: pathlib.Path = pathlib.Path() ): - if not path.exists(simulation_path): - makedirs(simulation_path, exist_ok=True) + simulation_path.mkdir(parents=True, exist_ok=True) data = Element("fiberfox") image_element = SubElement(data, "image") @@ -87,7 +87,7 @@ def generate_xml_configuration_file( xml_string = tostring(data, pretty_print=True).decode("utf-8") with open( - simulation_path / (output_naming + ".ffp"), "w+" + simulation_path / (output_naming + ".ffp"), "w+" ) as f: f.write(xml_string) From 0755582d43cec9eb3b07241bd94abad5e57fb9b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 21:25:14 -0400 Subject: [PATCH 058/100] Added more path type hints --- simulator/runner/datastore.py | 8 +++---- simulator/runner/simulation_runner.py | 30 +++++++++++++-------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/simulator/runner/datastore.py b/simulator/runner/datastore.py index b381811..3afead8 100644 --- a/simulator/runner/datastore.py +++ b/simulator/runner/datastore.py @@ -11,15 +11,15 @@ class Datastore: def __init__( self, - simulation_path, - fibers, + simulation_path: pathlib.Path, + fibers: pathlib.Path, compartment_ids, inter_axonal_fraction=None, ): - self.fibers = fibers + self.fibers: pathlib.Path = fibers self.compartments = [] self.ids = compartment_ids - self.stage_path = simulation_path + self.stage_path: pathlib.Path = simulation_path self.iaf = inter_axonal_fraction self._temp = TemporaryDirectory() diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index 641391a..382344e 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -70,10 +70,10 @@ def _create_outputs(path: pathlib.Path): def run( self, - run_name, + run_name: str, phantom_infos, simulation_infos, - output_folder, + output_folder: pathlib.Path, output_nifti=True, relative_fiber_fraction=True, inter_axonal_fraction=None, @@ -115,9 +115,9 @@ def run( def generate_phantom( self, - run_name, + run_name: str, phantom_infos, - output_folder, + output_folder: pathlib.Path, relative_fiber_fraction=True, output_nifti=True, loop_managed=False, @@ -125,17 +125,17 @@ def generate_phantom( loop_managed or self.start() - base_output_folder = output_folder - output_folder = self._create_outputs( + base_output_folder: pathlib.Path = output_folder + output_folder: pathlib.Path = self._create_outputs( output_folder / "phantom" ) - phantom_def = phantom_infos["file_path"] / phantom_infos["base_file"] + phantom_def: pathlib.Path = phantom_infos["file_path"] / phantom_infos["base_file"] resolution = ",".join([str(r) for r in phantom_infos["resolution"]]) spacing = ",".join([str(s) for s in phantom_infos["spacing"]]) fiber_fraction = "rel" if relative_fiber_fraction else "abs" - out_name = output_folder / "phantom, " / "{}_phantom".format(run_name) + out_name: pathlib.Path = output_folder / "phantom, " / "{}_phantom".format(run_name) arguments = "-f {} -r {} -s {} -o {} --comp-map {} --quiet".format( phantom_def, resolution, spacing, out_name, fiber_fraction @@ -146,17 +146,17 @@ def generate_phantom( bind_paths = ",".join([str(phantom_infos["file_path"]), str(output_folder)]) command = self._bind_singularity("phantom", bind_paths, arguments) - log_file = base_output_folder / "{}.log".format(run_name) + log_file: pathlib.Path = base_output_folder / "{}.log".format(run_name) self._run_command(command, log_file, "[PHANTOM]") loop_managed or self.stop() def simulate_diffusion_mri( self, - run_name, + run_name: str, simulation_infos, - output_folder, - fibers_file, + output_folder: pathlib.Path, + fibers_file: pathlib.Path, compartment_maps=None, bind_paths=None, output_nifti=True, @@ -166,8 +166,8 @@ def simulate_diffusion_mri( loop_managed or self.start() bind_paths = [] if bind_paths is None else bind_paths - base_output_folder = output_folder - output_folder = self._create_outputs( + base_output_folder: pathlib.Path = output_folder + output_folder: pathlib.Path = self._create_outputs( output_folder / "simulation" ) @@ -193,7 +193,7 @@ def simulate_diffusion_mri( arguments = "-p {} -i {} -o {}".format(ffp_file, fibers_file, out_name) command = self._bind_singularity("diffusion mri", bind_paths, arguments) - log_file = base_output_folder / "{}.log".format(run_name) + log_file: pathlib.Path = base_output_folder / "{}.log".format(run_name) self._run_command(command, log_file, "[DIFFUSION MRI]") loop_managed or self.stop() From 15f77e6482ee79fbde5fa11f0218f1d7f641d1e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 4 Jul 2022 21:29:49 -0400 Subject: [PATCH 059/100] Added type hints for infos in runners --- simulator/runner/legacy.py | 14 ++++++++++---- simulator/runner/simulation_runner.py | 10 ++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/simulator/runner/legacy.py b/simulator/runner/legacy.py index 455a299..020f69f 100644 --- a/simulator/runner/legacy.py +++ b/simulator/runner/legacy.py @@ -12,6 +12,8 @@ from .config import SingularityConfig from ..exceptions import SimulationRunnerException from ..utils.logging import RTLogging +from ..factory.geometry_factory.handlers import GeometryInfos +from ..factory.simulation_factory.handlers import SimulationInfos _logger = logging.getLogger(__name__) @@ -21,8 +23,8 @@ class SimulationRunner: def __init__( self, base_naming, - geometry_infos, - simulation_infos=None, + geometry_infos: GeometryInfos, + simulation_infos: SimulationInfos = None, singularity_conf=SingularityConfig(), output_nifti=False, ): @@ -59,7 +61,11 @@ def set_geometry_base_naming(self, name: str): self._geometry_base_naming = name def run_simulation_dwimage( - self, output_folder: pathlib.Path, image_file: pathlib.Path, simulation_infos, test_mode=False + self, + output_folder: pathlib.Path, + image_file: pathlib.Path, + simulation_infos: SimulationInfos, + test_mode=False ): self._start_loop_if_closed() @@ -114,7 +120,7 @@ def run_simulation_standalone( self, output_folder: pathlib.Path, geometry_folder: pathlib.Path, - simulation_infos, + simulation_infos: SimulationInfos, base_naming=None, test_mode=False, ): diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index 382344e..4f973da 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -7,6 +7,8 @@ from .config import SingularityConfig from .datastore import Datastore from ..utils.logging import RTLogging +from ..factory.geometry_factory.handlers import GeometryInfos +from ..factory.simulation_factory.handlers import SimulationInfos _logger = logging.getLogger(__name__) @@ -71,8 +73,8 @@ def _create_outputs(path: pathlib.Path): def run( self, run_name: str, - phantom_infos, - simulation_infos, + phantom_infos: GeometryInfos, + simulation_infos: SimulationInfos, output_folder: pathlib.Path, output_nifti=True, relative_fiber_fraction=True, @@ -116,7 +118,7 @@ def run( def generate_phantom( self, run_name: str, - phantom_infos, + phantom_infos: GeometryInfos, output_folder: pathlib.Path, relative_fiber_fraction=True, output_nifti=True, @@ -154,7 +156,7 @@ def generate_phantom( def simulate_diffusion_mri( self, run_name: str, - simulation_infos, + simulation_infos: SimulationInfos, output_folder: pathlib.Path, fibers_file: pathlib.Path, compartment_maps=None, From 52d4bcdbdbfe82ac456ecfb188fa0cc211213503 Mon Sep 17 00:00:00 2001 From: benoit-dubreuil <6900372+benoit-dubreuil@users.noreply.github.com> Date: Tue, 5 Jul 2022 16:55:04 -0400 Subject: [PATCH 060/100] Fixed open path --- simulator/factory/geometry_factory/handlers/geometry_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simulator/factory/geometry_factory/handlers/geometry_handler.py b/simulator/factory/geometry_factory/handlers/geometry_handler.py index fd02773..c6bc9f7 100755 --- a/simulator/factory/geometry_factory/handlers/geometry_handler.py +++ b/simulator/factory/geometry_factory/handlers/geometry_handler.py @@ -91,7 +91,7 @@ def generate_json_configuration_files( raise exc with open( - simulation_path / output_naming / "_base.json", "w+" + simulation_path / (output_naming + "_base.json"), "w+" ) as base_file: world = ConfigBuilder.create_world( len(self.get_resolution()), self.get_resolution() From 5e6976da283d681d409531d3b5862ca67e9be039 Mon Sep 17 00:00:00 2001 From: benoit-dubreuil <6900372+benoit-dubreuil@users.noreply.github.com> Date: Tue, 5 Jul 2022 17:20:35 -0400 Subject: [PATCH 061/100] No need to check args namespace --- scripts/geometry_factory.py | 2 +- scripts/simulation_runner.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/geometry_factory.py b/scripts/geometry_factory.py index e589ada..37f4bc5 100755 --- a/scripts/geometry_factory.py +++ b/scripts/geometry_factory.py @@ -80,7 +80,7 @@ def get_geometry_parameters(output_folder, output_naming): ) args = parser.parse_args() - if "out" in args and args.out: + if args.out: dest = args.out makedirs(args.out, exist_ok=True) else: diff --git a/scripts/simulation_runner.py b/scripts/simulation_runner.py index e00329a..e5adc65 100755 --- a/scripts/simulation_runner.py +++ b/scripts/simulation_runner.py @@ -46,7 +46,7 @@ def run_simulation(output_folder): ) args = parser.parse_args() - if "out" in args and args.out: + if args.out: dest = args.out makedirs(args.out, exist_ok=True) else: From 4aedd0be6dfdcc9c12f3716d7857c2f1a497c6c2 Mon Sep 17 00:00:00 2001 From: benoit-dubreuil <6900372+benoit-dubreuil@users.noreply.github.com> Date: Tue, 5 Jul 2022 17:21:27 -0400 Subject: [PATCH 062/100] Useless params --- scripts/geometry_factory.py | 2 +- scripts/simulation_runner.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/geometry_factory.py b/scripts/geometry_factory.py index 37f4bc5..6f8384e 100755 --- a/scripts/geometry_factory.py +++ b/scripts/geometry_factory.py @@ -76,7 +76,7 @@ def get_geometry_parameters(output_folder, output_naming): if __name__ == "__main__": parser = argparse.ArgumentParser("Geometry Factory Example Script") parser.add_argument( - "--out", type=str, required=False, help="Output directory for the files" + "--out", type=str, help="Output directory for the files" ) args = parser.parse_args() diff --git a/scripts/simulation_runner.py b/scripts/simulation_runner.py index e5adc65..f226258 100755 --- a/scripts/simulation_runner.py +++ b/scripts/simulation_runner.py @@ -42,7 +42,7 @@ def run_simulation(output_folder): if __name__ == "__main__": parser = argparse.ArgumentParser("Simulation Runner Example Script") parser.add_argument( - "--out", type=str, required=False, help="Output directory for the files" + "--out", type=str, help="Output directory for the files" ) args = parser.parse_args() From 83de82520fb379a590ed91039d8b12dcd151145d Mon Sep 17 00:00:00 2001 From: benoit-dubreuil <6900372+benoit-dubreuil@users.noreply.github.com> Date: Tue, 5 Jul 2022 17:27:31 -0400 Subject: [PATCH 063/100] Integrated pathlib into scripts --- scripts/geometry_factory.py | 13 ++++++------- scripts/simulation_runner.py | 15 +++++++-------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/scripts/geometry_factory.py b/scripts/geometry_factory.py index 6f8384e..241d4d5 100755 --- a/scripts/geometry_factory.py +++ b/scripts/geometry_factory.py @@ -1,13 +1,12 @@ #!/usr/bin/env python import argparse +import pathlib from math import pi -from os import makedirs from tempfile import mkdtemp from simulator.factory import GeometryFactory, Plane - resolution = [10, 10, 10] spacing = [2, 2, 2] @@ -42,7 +41,7 @@ ] -def get_geometry_parameters(output_folder, output_naming): +def get_geometry_parameters(output_folder: pathlib.Path, output_naming: str): geometry_handler = GeometryFactory.get_geometry_handler(resolution, spacing) bundle1 = GeometryFactory.create_bundle( @@ -76,15 +75,15 @@ def get_geometry_parameters(output_folder, output_naming): if __name__ == "__main__": parser = argparse.ArgumentParser("Geometry Factory Example Script") parser.add_argument( - "--out", type=str, help="Output directory for the files" + "--out", type=pathlib.Path, help="Output directory for the files" ) args = parser.parse_args() if args.out: - dest = args.out - makedirs(args.out, exist_ok=True) + dest: pathlib.Path = args.out + dest.mkdir(parents=True, exist_ok=True) else: - dest = mkdtemp(prefix="geo_factory") + dest = pathlib.Path(mkdtemp(prefix="geo_factory")) print("Script execution results are in : {}".format(dest)) get_geometry_parameters(dest, "geometry") diff --git a/scripts/simulation_runner.py b/scripts/simulation_runner.py index f226258..1aeff73 100755 --- a/scripts/simulation_runner.py +++ b/scripts/simulation_runner.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 import argparse -from os import makedirs -from os.path import join +import pathlib from tempfile import mkdtemp from scripts.geometry_factory import get_geometry_parameters @@ -10,7 +9,7 @@ from simulator.runner.legacy import SimulationRunner -def run_simulation(output_folder): +def run_simulation(output_folder: pathlib.Path): geometry_parameters = get_geometry_parameters( output_folder, "runner_test_geometry" ) @@ -32,7 +31,7 @@ def run_simulation(output_folder): output_folder, "runner_test_simulation_standalone" ) - standalone_output = join(output_folder, "standalone_test") + standalone_output = output_folder / "standalone_test" runner.run_simulation_standalone( standalone_output, output_folder, simulation_parameters, "standalone" @@ -42,15 +41,15 @@ def run_simulation(output_folder): if __name__ == "__main__": parser = argparse.ArgumentParser("Simulation Runner Example Script") parser.add_argument( - "--out", type=str, help="Output directory for the files" + "--out", type=pathlib.Path, help="Output directory for the files" ) args = parser.parse_args() if args.out: - dest = args.out - makedirs(args.out, exist_ok=True) + dest: pathlib.Path = args.out + dest.mkdir(parents=True, exist_ok=True) else: - dest = mkdtemp(prefix="sim_runner") + dest = pathlib.Path(mkdtemp(prefix="sim_runner")) print("Script execution results are in : {}".format(dest)) run_simulation(dest) From 8f47edf4eb8cc0e5d3f7d0add775ed3ffd177c28 Mon Sep 17 00:00:00 2001 From: benoit-dubreuil <6900372+benoit-dubreuil@users.noreply.github.com> Date: Tue, 5 Jul 2022 17:34:31 -0400 Subject: [PATCH 064/100] Removed `os` usage in `scripts` --- scripts/geometry_factory_multi_cluster.py | 15 ++++++------ scripts/simulation_factory.py | 14 ++++++------ scripts/wall_effect.py | 28 +++++++++++------------ 3 files changed, 27 insertions(+), 30 deletions(-) diff --git a/scripts/geometry_factory_multi_cluster.py b/scripts/geometry_factory_multi_cluster.py index 5004616..c542e20 100755 --- a/scripts/geometry_factory_multi_cluster.py +++ b/scripts/geometry_factory_multi_cluster.py @@ -1,13 +1,12 @@ #!/usr/bin/env python3 import argparse +import pathlib from math import pi -from os import makedirs from tempfile import mkdtemp from simulator.factory import GeometryFactory, Plane - resolution = [10, 10, 10] spacing = [2, 2, 2] @@ -43,7 +42,7 @@ ] -def run_multi_clusters(output_folder, output_naming): +def run_multi_clusters(output_folder: pathlib.Path, output_naming: str): geometry_handler = GeometryFactory.get_geometry_handler(resolution, spacing) bundle1 = GeometryFactory.create_bundle( @@ -87,15 +86,15 @@ def run_multi_clusters(output_folder, output_naming): if __name__ == "__main__": parser = argparse.ArgumentParser("Geometry Multi Clusters Example Script") parser.add_argument( - "--out", type=str, required=False, help="Output directory for the files" + "--out", type=pathlib.Path, help="Output directory for the files" ) args = parser.parse_args() - if "out" in args and args.out: - dest = args.out - makedirs(args.out, exist_ok=True) + if args.out: + dest: pathlib.Path = args.out + dest.mkdir(parents=True, exist_ok=True) else: - dest = mkdtemp(prefix="geo_factory_mc") + dest = pathlib.Path(mkdtemp(prefix="geo_factory_mc")) print("Script execution results are in : {}".format(dest)) run_multi_clusters(dest, "multi_clusters") diff --git a/scripts/simulation_factory.py b/scripts/simulation_factory.py index 433c85f..0d92345 100755 --- a/scripts/simulation_factory.py +++ b/scripts/simulation_factory.py @@ -1,7 +1,7 @@ #!/usr/bin/env python import argparse -from os import makedirs +import pathlib from tempfile import mkdtemp from random import uniform @@ -12,7 +12,7 @@ from simulator.utils.test_helpers import GeometryHelper -def get_simulation_parameters(output_folder, output_naming): +def get_simulation_parameters(output_folder: pathlib.Path, output_naming: str): fiber_compartment = SimulationFactory.generate_fiber_stick_compartment( 0.007, 900, 80, SimulationFactory.CompartmentType.INTRA_AXONAL ) @@ -71,15 +71,15 @@ def get_simulation_parameters(output_folder, output_naming): if __name__ == "__main__": parser = argparse.ArgumentParser("Simulation Factory Example Script") parser.add_argument( - "--out", type=str, required=False, help="Output directory for the files" + "--out", type=pathlib.Path, help="Output directory for the files" ) args = parser.parse_args() - if "out" in args and args.out: - dest = args.out - makedirs(args.out, exist_ok=True) + if args.out: + dest: pathlib.Path = args.out + dest.mkdir(parents=True, exist_ok=True) else: - dest = mkdtemp(prefix="sim_factory") + dest = pathlib.Path(mkdtemp(prefix="sim_factory")) print("Script execution results are in : {}".format(dest)) get_simulation_parameters(dest, "simulation") diff --git a/scripts/wall_effect.py b/scripts/wall_effect.py index f0571b6..bee00e0 100755 --- a/scripts/wall_effect.py +++ b/scripts/wall_effect.py @@ -1,9 +1,8 @@ #!/usr/bin/env python3 import argparse +import pathlib from math import pi, sqrt -from os import makedirs -from os.path import join from tempfile import mkdtemp from numpy import mean @@ -11,7 +10,6 @@ from simulator.factory import GeometryFactory, Plane, SimulationFactory from simulator.runner.legacy import SimulationRunner - resolution = [40, 40, 40] spacing = [1, 1, 1] point_per_centroid = 30 @@ -26,7 +24,7 @@ cluster_center = [0.5, 0.5, 0.5] -def create_geometry(output_folder, output_naming, fibers_per_bundle): +def create_geometry(output_folder: pathlib.Path, output_naming: str, fibers_per_bundle): geometry_handler = GeometryFactory.get_geometry_handler(resolution, spacing) bundle1 = GeometryFactory.create_bundle( @@ -54,7 +52,7 @@ def create_geometry(output_folder, output_naming, fibers_per_bundle): def create_split_geometry( - output_folder, output_naming, fibers_per_bundle, rotation=None + output_folder: pathlib.Path, output_naming: str, fibers_per_bundle, rotation=None ): geometry_handler = GeometryFactory.get_geometry_handler(resolution, spacing) @@ -142,7 +140,7 @@ def create_simulation_b1000(geometry_handler, output_folder, output_naming): def create_simulation_multishell( - geometry_handler, output_folder, output_naming + geometry_handler, output_folder, output_naming ): base_multishell_simulation_handler = get_base_simulation_handler( geometry_handler @@ -166,18 +164,18 @@ def create_simulation_multishell( if __name__ == "__main__": parser = argparse.ArgumentParser("Wall Effect Example Script") parser.add_argument( - "--out", type=str, required=False, help="Output directory for the files" + "--out", type=pathlib.Path, help="Output directory for the files" ) args = parser.parse_args() if "out" in args and args.out: - dest = args.out - makedirs(args.out, exist_ok=True) + dest: pathlib.Path = args.out + dest.mkdir(parents=True, exist_ok=True) else: - dest = mkdtemp(prefix="wall_effect") + dest = pathlib.Path(mkdtemp(prefix="wall_effect")) print("Script execution results are in : {}".format(dest)) - makedirs(join(dest, "runner_outputs"), exist_ok=True) + (dest / "runner_outputs").mkdir(parents=True, exist_ok=True) geometry_infos, geometry_handler = create_geometry( dest, "geometry", n_fibers_per_bundle @@ -189,7 +187,7 @@ def create_simulation_multishell( SimulationRunner( "simulation_b1000", geometry_infos, simulation_b1000_infos - ).run(join(dest, "runner_outputs"), relative_fiber_compartment=False) + ).run(dest / "runner_outputs", relative_fiber_compartment=False) geometry_infos, geometry_handler = create_split_geometry( dest, "geometry_split_b1", 3000 @@ -201,7 +199,7 @@ def create_simulation_multishell( SimulationRunner( "simulation_split_b1_b1000", geometry_infos, simulation_b1000_infos - ).run(join(dest, "runner_outputs")) + ).run(dest / "runner_outputs") geometry_infos, geometry_handler = create_split_geometry( dest, "geometry_split_b2", 3000, pi @@ -213,7 +211,7 @@ def create_simulation_multishell( SimulationRunner( "simulation_split_b2_b1000", geometry_infos, simulation_b1000_infos - ).run(join(dest, "runner_outputs")) + ).run(dest / "runner_outputs") simulation_multishell_infos = create_simulation_multishell( geometry_handler, dest, "simulation_multishell" @@ -221,4 +219,4 @@ def create_simulation_multishell( SimulationRunner( "simulation_multishell", geometry_infos, simulation_b1000_infos - ).run(join(dest, "runner_outputs")) + ).run(dest / "runner_outputs") From 3da34a17cf1742549f5a882873b77d5ba71becbc Mon Sep 17 00:00:00 2001 From: benoit-dubreuil <6900372+benoit-dubreuil@users.noreply.github.com> Date: Tue, 5 Jul 2022 17:51:20 -0400 Subject: [PATCH 065/100] Resolve paths in runners --- simulator/runner/legacy.py | 14 ++++++++++---- simulator/runner/simulation_runner.py | 17 ++++++++--------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/simulator/runner/legacy.py b/simulator/runner/legacy.py index 020f69f..78fbf06 100644 --- a/simulator/runner/legacy.py +++ b/simulator/runner/legacy.py @@ -43,7 +43,7 @@ def __init__( singularity_conf = ( singularity_conf if singularity_conf else SingularityConfig() ) - self._singularity: pathlib.Path = singularity_conf.singularity + self._singularity: pathlib.Path = singularity_conf.singularity.resolve(strict=True) self._singularity_exec: str = singularity_conf.singularity_exec self._run_simulation = True if simulation_infos else False @@ -71,6 +71,7 @@ def run_simulation_dwimage( simulation_output_folder: pathlib.Path = output_folder / "simulation_outputs" simulation_output_folder.mkdir(parents=True, exist_ok=True) + simulation_output_folder = simulation_output_folder.resolve(strict=True) simulation_command = ( "{} run -B {} --app launch_mitk {} -p {} -i {} -o {} {}".format( @@ -130,8 +131,11 @@ def run_simulation_standalone( self._start_loop_if_closed() simulation_output_folder = output_folder / "simulation_outputs" - geometry_output_folder = geometry_folder / "geometry_outputs" simulation_output_folder.mkdir(parents=True, exist_ok=True) + simulation_output_folder = simulation_output_folder.resolve(strict=True) + + geometry_output_folder = geometry_folder / "geometry_outputs" + geometry_output_folder = geometry_output_folder.resolve(strict=True) simulation_command = ( "{} run -B {} --app launch_mitk {} -p {} -i {} -o {} {}".format( @@ -194,10 +198,12 @@ def run( geometry_output_folder: pathlib.Path = output_folder / "geometry_outputs" geometry_output_folder.mkdir(parents=True, exist_ok=True) + geometry_output_folder = geometry_output_folder.resolve(strict=True) if self._run_simulation: simulation_output_folder: pathlib.Path = output_folder / "simulation_outputs" simulation_output_folder.mkdir(parents=True, exist_ok=True) + simulation_output_folder = simulation_output_folder.resolve(strict=True) geometry_command = ( "singularity run -B {} --app launch_voxsim {} -f {} -r {} " @@ -433,8 +439,8 @@ def _save_nrrd(self, data, header, name): def _generate_background_map( self, - geometry_output_folder, - simulation_output_folder, + geometry_output_folder: pathlib.Path, + simulation_output_folder: pathlib.Path, compartment_id, merged_maps=False, base_map=False, diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index 4f973da..eb0b19f 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -51,7 +51,7 @@ class SimulationRunner(AsyncRunner): _apps = {"phantom": "launch_voxsim", "diffusion mri": "launch_mitk"} def __init__(self, singularity_conf=SingularityConfig()): - self._singularity = singularity_conf.singularity + self._singularity = singularity_conf.singularity.resolve(strict=True) super().__init__() self._singularity_exec = singularity_conf.singularity_exec @@ -82,6 +82,8 @@ def run( ): self.start() + output_folder = output_folder.resolve(strict=True) + self.generate_phantom( run_name, phantom_infos, @@ -127,10 +129,8 @@ def generate_phantom( loop_managed or self.start() - base_output_folder: pathlib.Path = output_folder - output_folder: pathlib.Path = self._create_outputs( - output_folder / "phantom" - ) + base_output_folder: pathlib.Path = self._create_outputs(output_folder) + output_folder: pathlib.Path = self._create_outputs(output_folder / "phantom") phantom_def: pathlib.Path = phantom_infos["file_path"] / phantom_infos["base_file"] @@ -168,10 +168,9 @@ def simulate_diffusion_mri( loop_managed or self.start() bind_paths = [] if bind_paths is None else bind_paths - base_output_folder: pathlib.Path = output_folder - output_folder: pathlib.Path = self._create_outputs( - output_folder / "simulation" - ) + fibers_file = fibers_file.resolve(strict=True) + base_output_folder: pathlib.Path = self._create_outputs(output_folder) + output_folder: pathlib.Path = self._create_outputs(output_folder / "simulation") name = "{}_simulation".format(run_name) From ab3bfa2a75f0841e952c395b51a1ddbe4db56e5c Mon Sep 17 00:00:00 2001 From: benoit-dubreuil <6900372+benoit-dubreuil@users.noreply.github.com> Date: Tue, 5 Jul 2022 18:05:14 -0400 Subject: [PATCH 066/100] Fixed phatom output names --- simulator/runner/simulation_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index eb0b19f..27e9330 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -137,7 +137,7 @@ def generate_phantom( resolution = ",".join([str(r) for r in phantom_infos["resolution"]]) spacing = ",".join([str(s) for s in phantom_infos["spacing"]]) fiber_fraction = "rel" if relative_fiber_fraction else "abs" - out_name: pathlib.Path = output_folder / "phantom, " / "{}_phantom".format(run_name) + out_name: pathlib.Path = output_folder / "{}_phantom".format(run_name) arguments = "-f {} -r {} -s {} -o {} --comp-map {} --quiet".format( phantom_def, resolution, spacing, out_name, fiber_fraction From 627a87773e9c7df00bcdd045c6b134cd9c4e5b51 Mon Sep 17 00:00:00 2001 From: benoit-dubreuil <6900372+benoit-dubreuil@users.noreply.github.com> Date: Tue, 5 Jul 2022 18:10:00 -0400 Subject: [PATCH 067/100] Bumped version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f12a4d9..9bf4db0 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ if __name__ == "__main__": setup( name="simulation_generator", - version="1.0.5", + version="1.0.6", packages=find_packages(exclude=("tests", "tests.*")), url="", license="", From 8aeffe48bcc5c7fe3406f2573c560315368e0da5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 18 Jul 2022 11:30:20 -0400 Subject: [PATCH 068/100] Added gitattributes --- .gitattributes | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a2d5f25 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,26 @@ +############################### +# Git Line Endings # +############################### + +# Set default behaviour to normalize line endings to LF. +# Use test=auto to automatically normalize line endings. +* text eol=lf + +# Force batch scripts to always use CRLF line endings so that if a repo is accessed +# in Windows via a file share from Linux, the scripts will work. +*.{cmd,[cC][mM][dD]} text eol=crlf +*.{bat,[bB][aA][tT]} text eol=crlf + +*.sln text eol=crlf + + +# Images +*.gif -text +*.ico -text +*.jpg -text +*.pdf -text +*.png -text +*.tif -text +*.tiff -text +*.psd -text +*.webp -text From 9f96566c9be0dac561f419ca2d51351809130160 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Fri, 22 Jul 2022 21:06:58 -0400 Subject: [PATCH 069/100] Format to respect PEP --- simulator/runner/simulation_runner.py | 50 +++++++++++++-------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index 27e9330..755d8d3 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -71,14 +71,14 @@ def _create_outputs(path: pathlib.Path): return path.resolve(strict=True) def run( - self, - run_name: str, - phantom_infos: GeometryInfos, - simulation_infos: SimulationInfos, - output_folder: pathlib.Path, - output_nifti=True, - relative_fiber_fraction=True, - inter_axonal_fraction=None, + self, + run_name: str, + phantom_infos: GeometryInfos, + simulation_infos: SimulationInfos, + output_folder: pathlib.Path, + output_nifti=True, + relative_fiber_fraction=True, + inter_axonal_fraction=None, ): self.start() @@ -118,13 +118,13 @@ def run( self.stop() def generate_phantom( - self, - run_name: str, - phantom_infos: GeometryInfos, - output_folder: pathlib.Path, - relative_fiber_fraction=True, - output_nifti=True, - loop_managed=False, + self, + run_name: str, + phantom_infos: GeometryInfos, + output_folder: pathlib.Path, + relative_fiber_fraction=True, + output_nifti=True, + loop_managed=False, ): loop_managed or self.start() @@ -154,16 +154,16 @@ def generate_phantom( loop_managed or self.stop() def simulate_diffusion_mri( - self, - run_name: str, - simulation_infos: SimulationInfos, - output_folder: pathlib.Path, - fibers_file: pathlib.Path, - compartment_maps=None, - bind_paths=None, - output_nifti=True, - loop_managed=False, - compartments_staged=True, + self, + run_name: str, + simulation_infos: SimulationInfos, + output_folder: pathlib.Path, + fibers_file: pathlib.Path, + compartment_maps=None, + bind_paths=None, + output_nifti=True, + loop_managed=False, + compartments_staged=True, ): loop_managed or self.start() From 8aab0fc32b15eae73a9c46b2cf629946e2d418e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Fri, 22 Jul 2022 21:20:44 -0400 Subject: [PATCH 070/100] Fixed `except` statement that catches everything... --- simulator/utils/logging.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/simulator/utils/logging.py b/simulator/utils/logging.py index a75803b..43c8d0e 100755 --- a/simulator/utils/logging.py +++ b/simulator/utils/logging.py @@ -1,4 +1,4 @@ -from queue import Queue +import queue from threading import Thread import pathlib import time @@ -20,8 +20,8 @@ def join(self): self._thread.join() def _read_output(self, poll_timer=4, logging_callback=lambda a: None): - stdout_queue = Queue() - stderr_queue = Queue() + stdout_queue = queue.Queue() + stderr_queue = queue.Queue() t1 = Thread( target=self._enqueue_thread_output, args=(self._process.stdout, stdout_queue), @@ -49,15 +49,15 @@ def _read_output(self, poll_timer=4, logging_callback=lambda a: None): logging_callback(self._log) time.sleep(poll_timer) - def _enqueue_thread_output(self, pipe, queue): + def _enqueue_thread_output(self, pipe, output: queue.Queue): while self._process.poll() is None: ln = pipe.readline() - queue.put(ln) + output.put(ln) - def _dequeue_output(self, log_file, queue, tag): + def _dequeue_output(self, log_file, output: queue.Queue, tag): try: - while not queue.empty(): - ln = queue.get_nowait() + while not output.empty(): + ln = output.get_nowait() if ln: log_file.write( "\n".join( @@ -69,5 +69,5 @@ def _dequeue_output(self, log_file, queue, tag): + "\n" ) log_file.flush() - except: + except queue.Empty: pass From 44b7b83c462145a64547d46ebc0a9b57f1daf60a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Fri, 22 Jul 2022 21:26:46 -0400 Subject: [PATCH 071/100] Added TODO --- simulator/utils/logging.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/simulator/utils/logging.py b/simulator/utils/logging.py index 43c8d0e..788eea3 100755 --- a/simulator/utils/logging.py +++ b/simulator/utils/logging.py @@ -4,6 +4,9 @@ import time +# TODO : Replace with QueueListener and QueueHandler from logging +# https://docs.python.org/3.8/library/logging.handlers.html#logging.handlers.QueueHandler + class RTLogging: def __init__(self, process, log_file_path: pathlib.Path, log_tag=""): self._process = process From e0d0e83304b1eb96010b556599cfaa8b74e2ada8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Fri, 22 Jul 2022 21:27:42 -0400 Subject: [PATCH 072/100] WIP --- simulator/runner/simulation_runner.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index 755d8d3..d377fd7 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -1,5 +1,6 @@ import logging import pathlib +import typing from asyncio import get_event_loop, new_event_loop, set_event_loop from subprocess import PIPE, Popen @@ -14,9 +15,11 @@ class AsyncRunner: + _custom_log_handlers: typing.Set[logging.Handler] # TODO : WIP. See ..utils.logging def __init__(self): self._event_loop = new_event_loop() + self._custom_log_handlers = set() # TODO : WIP. See ..utils.logging def start(self): self._start_loop_if_closed() From 69df6d75063bc915835bedf4621bbfa11897201d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Sat, 23 Jul 2022 12:14:06 -0400 Subject: [PATCH 073/100] No need to return log file --- simulator/runner/simulation_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index d377fd7..5c915d1 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -47,7 +47,7 @@ async def _run_async(self, command, log_file, log_tag): _logger.start() _logger.join() - return process.returncode, log_file + return process.returncode class SimulationRunner(AsyncRunner): From 2f07aed0ae3bfeb2fc8f4746002fcef2dda9f7e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Sat, 23 Jul 2022 12:15:13 -0400 Subject: [PATCH 074/100] Typed run cmd and run async --- simulator/runner/simulation_runner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index 5c915d1..2e085fa 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -28,7 +28,7 @@ def stop(self): if not self._event_loop.is_closed(): self._event_loop.close() - def _run_command(self, command, log_file, log_tag): + def _run_command(self, command: str, log_file: pathlib.Path, log_tag: str): set_event_loop(self._event_loop) async_loop = get_event_loop() async_loop.run_until_complete( @@ -40,7 +40,7 @@ def _start_loop_if_closed(self): if self._event_loop.is_closed(): self._event_loop = new_event_loop() - async def _run_async(self, command, log_file, log_tag): + async def _run_async(self, command: str, log_file, log_tag) -> int: process = Popen(command.split(" "), stdout=PIPE, stderr=PIPE) _logger = RTLogging(process, log_file, log_tag) @@ -59,7 +59,7 @@ def __init__(self, singularity_conf=SingularityConfig()): self._singularity_exec = singularity_conf.singularity_exec - def _bind_singularity(self, step, paths, arguments): + def _bind_singularity(self, step, paths, arguments) -> str: return "{} run -B {} --app {} {} {}".format( self._singularity_exec, paths, From 90483e76c9f2c8b2d4f29a8a66836779b4b8ed33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Sat, 23 Jul 2022 12:15:35 -0400 Subject: [PATCH 075/100] Return the process returncode from run cmd --- simulator/runner/simulation_runner.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index 2e085fa..61fccb3 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -28,13 +28,16 @@ def stop(self): if not self._event_loop.is_closed(): self._event_loop.close() - def _run_command(self, command: str, log_file: pathlib.Path, log_tag: str): + def _run_command(self, command: str, log_file: pathlib.Path, log_tag: str) -> int: set_event_loop(self._event_loop) async_loop = get_event_loop() - async_loop.run_until_complete( + + returncode = async_loop.run_until_complete( self._run_async(command, log_file, log_tag) ) + async_loop.close() + return returncode def _start_loop_if_closed(self): if self._event_loop.is_closed(): From dd33bf74a615f2eb293e455631a657ec6de3cb3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Fri, 22 Jul 2022 21:27:42 -0400 Subject: [PATCH 076/100] Revert "WIP" This reverts commit e0d0e83304b1eb96010b556599cfaa8b74e2ada8. --- simulator/runner/simulation_runner.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index 61fccb3..2c243c9 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -1,6 +1,5 @@ import logging import pathlib -import typing from asyncio import get_event_loop, new_event_loop, set_event_loop from subprocess import PIPE, Popen @@ -15,11 +14,9 @@ class AsyncRunner: - _custom_log_handlers: typing.Set[logging.Handler] # TODO : WIP. See ..utils.logging def __init__(self): self._event_loop = new_event_loop() - self._custom_log_handlers = set() # TODO : WIP. See ..utils.logging def start(self): self._start_loop_if_closed() From 38e71db56a0a1e2e3c1638613d9127984d9dc3ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Fri, 22 Jul 2022 21:26:46 -0400 Subject: [PATCH 077/100] Revert "Added TODO" This reverts commit 44b7b83c462145a64547d46ebc0a9b57f1daf60a. --- simulator/utils/logging.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/simulator/utils/logging.py b/simulator/utils/logging.py index 788eea3..43c8d0e 100755 --- a/simulator/utils/logging.py +++ b/simulator/utils/logging.py @@ -4,9 +4,6 @@ import time -# TODO : Replace with QueueListener and QueueHandler from logging -# https://docs.python.org/3.8/library/logging.handlers.html#logging.handlers.QueueHandler - class RTLogging: def __init__(self, process, log_file_path: pathlib.Path, log_tag=""): self._process = process From d3d041c2f3cb110b776164346a9ec9450e6acf26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Sat, 23 Jul 2022 12:19:02 -0400 Subject: [PATCH 078/100] Return the process returncode in generation commands --- simulator/runner/simulation_runner.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index 2c243c9..70ad5ac 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -128,7 +128,7 @@ def generate_phantom( relative_fiber_fraction=True, output_nifti=True, loop_managed=False, - ): + ) -> int: loop_managed or self.start() @@ -152,9 +152,10 @@ def generate_phantom( bind_paths = ",".join([str(phantom_infos["file_path"]), str(output_folder)]) command = self._bind_singularity("phantom", bind_paths, arguments) log_file: pathlib.Path = base_output_folder / "{}.log".format(run_name) - self._run_command(command, log_file, "[PHANTOM]") + returncode = self._run_command(command, log_file, "[PHANTOM]") loop_managed or self.stop() + return returncode def simulate_diffusion_mri( self, @@ -167,7 +168,7 @@ def simulate_diffusion_mri( output_nifti=True, loop_managed=False, compartments_staged=True, - ): + ) -> int: loop_managed or self.start() bind_paths = [] if bind_paths is None else bind_paths @@ -198,6 +199,7 @@ def simulate_diffusion_mri( command = self._bind_singularity("diffusion mri", bind_paths, arguments) log_file: pathlib.Path = base_output_folder / "{}.log".format(run_name) - self._run_command(command, log_file, "[DIFFUSION MRI]") + returncode = self._run_command(command, log_file, "[DIFFUSION MRI]") loop_managed or self.stop() + return returncode From d49f6205ad6b10a46fe469248c9df6c442fe0247 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Sat, 23 Jul 2022 12:21:53 -0400 Subject: [PATCH 079/100] Return returncode from run simulation --- simulator/runner/simulation_runner.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index 70ad5ac..9d0f1b5 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -1,5 +1,6 @@ import logging import pathlib +import typing from asyncio import get_event_loop, new_event_loop, set_event_loop from subprocess import PIPE, Popen @@ -82,12 +83,12 @@ def run( output_nifti=True, relative_fiber_fraction=True, inter_axonal_fraction=None, - ): + ) -> typing.Tuple[int, int]: self.start() output_folder = output_folder.resolve(strict=True) - self.generate_phantom( + phantom_returncode = self.generate_phantom( run_name, phantom_infos, output_folder, @@ -106,7 +107,7 @@ def run( datastore.load_compartments(output_folder / "phantom", run_name, output_nifti) datastore.stage_compartments(run_name) - self.simulate_diffusion_mri( + mri_returncode = self.simulate_diffusion_mri( run_name, simulation_infos, output_folder, @@ -119,6 +120,7 @@ def run( ) self.stop() + return phantom_returncode, mri_returncode def generate_phantom( self, From 41b4cb23c8c57de42264b1904230906504562de6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Sat, 23 Jul 2022 12:23:01 -0400 Subject: [PATCH 080/100] The simulation runner global logger was useless --- simulator/runner/simulation_runner.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/simulator/runner/simulation_runner.py b/simulator/runner/simulation_runner.py index 9d0f1b5..d8eeb17 100755 --- a/simulator/runner/simulation_runner.py +++ b/simulator/runner/simulation_runner.py @@ -11,8 +11,6 @@ from ..factory.geometry_factory.handlers import GeometryInfos from ..factory.simulation_factory.handlers import SimulationInfos -_logger = logging.getLogger(__name__) - class AsyncRunner: @@ -41,7 +39,8 @@ def _start_loop_if_closed(self): if self._event_loop.is_closed(): self._event_loop = new_event_loop() - async def _run_async(self, command: str, log_file, log_tag) -> int: + @staticmethod + async def _run_async(command: str, log_file, log_tag) -> int: process = Popen(command.split(" "), stdout=PIPE, stderr=PIPE) _logger = RTLogging(process, log_file, log_tag) From 3973bde26fef1e7106c98cd622d3dc0f53a34b70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Sat, 23 Jul 2022 16:59:41 -0400 Subject: [PATCH 081/100] Increased version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9bf4db0..ed0df5d 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ if __name__ == "__main__": setup( name="simulation_generator", - version="1.0.6", + version="1.0.7", packages=find_packages(exclude=("tests", "tests.*")), url="", license="", From 7113b90362d0f89dab7b6fb2ed3b31c93da51cc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 25 Jul 2022 16:52:49 -0400 Subject: [PATCH 082/100] Removed `.gitattributes` --- .gitattributes | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index a2d5f25..0000000 --- a/.gitattributes +++ /dev/null @@ -1,26 +0,0 @@ -############################### -# Git Line Endings # -############################### - -# Set default behaviour to normalize line endings to LF. -# Use test=auto to automatically normalize line endings. -* text eol=lf - -# Force batch scripts to always use CRLF line endings so that if a repo is accessed -# in Windows via a file share from Linux, the scripts will work. -*.{cmd,[cC][mM][dD]} text eol=crlf -*.{bat,[bB][aA][tT]} text eol=crlf - -*.sln text eol=crlf - - -# Images -*.gif -text -*.ico -text -*.jpg -text -*.pdf -text -*.png -text -*.tif -text -*.tiff -text -*.psd -text -*.webp -text From d1757580d041612ecfb82159584c75b8a70d1782 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 25 Jul 2022 17:04:42 -0400 Subject: [PATCH 083/100] Completely remove LFS --- .cache/doc/bat/build_doc.bat | 14 +++++++------- .cache/doc/bat/generate_doc.bat | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.cache/doc/bat/build_doc.bat b/.cache/doc/bat/build_doc.bat index 57444ef..5d7a3e7 100644 --- a/.cache/doc/bat/build_doc.bat +++ b/.cache/doc/bat/build_doc.bat @@ -1,7 +1,7 @@ - -echo off -SET pth=%~dp0 -SET pth=%pth:~0, -5% -echo on - -bash %pth%/bash/build_doc.sh %* + +echo off +SET pth=%~dp0 +SET pth=%pth:~0, -5% +echo on + +bash %pth%/bash/build_doc.sh %* diff --git a/.cache/doc/bat/generate_doc.bat b/.cache/doc/bat/generate_doc.bat index ceee36c..cf646a8 100644 --- a/.cache/doc/bat/generate_doc.bat +++ b/.cache/doc/bat/generate_doc.bat @@ -1,7 +1,7 @@ - -echo off -SET pth=%~dp0 -SET pth=%pth:~0, -5% -echo on - -bash %pth%/bash/generate_doc.sh %* + +echo off +SET pth=%~dp0 +SET pth=%pth:~0, -5% +echo on + +bash %pth%/bash/generate_doc.sh %* From 3db3f2364c10121cf580de94abb2cb6c5fb35813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Tue, 16 Aug 2022 11:52:03 -0400 Subject: [PATCH 084/100] Fixed anchors type in doc --- simulator/factory/geometry_factory/_geometry_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simulator/factory/geometry_factory/_geometry_factory.py b/simulator/factory/geometry_factory/_geometry_factory.py index 898bfcc..b68f89b 100644 --- a/simulator/factory/geometry_factory/_geometry_factory.py +++ b/simulator/factory/geometry_factory/_geometry_factory.py @@ -107,7 +107,7 @@ def create_bundle(radius, symmetry, n_point_per_centroid, anchors=list): n_point_per_centroid : int Number of points sampling points along the spline representing the centroid - anchors : list(list(int)) or list, optional + anchors : list(list(float)) or list, optional List of anchor points defining the centroid of the bundle, default : [] From 16ecc587df8c5eaf2da83cc528da16d03aaa34bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Fri, 19 Aug 2022 16:49:37 -0400 Subject: [PATCH 085/100] Fixed print error log path --- simulator/runner/legacy.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/simulator/runner/legacy.py b/simulator/runner/legacy.py index 78fbf06..bd7949d 100644 --- a/simulator/runner/legacy.py +++ b/simulator/runner/legacy.py @@ -97,7 +97,7 @@ def run_simulation_dwimage( log_file: pathlib.Path = output_folder / "{}.log".format(self._base_naming) _logger.info("Simulating DWI signal") - return_code, log = async_loop.run_until_complete( + return_code, _ = async_loop.run_until_complete( self._launch_command( simulation_command, log_file, "[RUNNING FIBERFOX]" ) @@ -107,7 +107,7 @@ def run_simulation_dwimage( "Simulation ended in error", SimulationRunnerException.ExceptionType.Fiberfox, return_code, - (log,), + (str(log_file),), ) _logger.debug( @@ -172,7 +172,7 @@ def run_simulation_standalone( ) _logger.info("Simulating DWI signal") - return_code, log = async_loop.run_until_complete( + return_code, _ = async_loop.run_until_complete( self._launch_command( simulation_command, log_file, "[RUNNING FIBERFOX]" ) @@ -182,7 +182,7 @@ def run_simulation_standalone( "Simulation ended in error", SimulationRunnerException.ExceptionType.Fiberfox, return_code, - (log,), + (str(log_file),), ) _logger.debug( @@ -257,7 +257,7 @@ def run( ) _logger.info("Simulating DWI signal") if self._run_simulation: - return_code, log = async_loop.run_until_complete( + return_code, _ = async_loop.run_until_complete( self._launch_command( simulation_command, log_file, "[RUNNING FIBERFOX]" ) @@ -267,7 +267,7 @@ def run( "Simulation ended in error", SimulationRunnerException.ExceptionType.Fiberfox, return_code, - (log,), + (str(log_file),), ) _logger.debug("Simulation ended with code {}".format(return_code)) From 035e174ecac9cc0ca6ae19e02488a1ee9e2086e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Fri, 19 Aug 2022 16:52:41 -0400 Subject: [PATCH 086/100] Improved print error log path msg --- simulator/runner/legacy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/simulator/runner/legacy.py b/simulator/runner/legacy.py index bd7949d..b040f78 100644 --- a/simulator/runner/legacy.py +++ b/simulator/runner/legacy.py @@ -107,7 +107,7 @@ def run_simulation_dwimage( "Simulation ended in error", SimulationRunnerException.ExceptionType.Fiberfox, return_code, - (str(log_file),), + str(log_file), ) _logger.debug( @@ -182,7 +182,7 @@ def run_simulation_standalone( "Simulation ended in error", SimulationRunnerException.ExceptionType.Fiberfox, return_code, - (str(log_file),), + str(log_file), ) _logger.debug( @@ -267,7 +267,7 @@ def run( "Simulation ended in error", SimulationRunnerException.ExceptionType.Fiberfox, return_code, - (str(log_file),), + str(log_file), ) _logger.debug("Simulation ended with code {}".format(return_code)) From 695ba5c283d535377b0300e2526562e63dd270bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 22 Aug 2022 10:59:32 -0400 Subject: [PATCH 087/100] WIP create script for quick testing --- scripts/quick_simulation_runner.py | 116 +++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 scripts/quick_simulation_runner.py diff --git a/scripts/quick_simulation_runner.py b/scripts/quick_simulation_runner.py new file mode 100644 index 0000000..e9e68a9 --- /dev/null +++ b/scripts/quick_simulation_runner.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 + +import argparse +import pathlib +import random +import tempfile + +import numpy as np +import numpy.linalg + +from scripts.geometry_factory import get_geometry_parameters +from simulator.factory import SimulationFactory +from simulator.runner.legacy import SimulationRunner +from simulator.utils.test_helpers import GeometryHelper + + +def get_simulation_parameters(output_folder: pathlib.Path, output_naming: str): + fiber_compartment = SimulationFactory.generate_fiber_stick_compartment( + 0.007, 900, 80, SimulationFactory.CompartmentType.INTRA_AXONAL + ) + + restricted_fluid_compartment = ( + SimulationFactory.generate_extra_ball_compartment( + 2.0, 4000, 2000, SimulationFactory.CompartmentType.EXTRA_AXONAL_1 + ) + ) + + csf_compartment = SimulationFactory.generate_extra_ball_compartment( + 3.0, 4000, 2000, SimulationFactory.CompartmentType.EXTRA_AXONAL_2 + ) + + simulation_handler = SimulationFactory.get_simulation_handler( + GeometryHelper.get_dummy_empty_geometry_handler(), + [fiber_compartment, restricted_fluid_compartment, csf_compartment], + ) + + simulation_handler.set_acquisition_profile( + SimulationFactory.generate_acquisition_profile(1, 1, 1, inhomogen_time=1, echo_train_length=1) + ) + + noise_artifact = SimulationFactory.generate_noise_model("gaussian", 30) + motion_artifact = SimulationFactory.generate_motion_model( + True, "random", [3.1415 / 6, 0, 0], [4, 0, 0] + ) + + simulation_handler.set_artifact_model( + SimulationFactory.generate_artifact_model( + noise_artifact, motion_artifact + ) + ) + + normalize = lambda a: (np.array(a) / np.linalg.norm(a)).tolist() + + simulation_handler.set_gradient_profile( + SimulationFactory.generate_gradient_profile( + [500 for i in range(9)] + + [1000 for i in range(10)] + + [2000 for i in range(10)], + [ + normalize([random.uniform(-1, 1), random.uniform(-1, 1), random.uniform(-1, 1)]) + for i in range(30) + ], + 1, + g_type=SimulationFactory.AcquisitionType.STEJSKAL_TANNER, + ) + ) + + return simulation_handler.generate_xml_configuration_file( + output_naming, output_folder + ) + + +def run_simulation(output_folder: pathlib.Path): + geometry_parameters = get_geometry_parameters( + output_folder, "runner_test_geometry" + ) + + simulation_parameters = get_simulation_parameters( + output_folder, "runner_test_simulation" + ) + + runner = SimulationRunner( + "runner_test", + geometry_parameters, + simulation_parameters, + output_nifti=False, + ) + + runner.run(output_folder, True) + + simulation_parameters = get_simulation_parameters( + output_folder, "runner_test_simulation_standalone" + ) + + standalone_output = output_folder / "standalone_test" + + runner.run_simulation_standalone( + standalone_output, output_folder, simulation_parameters, "standalone" + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser("Simulation Runner Example Script") + parser.add_argument( + "--out", type=pathlib.Path, help="Output directory for the files" + ) + + args = parser.parse_args() + if args.out: + dest: pathlib.Path = args.out + dest.mkdir(parents=True, exist_ok=True) + else: + dest = pathlib.Path(tempfile.mkdtemp(prefix="sim_runner")) + + print("Script execution results are in : {}".format(dest)) + run_simulation(dest) From ecb714508321b23571b7868fbec7ac0bfcd0402a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 22 Aug 2022 11:07:43 -0400 Subject: [PATCH 088/100] Added quick geometry generation to test script --- scripts/quick_simulation_runner.py | 68 +++++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 2 deletions(-) diff --git a/scripts/quick_simulation_runner.py b/scripts/quick_simulation_runner.py index e9e68a9..a0911fe 100644 --- a/scripts/quick_simulation_runner.py +++ b/scripts/quick_simulation_runner.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import argparse +import math import pathlib import random import tempfile @@ -8,11 +9,74 @@ import numpy as np import numpy.linalg -from scripts.geometry_factory import get_geometry_parameters -from simulator.factory import SimulationFactory +from simulator.factory import GeometryFactory, Plane, SimulationFactory from simulator.runner.legacy import SimulationRunner from simulator.utils.test_helpers import GeometryHelper +resolution = [1, 1, 1] +spacing = [1, 1, 1] + +spheres_center = [[-2, 7, 10], [2, -1, 11]] +sphere_radius = 5 + +n_point_per_centroid = 5 +bundle_radius = 4 +bundle_symmetry = 1 +bundle_n_fibers = 200 +bundle_limits = [[0, 1], [0, 1], [0, 1]] +bundle_center = [0.5, 0.5, 0.5] +world_center = [5, 5, 5] + +base_anchors = [ + [0.5, -0.3, 0.5], + [0.5, -0.2, 0.5], + [0.5, -0.1, 0.5], + [0.5, 0, 0.5], + [0.5, 0.1, 0.5], + [0.5, 0.2, 0.5], + [0.5, 0.3, 0.5], + [0.5, 0.4, 0.5], + [0.5, 0.5, 0.5], + [0.5, 0.6, 0.5], + [0.5, 0.7, 0.5], + [0.5, 0.8, 0.5], + [0.5, 0.9, 0.5], + [0.5, 1.1, 0.5], + [0.5, 1.2, 0.5], + [0.5, 1.3, 0.5], +] + + +def get_geometry_parameters(output_folder: pathlib.Path, output_naming: str): + geometry_handler = GeometryFactory.get_geometry_handler(resolution, spacing) + + bundle1 = GeometryFactory.create_bundle( + bundle_radius, bundle_symmetry, n_point_per_centroid, base_anchors + ) + _, bundle2 = GeometryFactory.rotate_bundle( + bundle1, [0.5, 0.5, 0.5], math.pi / 6.0, Plane.YZ + ) + + cluster = GeometryFactory.create_cluster( + GeometryFactory.create_cluster_meta( + 3, bundle_n_fibers, 1, bundle_center, bundle_limits + ), + [bundle1, bundle2], + world_center, + ) + + geometry_handler.add_cluster(cluster) + + sphere_1 = GeometryFactory.create_sphere(sphere_radius, spheres_center[0]) + sphere_2 = GeometryFactory.create_sphere(sphere_radius, spheres_center[1]) + + geometry_handler.add_sphere(sphere_1) + geometry_handler.add_sphere(sphere_2) + + return geometry_handler.generate_json_configuration_files( + output_naming, output_folder + ) + def get_simulation_parameters(output_folder: pathlib.Path, output_naming: str): fiber_compartment = SimulationFactory.generate_fiber_stick_compartment( From c7bb064db0489adf8b705f3e15c67ffe824629f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 22 Aug 2022 11:38:19 -0400 Subject: [PATCH 089/100] Fixed path absolute path problems of VoxSim by resolving every path --- .../handlers/simulation_handler.py | 1 + simulator/runner/legacy.py | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/simulator/factory/simulation_factory/handlers/simulation_handler.py b/simulator/factory/simulation_factory/handlers/simulation_handler.py index 5e9d135..428486f 100755 --- a/simulator/factory/simulation_factory/handlers/simulation_handler.py +++ b/simulator/factory/simulation_factory/handlers/simulation_handler.py @@ -74,6 +74,7 @@ def generate_xml_configuration_file( self, output_naming: str, simulation_path: pathlib.Path = pathlib.Path() ): simulation_path.mkdir(parents=True, exist_ok=True) + simulation_path = simulation_path.resolve(strict=True) data = Element("fiberfox") image_element = SubElement(data, "image") diff --git a/simulator/runner/legacy.py b/simulator/runner/legacy.py index b040f78..ec05b53 100644 --- a/simulator/runner/legacy.py +++ b/simulator/runner/legacy.py @@ -28,7 +28,7 @@ def __init__( singularity_conf=SingularityConfig(), output_nifti=False, ): - self._geometry_path: pathlib.Path = geometry_infos["file_path"] + self._geometry_path: pathlib.Path = geometry_infos["file_path"].resolve(strict=True) self._geometry_base_file: str = geometry_infos["base_file"] self._geometry_resolution = geometry_infos["resolution"] self._geometry_spacing = geometry_infos["spacing"] @@ -36,7 +36,7 @@ def __init__( self._geometry_base_naming: str = base_naming if simulation_infos: self._number_of_maps = len(simulation_infos["compartment_ids"]) - self._simulation_path: pathlib.Path = simulation_infos["file_path"] + self._simulation_path: pathlib.Path = simulation_infos["file_path"].resolve(strict=True) self._simulation_parameters: str = simulation_infos["param_file"] self._compartment_ids = simulation_infos["compartment_ids"] @@ -73,6 +73,8 @@ def run_simulation_dwimage( simulation_output_folder.mkdir(parents=True, exist_ok=True) simulation_output_folder = simulation_output_folder.resolve(strict=True) + output_folder = output_folder.resolve(strict=True) + simulation_command = ( "{} run -B {} --app launch_mitk {} -p {} -i {} -o {} {}".format( self._singularity_exec, @@ -134,6 +136,8 @@ def run_simulation_standalone( simulation_output_folder.mkdir(parents=True, exist_ok=True) simulation_output_folder = simulation_output_folder.resolve(strict=True) + output_folder = output_folder.resolve(strict=True) + geometry_output_folder = geometry_folder / "geometry_outputs" geometry_output_folder = geometry_output_folder.resolve(strict=True) @@ -200,6 +204,8 @@ def run( geometry_output_folder.mkdir(parents=True, exist_ok=True) geometry_output_folder = geometry_output_folder.resolve(strict=True) + output_folder = output_folder.resolve(strict=True) + if self._run_simulation: simulation_output_folder: pathlib.Path = output_folder / "simulation_outputs" simulation_output_folder.mkdir(parents=True, exist_ok=True) @@ -285,6 +291,8 @@ def _rename_and_copy_compartments_standalone( simulation_output_folder: pathlib.Path, base_naming, ): + geometry_output_folder = geometry_output_folder.resolve(strict=True) + copyfile( geometry_output_folder / (self._geometry_base_naming + "_mergedBundlesMaps.{}".format(self._extension)), simulation_output_folder / @@ -360,6 +368,8 @@ def _rename_and_copy_compartments_standalone( def _rename_and_copy_compartments( self, geometry_output_folder, simulation_output_folder ): + geometry_output_folder = geometry_output_folder.resolve(strict=True) + copyfile( geometry_output_folder / (self._geometry_base_naming + "_mergedBundlesMaps.{}".format(self._extension)), simulation_output_folder / @@ -446,6 +456,9 @@ def _generate_background_map( base_map=False, base_naming=None, ): + geometry_output_folder = geometry_output_folder.resolve(strict=True) + simulation_output_folder = simulation_output_folder.resolve(strict=True) + if not base_naming: base_naming = self._base_naming From 85aebd4abe7985a3514dca216f775e2309a028f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 22 Aug 2022 11:38:38 -0400 Subject: [PATCH 090/100] Removed quick simulation --- scripts/quick_simulation_runner.py | 180 ----------------------------- 1 file changed, 180 deletions(-) delete mode 100644 scripts/quick_simulation_runner.py diff --git a/scripts/quick_simulation_runner.py b/scripts/quick_simulation_runner.py deleted file mode 100644 index a0911fe..0000000 --- a/scripts/quick_simulation_runner.py +++ /dev/null @@ -1,180 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import math -import pathlib -import random -import tempfile - -import numpy as np -import numpy.linalg - -from simulator.factory import GeometryFactory, Plane, SimulationFactory -from simulator.runner.legacy import SimulationRunner -from simulator.utils.test_helpers import GeometryHelper - -resolution = [1, 1, 1] -spacing = [1, 1, 1] - -spheres_center = [[-2, 7, 10], [2, -1, 11]] -sphere_radius = 5 - -n_point_per_centroid = 5 -bundle_radius = 4 -bundle_symmetry = 1 -bundle_n_fibers = 200 -bundle_limits = [[0, 1], [0, 1], [0, 1]] -bundle_center = [0.5, 0.5, 0.5] -world_center = [5, 5, 5] - -base_anchors = [ - [0.5, -0.3, 0.5], - [0.5, -0.2, 0.5], - [0.5, -0.1, 0.5], - [0.5, 0, 0.5], - [0.5, 0.1, 0.5], - [0.5, 0.2, 0.5], - [0.5, 0.3, 0.5], - [0.5, 0.4, 0.5], - [0.5, 0.5, 0.5], - [0.5, 0.6, 0.5], - [0.5, 0.7, 0.5], - [0.5, 0.8, 0.5], - [0.5, 0.9, 0.5], - [0.5, 1.1, 0.5], - [0.5, 1.2, 0.5], - [0.5, 1.3, 0.5], -] - - -def get_geometry_parameters(output_folder: pathlib.Path, output_naming: str): - geometry_handler = GeometryFactory.get_geometry_handler(resolution, spacing) - - bundle1 = GeometryFactory.create_bundle( - bundle_radius, bundle_symmetry, n_point_per_centroid, base_anchors - ) - _, bundle2 = GeometryFactory.rotate_bundle( - bundle1, [0.5, 0.5, 0.5], math.pi / 6.0, Plane.YZ - ) - - cluster = GeometryFactory.create_cluster( - GeometryFactory.create_cluster_meta( - 3, bundle_n_fibers, 1, bundle_center, bundle_limits - ), - [bundle1, bundle2], - world_center, - ) - - geometry_handler.add_cluster(cluster) - - sphere_1 = GeometryFactory.create_sphere(sphere_radius, spheres_center[0]) - sphere_2 = GeometryFactory.create_sphere(sphere_radius, spheres_center[1]) - - geometry_handler.add_sphere(sphere_1) - geometry_handler.add_sphere(sphere_2) - - return geometry_handler.generate_json_configuration_files( - output_naming, output_folder - ) - - -def get_simulation_parameters(output_folder: pathlib.Path, output_naming: str): - fiber_compartment = SimulationFactory.generate_fiber_stick_compartment( - 0.007, 900, 80, SimulationFactory.CompartmentType.INTRA_AXONAL - ) - - restricted_fluid_compartment = ( - SimulationFactory.generate_extra_ball_compartment( - 2.0, 4000, 2000, SimulationFactory.CompartmentType.EXTRA_AXONAL_1 - ) - ) - - csf_compartment = SimulationFactory.generate_extra_ball_compartment( - 3.0, 4000, 2000, SimulationFactory.CompartmentType.EXTRA_AXONAL_2 - ) - - simulation_handler = SimulationFactory.get_simulation_handler( - GeometryHelper.get_dummy_empty_geometry_handler(), - [fiber_compartment, restricted_fluid_compartment, csf_compartment], - ) - - simulation_handler.set_acquisition_profile( - SimulationFactory.generate_acquisition_profile(1, 1, 1, inhomogen_time=1, echo_train_length=1) - ) - - noise_artifact = SimulationFactory.generate_noise_model("gaussian", 30) - motion_artifact = SimulationFactory.generate_motion_model( - True, "random", [3.1415 / 6, 0, 0], [4, 0, 0] - ) - - simulation_handler.set_artifact_model( - SimulationFactory.generate_artifact_model( - noise_artifact, motion_artifact - ) - ) - - normalize = lambda a: (np.array(a) / np.linalg.norm(a)).tolist() - - simulation_handler.set_gradient_profile( - SimulationFactory.generate_gradient_profile( - [500 for i in range(9)] - + [1000 for i in range(10)] - + [2000 for i in range(10)], - [ - normalize([random.uniform(-1, 1), random.uniform(-1, 1), random.uniform(-1, 1)]) - for i in range(30) - ], - 1, - g_type=SimulationFactory.AcquisitionType.STEJSKAL_TANNER, - ) - ) - - return simulation_handler.generate_xml_configuration_file( - output_naming, output_folder - ) - - -def run_simulation(output_folder: pathlib.Path): - geometry_parameters = get_geometry_parameters( - output_folder, "runner_test_geometry" - ) - - simulation_parameters = get_simulation_parameters( - output_folder, "runner_test_simulation" - ) - - runner = SimulationRunner( - "runner_test", - geometry_parameters, - simulation_parameters, - output_nifti=False, - ) - - runner.run(output_folder, True) - - simulation_parameters = get_simulation_parameters( - output_folder, "runner_test_simulation_standalone" - ) - - standalone_output = output_folder / "standalone_test" - - runner.run_simulation_standalone( - standalone_output, output_folder, simulation_parameters, "standalone" - ) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser("Simulation Runner Example Script") - parser.add_argument( - "--out", type=pathlib.Path, help="Output directory for the files" - ) - - args = parser.parse_args() - if args.out: - dest: pathlib.Path = args.out - dest.mkdir(parents=True, exist_ok=True) - else: - dest = pathlib.Path(tempfile.mkdtemp(prefix="sim_runner")) - - print("Script execution results are in : {}".format(dest)) - run_simulation(dest) From b095d6c1721c021e6937e8e2893f3276572683ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 22 Aug 2022 11:41:07 -0400 Subject: [PATCH 091/100] Increased version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ed0df5d..25e6e52 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ if __name__ == "__main__": setup( name="simulation_generator", - version="1.0.7", + version="1.0.8", packages=find_packages(exclude=("tests", "tests.*")), url="", license="", From 23aace3cc922f432d355471c8be91c781848aebf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 22 Aug 2022 12:10:00 -0400 Subject: [PATCH 092/100] Added modern script --- scripts/modern_simulation_runner.py | 36 +++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 scripts/modern_simulation_runner.py diff --git a/scripts/modern_simulation_runner.py b/scripts/modern_simulation_runner.py new file mode 100644 index 0000000..031f783 --- /dev/null +++ b/scripts/modern_simulation_runner.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 + +import argparse +import pathlib + +from scripts.geometry_factory import get_geometry_parameters +from simulator.runner.simulation_runner import SimulationRunner + + +def run_simulation(output_folder: pathlib.Path): + geometry_parameters = get_geometry_parameters( + output_folder, "runner_test_geometry" + ) + + runner = SimulationRunner() + runner.generate_phantom( + run_name="runner_test_simulation_standalone", + phantom_infos=geometry_parameters, + output_folder=output_folder, + output_nifti=False, + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser("Simulation Runner Example Script") + parser.add_argument( + "--out", type=pathlib.Path, default="./out/", help="Output directory for the files" + ) + + args = parser.parse_args() + dest: pathlib.Path = args.out + dest.mkdir(parents=True, exist_ok=True) + dest = dest.resolve(strict=True) + + print("Script execution results are in : {}".format(dest)) + run_simulation(dest) From b5e66abcc6d94c13b309c620b606d2634bf86eb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 22 Aug 2022 12:10:46 -0400 Subject: [PATCH 093/100] Renamed legacy simulation runner --- scripts/{simulation_runner.py => legacy_simulation_runner.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{simulation_runner.py => legacy_simulation_runner.py} (100%) mode change 100755 => 100644 diff --git a/scripts/simulation_runner.py b/scripts/legacy_simulation_runner.py old mode 100755 new mode 100644 similarity index 100% rename from scripts/simulation_runner.py rename to scripts/legacy_simulation_runner.py From 411b21b4fdd1684e64bc0ce7023cf9d70561f77c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 22 Aug 2022 12:11:01 -0400 Subject: [PATCH 094/100] Renamed modern simulation runner --- scripts/{modern_simulation_runner.py => simulation_runner.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{modern_simulation_runner.py => simulation_runner.py} (100%) diff --git a/scripts/modern_simulation_runner.py b/scripts/simulation_runner.py similarity index 100% rename from scripts/modern_simulation_runner.py rename to scripts/simulation_runner.py From e831958ecbb8a49280a40e6241879f259273214a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 22 Aug 2022 13:57:20 -0400 Subject: [PATCH 095/100] Renamed new simulation runner as it generates phantoms --- scripts/{simulation_runner.py => phantom_generation_runner.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{simulation_runner.py => phantom_generation_runner.py} (100%) diff --git a/scripts/simulation_runner.py b/scripts/phantom_generation_runner.py similarity index 100% rename from scripts/simulation_runner.py rename to scripts/phantom_generation_runner.py From 4342a41141a7bc171fea29cb25681cae3312441f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 22 Aug 2022 14:08:15 -0400 Subject: [PATCH 096/100] Increased version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 25e6e52..2e76f14 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ if __name__ == "__main__": setup( name="simulation_generator", - version="1.0.8", + version="1.0.9", packages=find_packages(exclude=("tests", "tests.*")), url="", license="", From dd72c010a42a2ab440a61afb90e763e25b1afd0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Sun, 28 Aug 2022 15:11:53 -0400 Subject: [PATCH 097/100] Fixed wall_effect spacing --- scripts/wall_effect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wall_effect.py b/scripts/wall_effect.py index bee00e0..eba1ee3 100755 --- a/scripts/wall_effect.py +++ b/scripts/wall_effect.py @@ -11,7 +11,7 @@ from simulator.runner.legacy import SimulationRunner resolution = [40, 40, 40] -spacing = [1, 1, 1] +spacing = [2, 2, 2] point_per_centroid = 30 x_anchors = [0.15, 0.25, 0.5, 0.75, 0.85] From 81478304a3aeabdbb53d1f3ae2c9ca3ed1a2f10d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 5 Sep 2022 18:36:53 -0400 Subject: [PATCH 098/100] Added "sync LINUM mirror repository" GitHub workflow --- .github/workflows/sync-linum-mirror-repo.yml | 28 ++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/sync-linum-mirror-repo.yml diff --git a/.github/workflows/sync-linum-mirror-repo.yml b/.github/workflows/sync-linum-mirror-repo.yml new file mode 100644 index 0000000..f8104b2 --- /dev/null +++ b/.github/workflows/sync-linum-mirror-repo.yml @@ -0,0 +1,28 @@ +name: Sync LINUM mirror repository + +# GitHub App/Action mirror-repository: https://github.com/yesolutions/mirror-action + +on: + push: + branches: + - prod + +jobs: + sync_mirror_repo: + runs-on: ubuntu-22.04 + + environment: + name: production + url: https://github.com/linum-uqam/inm5803-ete2022-benoit-dubreuil + + steps: + - uses: actions/checkout@v3 + with: + ref: 'master' + fetch-depth: 0 + + - uses: yesolutions/mirror-action@master + with: + REMOTE: 'git@github.com:linum-uqam/voxsim.git' + GIT_SSH_PRIVATE_KEY: ${{ secrets.LINUM_DEPLOY_GIT_SSH_PRIVATE_KEY }} # The public key is a deploy key on the remote repository + GIT_SSH_NO_VERIFY_HOST: 'true' # We're pushing from a public repo to a public repo, nothing can really be stolen. \ No newline at end of file From 064870d336a34c4ea0b5766d968353624512117a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 5 Sep 2022 18:39:57 -0400 Subject: [PATCH 099/100] This sync-bitbucket workflow is proprietary to the original repo owner --- .github/workflows/sync-bitbucket.yml | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 .github/workflows/sync-bitbucket.yml diff --git a/.github/workflows/sync-bitbucket.yml b/.github/workflows/sync-bitbucket.yml deleted file mode 100644 index 0df9264..0000000 --- a/.github/workflows/sync-bitbucket.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Sync voXSim remote repository - -# GitHub App/Action mirror-repository: https://github.com/yesolutions/mirror-action - -on: - push: - branches: - - master - -jobs: - sync_mirror_repo: - runs-on: ubuntu-latest - - environment: - name: MIRRORING - - steps: - - uses: actions/checkout@v3 - with: - ref: "master" - fetch-depth: 0 - - - uses: yesolutions/mirror-action@master - with: - REMOTE: "${{ secrets.VOXSIM_REMOTE }}" - GIT_SSH_PRIVATE_KEY: ${{ secrets.VOXSIM_SSH_PRIVATE_KEY }} - GIT_SSH_KNOWN_HOSTS: ${{ secrets.VOXSIM_SSH_KNOWN_HOSTS }} From 4abb6936de111569401f956d6050739ff042e42b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Dubreuil?= Date: Mon, 5 Sep 2022 18:41:22 -0400 Subject: [PATCH 100/100] Fixed push branch --- .github/workflows/sync-linum-mirror-repo.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync-linum-mirror-repo.yml b/.github/workflows/sync-linum-mirror-repo.yml index f8104b2..23ef078 100644 --- a/.github/workflows/sync-linum-mirror-repo.yml +++ b/.github/workflows/sync-linum-mirror-repo.yml @@ -5,7 +5,7 @@ name: Sync LINUM mirror repository on: push: branches: - - prod + - master jobs: sync_mirror_repo: