From 3470489215fc19dff5fec008a60f8b91d9d4c81d Mon Sep 17 00:00:00 2001 From: JonasFischer1 <90911204+JonasFischer1@users.noreply.github.com> Date: Tue, 19 May 2026 11:04:25 +0200 Subject: [PATCH 01/24] feat(util): add raw movie discovery helper and tests --- picasso_workflow/tests/test_util.py | 36 +++++++++ picasso_workflow/util.py | 118 ++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+) diff --git a/picasso_workflow/tests/test_util.py b/picasso_workflow/tests/test_util.py index e7d3dd6..28e46b1 100644 --- a/picasso_workflow/tests/test_util.py +++ b/picasso_workflow/tests/test_util.py @@ -215,3 +215,39 @@ def test_06_valid_expression(self): expression = "* 3.1415" val = util.is_valid_expression(expression) assert val + + def test_07_get_movie_groups(self): + from picasso_workflow.metaworkflow import get_movie_groups + + paths = [ + "movie_1.ome.tif", + "movie_0.ome.tif", + "movie_2.ome.tif", + "other.ome.tif", + ] + groups = get_movie_groups(paths, ".ome.tif") + assert list(groups.keys()) == ["movie", "other"] + assert groups["movie"] == [ + "movie_0.ome.tif", + "movie_1.ome.tif", + "movie_2.ome.tif", + ] + assert groups["other"] == ["other.ome.tif"] + + def test_08_find_dnapaint_raw(self): + import tempfile + from pathlib import Path + from picasso_workflow.metaworkflow import find_dnapaint_raw + + with tempfile.TemporaryDirectory() as tempdir: + temp_path = Path(tempdir) + (temp_path / "cond1" / "pos10").mkdir(parents=True) + (temp_path / "cond1" / "pos10" / "movie_0.ome.tif").touch() + (temp_path / "cond1" / "pos2").mkdir(parents=True) + (temp_path / "cond1" / "pos2" / "movie_0.ome.tif").touch() + + datasets, _ = find_dnapaint_raw(tempdir) + keys = list(datasets.keys()) + assert "pos2" in keys + assert "pos10" in keys + assert keys.index("pos2") < keys.index("pos10") diff --git a/picasso_workflow/util.py b/picasso_workflow/util.py index 01fa72e..b7aa8ee 100644 --- a/picasso_workflow/util.py +++ b/picasso_workflow/util.py @@ -2731,3 +2731,121 @@ def stripplot(data, positions, jitter, ax, color, alpha=1): x = pos * np.ones(len(d)) x += np.random.uniform(-jitter / 2, jitter / 2, size=len(d)) ax.scatter(x, d, color=color, alpha=alpha) + + +def get_movie_groups(paths, extension): + """ + Groups files based on basename and index, supporting variable extensions. + + Args: + paths: list of filenames + extension: the extension to match (e.g. '.tif' or '.ome.tif') + Returns: + dict: mapping from base name to list of file paths (sorted by index) + """ + import re + + groups = {} + if not paths: + return groups + + ext_pattern = re.escape(extension) + pattern = re.compile(rf"(.*?)(?:_(\d+))?{ext_pattern}$") + + match_infos = [] + for path in paths: + match = pattern.match(path) + if match: + base, index = match.groups() + match_infos.append( + { + "path": path, + "base": base, + "index": int(index) if index else 0, + } + ) + + # Grouping logic + basenames = {m["base"] for m in match_infos} + for base in basenames: + group_items = [m for m in match_infos if m["base"] == base] + # Sort by index + group_items.sort(key=lambda x: x["index"]) + groups[base] = [item["path"] for item in group_items] + + return groups + + +def find_raw_movies(working_folder): + """ + Recursively finds raw movie files (.tif, .ome.tif, .nd2) in a folder. + + Args: + working_folder: path to search + Returns: + dict: mapping from dataset name to path or list of paths + """ + import os + import fnmatch + from pathlib import Path + + datasets = {} + + for root, dirs, files in os.walk(working_folder): + p = Path(root) + + # Check what extensions exist here + has_nd2 = list(p.glob("*.nd2")) + has_ometif = list(p.glob("*.ome.tif")) + # .tif check needs to exclude .ome.tif to be accurate + has_tif = [ + f for f in p.glob("*.tif") if not f.name.endswith(".ome.tif") + ] + + found_types = [] + if has_nd2: + found_types.append(".nd2") + if has_ometif: + found_types.append(".ome.tif") + if has_tif: + found_types.append(".tif") + + if not found_types: + continue + + # Priority: .nd2 > .ome.tif > .tif + ext = found_types[0] + + if ext == ".nd2": + nd2_files = sorted(fnmatch.filter(os.listdir(root), "*.nd2")) + for f in nd2_files: + stem = Path(f).stem + # Use parent folder name as key if unique, otherwise include stem + key = p.name if p.name else stem + if len(nd2_files) > 1: + key = f"{p.name}_{stem}" + datasets[key] = str(p / f) + else: + # tif or ome.tif + tif_files = sorted([f.name for f in p.glob(f"*{ext}")]) + groups = get_movie_groups(tif_files, ext) + for base, group_paths in groups.items(): + # Use directory name as key if it's the only group, otherwise use base + if len(groups) == 1: + key = p.name + else: + key = base + + full_paths = [str(p / fname) for fname in group_paths] + datasets[key] = full_paths[0] + + # Natural sort the datasets by key + import re + + def natsort_key(s): + return [ + int(t) if t.isdigit() else t.lower() for t in re.split(r"(\d+)", s) + ] + + sorted_keys = sorted(datasets.keys(), key=natsort_key) + return {k: datasets[k] for k in sorted_keys} From 6f5e1517390dfa22a1c541542898468a4c0f07da Mon Sep 17 00:00:00 2001 From: JonasFischer1 <90911204+JonasFischer1@users.noreply.github.com> Date: Tue, 19 May 2026 11:05:41 +0200 Subject: [PATCH 02/24] transition raw movie discovery from metaworkflow to util --- picasso_workflow/metaworkflow.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/picasso_workflow/metaworkflow.py b/picasso_workflow/metaworkflow.py index fbd0dab..94fde8d 100644 --- a/picasso_workflow/metaworkflow.py +++ b/picasso_workflow/metaworkflow.py @@ -358,19 +358,11 @@ def parse_source( def find_dnapaint_raw(working_folder): - datasets = {} - for root, dirs, files in os.walk(working_folder): - for file in files: - match = re.search(r"_MMStack_Pos(\d+).ome.tif", file) - if match is not None: - key = os.path.split(root)[-1] - datasets[key] = os.path.join(root, file) - continue - match = re.search(r"_NDTiffStack.tif", file) - if match is not None: - key = os.path.split(root)[-1] - datasets[key] = os.path.join(root, file) - continue + from picasso_workflow import util + from picasso import io + + datasets = util.find_raw_movies(working_folder) + dest_file = os.path.join(working_folder, "src_loc.yaml") io.save_info(dest_file, [datasets]) return datasets, dest_file From 00590069f18c92431423a6930e68c1a057082bfe Mon Sep 17 00:00:00 2001 From: JonasFischer1 <90911204+JonasFischer1@users.noreply.github.com> Date: Tue, 19 May 2026 11:08:01 +0200 Subject: [PATCH 03/24] Adds find files in folder button to gui --- picasso_workflow/gui.py | 105 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 100 insertions(+), 5 deletions(-) diff --git a/picasso_workflow/gui.py b/picasso_workflow/gui.py index d4b8a0a..1bd67e7 100644 --- a/picasso_workflow/gui.py +++ b/picasso_workflow/gui.py @@ -9282,6 +9282,10 @@ def _update_file_buttons(self, workflow_type_index): if item.widget(): item.widget().deleteLater() + self.find_files_button = QtWidgets.QPushButton("Find in folder") + self.find_files_button.clicked.connect(self.find_dataset_files) + self.file_buttons_layout.addWidget(self.find_files_button) + if workflow_type_index == 0: # Single Workflow self.add_files_button = QtWidgets.QPushButton("Add files") self.add_files_button.clicked.connect(self.add_files) @@ -9331,6 +9335,93 @@ def _update_file_buttons(self, workflow_type_index): self.file_buttons_layout.addWidget(self.remove_dataset_button) self.file_buttons_layout.addWidget(self.clear_tree_button) + def find_dataset_files(self): + """Find dataset files (.tif, .ome.tif, .nd2) in a selected folder.""" + folder = QtWidgets.QFileDialog.getExistingDirectory( + self, + "Select Data Folder", + "", + ) + if folder: + from picasso_workflow.metaworkflow import find_dnapaint_raw + + datasets, _ = find_dnapaint_raw(folder) + if datasets: + workflow_type_index = self.workflow_type.currentIndex() + if workflow_type_index == 0: + # Single workflow + for key, paths in datasets.items(): + row = self.files_table.rowCount() + self.files_table.insertRow(row) + self.files_table.setItem( + row, 0, QtWidgets.QTableWidgetItem(key) + ) + if isinstance(paths, list): + if len(paths) == 1: + self.files_table.setItem( + row, + 1, + QtWidgets.QTableWidgetItem(paths[0]), + ) + else: + self.files_table.setItem( + row, + 1, + QtWidgets.QTableWidgetItem(repr(paths)), + ) + else: + self.files_table.setItem( + row, 1, QtWidgets.QTableWidgetItem(paths) + ) + else: + # Aggregation / Investigation workflow + if not self.tree_data.get("channels"): + self.tree_data["channels"] = ["ch1"] + + for key, paths in datasets.items(): + # Ensure unique key + unique_key = key + counter = 1 + while unique_key in self.tree_data.get("datasets", []): + unique_key = f"{key}_{counter}" + counter += 1 + + if "datasets" not in self.tree_data: + self.tree_data["datasets"] = [] + if "conditions" not in self.tree_data: + self.tree_data["conditions"] = {} + if "file_paths" not in self.tree_data: + self.tree_data["file_paths"] = {} + + self.tree_data["datasets"].append(unique_key) + self.tree_data["conditions"][unique_key] = "" + self.tree_data["file_paths"][unique_key] = {} + + # Assign to the first channel + target_channel = self.tree_data["channels"][0] + + if isinstance(paths, list): + if len(paths) == 1: + self.tree_data["file_paths"][unique_key][ + target_channel + ] = paths[0] + else: + self.tree_data["file_paths"][unique_key][ + target_channel + ] = repr(paths) + else: + self.tree_data["file_paths"][unique_key][ + target_channel + ] = paths + + self._populate_tree_from_data() + else: + QtWidgets.QMessageBox.information( + self, + "No files found", + f"Could not find any supported dataset files in {folder}", + ) + def _set_widgets_enabled(self, enabled): """Enable or disable file and module widgets based on results folder selection.""" self.workflow_type.setEnabled(enabled) @@ -9446,13 +9537,17 @@ def on_results_folder_dropped(self, folder): def on_template_changed(self, template_name): """Load a template""" try: - template_folder = CONFIG["Templates"][template_name] - if template_folder[:2] == 'r"' or template_folder[:2] == "r'": - template_folder = template_folder[2:-1] + template_path = CONFIG["Templates"][template_name] + if template_path[:2] == 'r"' or template_path[:2] == "r'": + template_path = template_path[2:-1] + + # If path points to a file, use its directory + if template_path.endswith(".py"): + template_folder = os.path.dirname(template_path) + else: + template_folder = template_path except KeyError: return - # # Enable widgets when a folder is selected - # self._set_widgets_enabled(True) # Search for YAML files and load file list self._load_yaml_file_list(template_folder) From 888479b0895b24ce82169845d32fbce25c5c18a0 Mon Sep 17 00:00:00 2001 From: JonasFischer1 <90911204+JonasFischer1@users.noreply.github.com> Date: Tue, 19 May 2026 11:09:21 +0200 Subject: [PATCH 04/24] removes grid and padding from images --- picasso_workflow/outpost_modules/render.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/picasso_workflow/outpost_modules/render.py b/picasso_workflow/outpost_modules/render.py index edb73d1..65c90c0 100644 --- a/picasso_workflow/outpost_modules/render.py +++ b/picasso_workflow/outpost_modules/render.py @@ -345,9 +345,7 @@ def plot_scene( (bgra.shape[0] * image_px_size + y_offset) / 1000, ], ) - ax.set_xlabel("x [µm]") - ax.set_ylabel("y [µm]") - ax.set_title(title) + ax.axis("off") if fp is not None: - fig.savefig(fp) + fig.savefig(fp, bbox_inches="tight", pad_inches=0) return fig, ax From 36e2d46fe6d9c9afbe10060d4a02b5847a85e7c5 Mon Sep 17 00:00:00 2001 From: JonasFischer1 <90911204+JonasFischer1@users.noreply.github.com> Date: Tue, 19 May 2026 11:10:47 +0200 Subject: [PATCH 05/24] adds NumPy scalar type casting --- picasso_workflow/confluence.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/picasso_workflow/confluence.py b/picasso_workflow/confluence.py index 1d16cf8..d6e89db 100644 --- a/picasso_workflow/confluence.py +++ b/picasso_workflow/confluence.py @@ -31,9 +31,17 @@ def module_wrapper(self, i, parameters, results, postpone_report=False):