diff --git a/picasso_workflow/analyse.py b/picasso_workflow/analyse.py index dca6293..de0d999 100644 --- a/picasso_workflow/analyse.py +++ b/picasso_workflow/analyse.py @@ -7,6 +7,7 @@ """ from picasso import lib, io, localize, gausslq, postprocess, clusterer from picasso import aim, spinna + # from picasso_workflow.outpost_modules import g5m from picasso import g5m from picasso import __version__ as picassoversion @@ -35,7 +36,7 @@ import pandas as pd import psutil import yaml -from matplotlib import cm +from matplotlib import cm, colormaps from memory_profiler import memory_usage from picasso import CONFIG as pCONFIG from picasso import __version__ as picassoversion @@ -50,6 +51,7 @@ postprocess, spinna, ) + # from picasso_workflow.outpost_modules import g5m from scipy.ndimage import label from scipy.spatial import KDTree, distance @@ -414,14 +416,16 @@ def camera_info(self): "Micro-Manager Metadata" ].get(f"{cam_name}-{category}") cat_vals += f"{category}: {category_value}; " - + if category_value in sensitivity: sensitivity = sensitivity[category_value] elif str(category_value) in sensitivity: sensitivity = sensitivity[str(category_value)] else: try: - sensitivity = sensitivity.get(int(category_value), {}) + sensitivity = sensitivity.get( + int(category_value), {} + ) except (ValueError, TypeError): sensitivity = {} if isinstance(sensitivity, dict): @@ -440,8 +444,7 @@ def camera_info(self): } for category in cam_config.get("Sensitivity Categories"): category_key = f"{cam_name}-{category}" - - + category_value = self.info[0].get(category_key) self.analysis_config["camera_info"] = camera_info return camera_info @@ -860,18 +863,27 @@ def load_dataset_movie(self, i, parameters, results): # sensitivity starts being a dict, and ends as a value cat_vals = "" for category in cam_config.get("Sensitivity Categories"): - category_value = self.info[0].get(f"{cam_name}-{category}") - if category_value is None and "Micro-Manager Metadata" in self.info[0]: - category_value = self.info[0]["Micro-Manager Metadata"].get(f"{cam_name}-{category}") + category_value = self.info[0].get( + f"{cam_name}-{category}" + ) + if ( + category_value is None + and "Micro-Manager Metadata" in self.info[0] + ): + category_value = self.info[0][ + "Micro-Manager Metadata" + ].get(f"{cam_name}-{category}") cat_vals += f"{category}: {category_value}; " - + if category_value in sensitivity: sensitivity = sensitivity[category_value] elif str(category_value) in sensitivity: sensitivity = sensitivity[str(category_value)] else: try: - sensitivity = sensitivity.get(int(category_value), {}) + sensitivity = sensitivity.get( + int(category_value), {} + ) except (ValueError, TypeError): sensitivity = {} if isinstance(sensitivity, dict): @@ -1272,13 +1284,17 @@ def identify(self, i, parameters, results): """ # auto-detect net grad if required: if (autograd_pars := parameters.get("auto_netgrad")) is not None: - if "filename" in autograd_pars.keys() and autograd_pars["filename"]: + if ( + "filename" in autograd_pars.keys() + and autograd_pars["filename"] + ): autograd_pars["filename"] = os.path.join( results["folder"], autograd_pars["filename"] ) else: autograd_pars["filename"] = os.path.join( - results["folder"], "auto_identification.png") + results["folder"], "auto_identification.png" + ) potential_pars = [ "box_size", @@ -1725,6 +1741,8 @@ def render(self, i, parameters, results): filepath to full FOV rendering fp_scene_ctrmass : str filepath to center of mass zoom rendering (conditional, only if ctrmass_fov_nm provided) + fp_scene_tiles : list of lists of str + filepaths to the 5x5 tiled renderings """ pixelsize = self.pixelsize rcode = generate_random_code(6) @@ -1738,46 +1756,307 @@ def render(self, i, parameters, results): x_mean = np.mean(self.locs["x"]) y_mean = np.mean(self.locs["y"]) + # Check if the dataset is 3D (has a 'z' column) to prevent rotation KeyErrors in 2D + def check_has_z(locs): + if isinstance(locs, list): + if not locs: + return False + locs = locs[0] + try: + return "z" in locs.dtype.names + except AttributeError: + try: + return "z" in locs.columns + except (AttributeError, TypeError): + return False + + has_z = check_has_z(render_locs) + + # Read colormap choice (default to magma) + cmap_choice = parameters.get("colormap", "magma") + # render whole field of view - fullfov_pixelsize = parameters.get("fullfov_pixelsize", pixelsize) + fullfov_pixelsize = parameters.get("fullfov_pixelsize", pixelsize / 10) + + # Normalize localizations to a list + locs_list = ( + render_locs if isinstance(render_locs, list) else [render_locs] + ) + + # Calculate full FOV boundaries in camera pixels + x_min = min([lcs["x"].min() for lcs in locs_list]) + x_max = max([lcs["x"].max() for lcs in locs_list]) + y_min = min([lcs["y"].min() for lcs in locs_list]) + y_max = max([lcs["y"].max() for lcs in locs_list]) + + # Density-driven ROI selection (Alternative A) + selected_rois = ( + [] + ) # list of (x_ctr, y_ctr, tile_x_min, tile_x_max, tile_y_min, tile_y_max) + roi_files = [] + + if parameters.get("generate_active_rois", True): + all_x = np.concatenate([lcs["x"] for lcs in locs_list]) + all_y = np.concatenate([lcs["y"] for lcs in locs_list]) + + roi_size = parameters.get("ctrmass_fov_nm", 20000.0) / pixelsize + if roi_size <= 0: + roi_size = 100.0 # fallback + + # Finer grid peak finding (bin size = roi_size / 2) + grid_step = roi_size / 2.0 + bin_edges_x = np.arange(x_min, x_max + grid_step, grid_step) + bin_edges_y = np.arange(y_min, y_max + grid_step, grid_step) + + hist, x_edges, y_edges = np.histogram2d( + all_x, all_y, bins=[bin_edges_x, bin_edges_y] + ) + + candidates = [] + for ix in range(hist.shape[0]): + for iy in range(hist.shape[1]): + count = hist[ix, iy] + if count > 0: + center_x = (x_edges[ix] + x_edges[ix + 1]) / 2 + center_y = (y_edges[iy] + y_edges[iy + 1]) / 2 + candidates.append((count, center_x, center_y)) + + candidates.sort(key=lambda item: item[0], reverse=True) + n_rois = parameters.get("n_active_rois") + if n_rois is None: + n_rois = 4 + roi_centers = [] + min_distance = roi_size # non-overlapping + + for count, cx, cy in candidates: + if len(roi_centers) >= n_rois: + break + + # Shift ROI center if it would place the ROI viewport boundary outside the image area + if (x_max - x_min) >= roi_size: + cx = np.clip( + cx, x_min + roi_size / 2.0, x_max - roi_size / 2.0 + ) + else: + cx = (x_min + x_max) / 2.0 + + if (y_max - y_min) >= roi_size: + cy = np.clip( + cy, y_min + roi_size / 2.0, y_max - roi_size / 2.0 + ) + else: + cy = (y_min + y_max) / 2.0 + + too_close = False + for sx, sy in roi_centers: + dist = np.sqrt((cx - sx) ** 2 + (cy - sy) ** 2) + if dist < min_distance: + too_close = True + break + if not too_close: + roi_centers.append((cx, cy)) + + # Form ROIs and render them + tile_pixelsize = parameters.get("ctrmass_pixelsize", pixelsize / 2) + for idx, (cx, cy) in enumerate(roi_centers): + tile_x_min = cx - roi_size / 2 + tile_x_max = cx + roi_size / 2 + tile_y_min = cy - roi_size / 2 + tile_y_max = cy + roi_size / 2 + + selected_rois.append( + (cx, cy, tile_x_min, tile_x_max, tile_y_min, tile_y_max) + ) + + tile_kwargs = { + "oversampling": pixelsize / tile_pixelsize, + "viewport": [ + (tile_y_min, tile_x_min), + (tile_y_max, tile_x_max), + ], + "blur_method": parameters.get("ctrmass_blur_method"), + "min_blur_width": parameters.get( + "ctrmass_min_blur_width", 0 + ), + "cmap": cmap_choice, + } + if has_z and parameters.get("ctrmass_ang") is not None: + tile_kwargs["ang"] = parameters.get("ctrmass_ang") + + roi_fp = os.path.join( + results["folder"], f"locs_active_roi_{idx + 1}_{rcode}.png" + ) + fig_roi, _ = render.plot_scene( + render_locs, + tile_pixelsize, + pixelsize, + fp=roi_fp, + render_kwargs=tile_kwargs, + ) + plt.close(fig_roi) + roi_files.append(roi_fp) + + results["fp_scene_rois"] = roi_files + + # Render whole field of view (overview) and draw outlines results["fp_scene_fullfov"] = os.path.join( results["folder"], f"locs_fullfov_{rcode}.png" ) - render.plot_scene( + + fig_overview, ax_overview = render.plot_scene( render_locs, fullfov_pixelsize, pixelsize, - fp=results["fp_scene_fullfov"], + fp=None, + render_kwargs={"cmap": cmap_choice}, ) + # annotation borders at the image edge don't widen the tight bbox. + _overview_xlim = ax_overview.get_xlim() + _overview_ylim = ax_overview.get_ylim() - # render zoom into the center of mass + # Save unmarked copy of the overview image + results["fp_scene_fullfov_unmarked"] = os.path.join( + results["folder"], f"locs_fullfov_unmarked_{rcode}.png" + ) + fig_overview.savefig( + results["fp_scene_fullfov_unmarked"], + pad_inches=0, + ) + + # Draw outlines of selected ROIs if present, or Zoom-In outline if Zoom-In is displayed + x_mean_clipped = x_mean + y_mean_clipped = y_mean if parameters.get("ctrmass_fov_nm"): - ctrmass_pixelsize = parameters.get("ctrmass_pixelsize", pixelsize) + zoom_size = parameters.get("ctrmass_fov_nm") / pixelsize + if (x_max - x_min) >= zoom_size: + x_mean_clipped = np.clip( + x_mean, x_min + zoom_size / 2.0, x_max - zoom_size / 2.0 + ) + else: + x_mean_clipped = (x_min + x_max) / 2.0 + + if (y_max - y_min) >= zoom_size: + y_mean_clipped = np.clip( + y_mean, y_min + zoom_size / 2.0, y_max - zoom_size / 2.0 + ) + else: + y_mean_clipped = (y_min + y_max) / 2.0 + + if selected_rois: + for idx, (_, _, t_xmin, t_xmax, t_ymin, t_ymax) in enumerate( + selected_rois + ): + x_min_um = (t_xmin * pixelsize) / 1000.0 + x_max_um = (t_xmax * pixelsize) / 1000.0 + y_min_um = (t_ymin * pixelsize) / 1000.0 + y_max_um = (t_ymax * pixelsize) / 1000.0 + + width_um = x_max_um - x_min_um + height_um = y_max_um - y_min_um + + ax_overview.plot( + [x_min_um, x_max_um, x_max_um, x_min_um, x_min_um], + [y_min_um, y_min_um, y_max_um, y_max_um, y_min_um], + color="red", + linewidth=1.5, + clip_on=True, + ) + + # Add text label for each active site + ax_overview.text( + x_min_um + 0.03 * width_um, + y_min_um + 0.12 * height_um, + str(idx + 1), + color="red", + fontsize=10, + fontweight="bold", + bbox=dict( + facecolor="black", + alpha=0.6, + boxstyle="round,pad=0.2", + edgecolor="none", + ), + ) + elif parameters.get("ctrmass_fov_nm"): + # Draw standard Zoom-In outline on overview image fov_half = parameters.get("ctrmass_fov_nm") / 2 - x_min = x_mean - fov_half / pixelsize - x_max = x_mean + fov_half / pixelsize - y_min = y_mean - fov_half / pixelsize - y_max = y_mean + fov_half / pixelsize + x_min_zoom = x_mean_clipped - fov_half / pixelsize + x_max_zoom = x_mean_clipped + fov_half / pixelsize + y_min_zoom = y_mean_clipped - fov_half / pixelsize + y_max_zoom = y_mean_clipped + fov_half / pixelsize + + x_min_um = (x_min_zoom * pixelsize) / 1000.0 + x_max_um = (x_max_zoom * pixelsize) / 1000.0 + y_min_um = (y_min_zoom * pixelsize) / 1000.0 + y_max_um = (y_max_zoom * pixelsize) / 1000.0 + + width_um = x_max_um - x_min_um + height_um = y_max_um - y_min_um + + ax_overview.plot( + [x_min_um, x_max_um, x_max_um, x_min_um, x_min_um], + [y_min_um, y_min_um, y_max_um, y_max_um, y_min_um], + color="red", + linewidth=1.5, + clip_on=True, + ) + + ax_overview.text( + x_min_um + 0.03 * width_um, + y_min_um + 0.12 * height_um, + "Zoom-In", + color="red", + fontsize=10, + fontweight="bold", + bbox=dict( + facecolor="black", + alpha=0.6, + boxstyle="round,pad=0.2", + edgecolor="none", + ), + ) + + ax_overview.set_xlim(_overview_xlim) + ax_overview.set_ylim(_overview_ylim) + fig_overview.savefig( + results["fp_scene_fullfov"], + pad_inches=0, + ) + plt.close(fig_overview) + + # render zoom into the center of mass + if parameters.get("ctrmass_fov_nm"): + ctrmass_pixelsize = parameters.get("ctrmass_pixelsize", pixelsize / 2) + fov_half = parameters.get("ctrmass_fov_nm") + x_min_zoom = x_mean_clipped - fov_half / pixelsize + x_max_zoom = x_mean_clipped + fov_half / pixelsize + y_min_zoom = y_mean_clipped - fov_half / pixelsize + y_max_zoom = y_mean_clipped + fov_half / pixelsize render_kwargs = { "oversampling": pixelsize / ctrmass_pixelsize, - "viewport": [(y_min, x_min), (y_max, x_max)], + "viewport": [ + (y_min_zoom, x_min_zoom), + (y_max_zoom, x_max_zoom), + ], "blur_method": parameters.get("ctrmass_blur_method"), "min_blur_width": parameters.get("ctrmass_min_blur_width", 0), - "ang": parameters.get("ctrmass_ang"), + "cmap": cmap_choice, } + if has_z and parameters.get("ctrmass_ang") is not None: + render_kwargs["ang"] = parameters.get("ctrmass_ang") results["fp_scene_ctrmass"] = os.path.join( results["folder"], f"locs_ctrmass_{rcode}.png" ) - render.plot_scene( + fig_ctrmass, _ = render.plot_scene( render_locs, ctrmass_pixelsize, pixelsize, fp=results["fp_scene_ctrmass"], render_kwargs=render_kwargs, - # x_offset=x_min * pixelsize, - # y_offset=y_min * pixelsize, ) + plt.close(fig_ctrmass) + return parameters, results # @profile_resource_usage @@ -4441,7 +4720,8 @@ def dbscan(self, i, parameters, results): min_locs = parameters["min_locs"] # label locs according to clusters self.locs = clusterer.dbscan( - self.locs, radius, min_samples, min_locs, pixelsize) + self.locs, radius, min_samples, min_locs, pixelsize + ) dbscan_info = { "Generated by": "Picasso DBSCAN", "Radius": radius, @@ -6555,8 +6835,12 @@ def gaussian_mixture_cluster(self, i, parameters, results): results["folder"], "subcluster_test.png" ) # g5m.test_subclustering(center_locs, results["fp_fig_subclustering"]) - clustered_nevents, sparse_nevents = clusterer.test_subclustering(center_locs, self.info) - lib.plot_subclustering_check(clustered_nevents, sparse_nevents, results["fp_fig_subclustering"]) + clustered_nevents, sparse_nevents = clusterer.test_subclustering( + center_locs, self.info + ) + lib.plot_subclustering_check( + clustered_nevents, sparse_nevents, results["fp_fig_subclustering"] + ) results["n_locs_in"] = len(self.locs) results["n_locs_clustered"] = len(clustered_locs) @@ -6695,7 +6979,7 @@ def nneighbor(self, i, parameters, results): density_results.append(density) # plot results - colors = cm.get_cmap("viridis", nneighbors.shape[1]).colors + colors = colormaps["viridis"].resampled(nneighbors.shape[1]).colors bins = np.arange(0, rmax_NN, step=deltar) nnhist_obs = np.zeros((len(bins), nneighbors.shape[1])) for i in range(nnhist_obs.shape[1]): @@ -7018,7 +7302,7 @@ def csr_cdf(x): # plot results fig, ax = plt.subplots() - colors = cm.get_cmap("viridis", k_max).colors + colors = colormaps["viridis"].resampled(k_max).colors bin_max = np.quantile(nneighbors[:, -1], 0.95) median_1stNN = np.median(nneighbors[:, 0]) # sample bins such that there are 5 bins from 0 to middle of 1stNN diff --git a/picasso_workflow/confluence.py b/picasso_workflow/confluence.py index 1d16cf8..b91988e 100644 --- a/picasso_workflow/confluence.py +++ b/picasso_workflow/confluence.py @@ -31,10 +31,17 @@ def module_wrapper(self, i, parameters, results, postpone_report=False): @@ -49,9 +56,7 @@ def module_wrapper(self, i, parameters, results, postpone_report=False): @@ -443,12 +448,17 @@ def analysis_documentation( text = f"""

Module {i:02d}: Analysis Hard- and Software

+ + Parameters +
    """ for k, v in results.items(): - text += f"
  • {k}: {v}
    • " + text += f"
  • {html.escape(str(k))}: {html.escape(str(v))}
    • " text += """
+ + """ self.ci.update_page_content( @@ -641,7 +651,10 @@ def load_dataset_localizations( @module_decorator def identify( - self, i, parameters, results, + self, + i, + parameters, + results, parameter_text, result_text, postpone_report=False, @@ -1098,6 +1111,8 @@ def render( filepath to full FOV rendering fp_scene_ctrmass : str filepath to center of mass zoom rendering (conditional, only if ctrmass_fov_nm provided) + fp_scene_tiles : list of lists of str + filepaths to the 5x5 tiled renderings """ logger.debug("Reporting render.") text = f""" @@ -1111,38 +1126,92 @@ def render( {parameter_text} {result_text} """ - fig_fps = [] - titles = [] - if fp_fig := results.get("fp_scene_fullfov"): - fig_fps.append(fp_fig) - titles.append("Localizations in whole Field of View") - if fp_fig := results.get("fp_scene_ctrmass"): - fig_fps.append(fp_fig) - titles.append("Localizations in Zoom-In") - if len(fig_fps) > 1: - fn_figs = [] - for fp in fig_fps: + generate_active_rois = parameters.get("generate_active_rois", True) + rois = results.get("fp_scene_rois", []) + + text += "" + + # Left column: Field of View Overviews + text += "
" + if fp_fullfov := results.get("fp_scene_fullfov"): + try: + self.ci.upload_attachment(self.report_page_id, fp_fullfov) + except ConfluenceInterfaceError: + pass + fn_fullfov = os.path.split(fp_fullfov)[1] + + fp_unmarked = results.get("fp_scene_fullfov_unmarked") + if fp_unmarked: try: - self.ci.upload_attachment(self.report_page_id, fp) + self.ci.upload_attachment(self.report_page_id, fp_unmarked) except ConfluenceInterfaceError: pass - fn_figs.append(os.path.split(fp)[1]) + fn_unmarked = os.path.split(fp_unmarked)[1] - text += "" - for tit in titles: - text += f"" - text += "" - text += "" - for fn in fn_figs: text += f""" - """ - text += "" +

Overview Images

+

+ + + +    + + + +

+ """ + else: + text += f""" +

Overview Image

+ + + + """ + text += "" + + # Right column: Either active site images OR the Zoom-in image depending on selection + text += "" + + text += "
{tit}
- - - -
" + if generate_active_rois and rois: + text += "

Zoom-In Images (Density-Driven)

" + text += "" + for idx in range(0, len(rois), 2): + text += "" + for sub_idx in range(idx, min(idx + 2, len(rois))): + fp_roi = rois[sub_idx] + try: + self.ci.upload_attachment(self.report_page_id, fp_roi) + except ConfluenceInterfaceError: + pass + fn_roi = os.path.split(fp_roi)[1] + text += f""" + """ + # If odd number of ROIs and this is the last row, add an empty cell for layout alignment + if len(rois) % 2 != 0 and idx + 1 >= len(rois): + text += "" + text += "" text += "
+ + + +
Site {sub_idx + 1} +
" + else: + if fp_ctrmass := results.get("fp_scene_ctrmass"): + try: + self.ci.upload_attachment(self.report_page_id, fp_ctrmass) + except ConfluenceInterfaceError: + pass + fn_ctrmass = os.path.split(fp_ctrmass)[1] + text += f""" +

Zoom-In Image

+ + + + """ + text += "
" text += """ @@ -4551,7 +4620,7 @@ def filter_locs( {result_text}""" if fp_fig := results.get("fp_fig_before"): - text += "
    " + text += "
    " text += "" text += "" text += "
    Before FilteringAfter Filtering
    " @@ -4575,7 +4644,7 @@ def filter_locs( """ - text += "
" + text += "
" text += """ @@ -4649,7 +4718,7 @@ def filter_transient_binding( """ if fp_fig := results.get("fp_fig_before"): - text += "" + text += "" text += """ diff --git a/picasso_workflow/gui.py b/picasso_workflow/gui.py index d4b8a0a..ee2df76 100644 --- a/picasso_workflow/gui.py +++ b/picasso_workflow/gui.py @@ -1332,6 +1332,27 @@ def render(self): "description": "angle", "required": False, }, + "generate_active_rois": { + "type": "bool", + "description": "Whether to generate density-driven active site zoom-in previews next to the overview image", + "default": True, + "required": False, + }, + "n_active_rois": { + "type": "int", + "description": "Number of high-density active sites (ROIs) to generate", + "min": 1, + "max": 25, + "default": 4, + "required": False, + }, + "colormap": { + "type": "str", + "description": "Colormap of the generated images", + "options": ["magma", "hot", "inferno", "viridis", "gray"], + "default": "magma", + "required": False, + }, } results_spec = { @@ -1356,11 +1377,19 @@ def render(self): "type": "str", "description": "filepath to full FOV rendering", }, + "fp_scene_fullfov_unmarked": { + "type": "str", + "description": "filepath to full FOV rendering without outlines", + }, "fp_scene_ctrmass": { "type": "str", "description": "filepath to center of mass zoom rendering \ (conditional, only if ctrmass_fov_nm provided)", }, + "fp_scene_rois": { + "type": "list", + "description": "list of filepaths to the density-driven active site Zoom-In previews", + }, } return parameters_spec, results_spec @@ -8425,9 +8454,7 @@ def __init__(self): self.autodetect_datasets_checkbox.stateChanged.connect( self._on_autodetect_toggled ) - self.files_box.addWidget( - self.autodetect_datasets_checkbox, 0, 0, 1, 3 - ) + self.files_box.addWidget(self.autodetect_datasets_checkbox, 0, 0, 1, 3) # Create button container for dynamic button layout self.file_buttons_widget = QtWidgets.QWidget() @@ -9282,6 +9309,10 @@ def _update_file_buttons(self, workflow_type_index): if item.widget(): item.widget().deleteLater() + self.find_files_button = QtWidgets.QPushButton("Find in folder") + self.find_files_button.clicked.connect(self.find_dataset_files) + self.file_buttons_layout.addWidget(self.find_files_button) + if workflow_type_index == 0: # Single Workflow self.add_files_button = QtWidgets.QPushButton("Add files") self.add_files_button.clicked.connect(self.add_files) @@ -9331,6 +9362,93 @@ def _update_file_buttons(self, workflow_type_index): self.file_buttons_layout.addWidget(self.remove_dataset_button) self.file_buttons_layout.addWidget(self.clear_tree_button) + def find_dataset_files(self): + """Find dataset files (.tif, .ome.tif, .nd2) in a selected folder.""" + folder = QtWidgets.QFileDialog.getExistingDirectory( + self, + "Select Data Folder", + "", + ) + if folder: + from picasso_workflow.metaworkflow import find_dnapaint_raw + + datasets, _ = find_dnapaint_raw(folder) + if datasets: + workflow_type_index = self.workflow_type.currentIndex() + if workflow_type_index == 0: + # Single workflow + for key, paths in datasets.items(): + row = self.files_table.rowCount() + self.files_table.insertRow(row) + self.files_table.setItem( + row, 0, QtWidgets.QTableWidgetItem(key) + ) + if isinstance(paths, list): + if len(paths) == 1: + self.files_table.setItem( + row, + 1, + QtWidgets.QTableWidgetItem(paths[0]), + ) + else: + self.files_table.setItem( + row, + 1, + QtWidgets.QTableWidgetItem(repr(paths)), + ) + else: + self.files_table.setItem( + row, 1, QtWidgets.QTableWidgetItem(paths) + ) + else: + # Aggregation / Investigation workflow + if not self.tree_data.get("channels"): + self.tree_data["channels"] = ["ch1"] + + for key, paths in datasets.items(): + # Ensure unique key + unique_key = key + counter = 1 + while unique_key in self.tree_data.get("datasets", []): + unique_key = f"{key}_{counter}" + counter += 1 + + if "datasets" not in self.tree_data: + self.tree_data["datasets"] = [] + if "conditions" not in self.tree_data: + self.tree_data["conditions"] = {} + if "file_paths" not in self.tree_data: + self.tree_data["file_paths"] = {} + + self.tree_data["datasets"].append(unique_key) + self.tree_data["conditions"][unique_key] = "" + self.tree_data["file_paths"][unique_key] = {} + + # Assign to the first channel + target_channel = self.tree_data["channels"][0] + + if isinstance(paths, list): + if len(paths) == 1: + self.tree_data["file_paths"][unique_key][ + target_channel + ] = paths[0] + else: + self.tree_data["file_paths"][unique_key][ + target_channel + ] = repr(paths) + else: + self.tree_data["file_paths"][unique_key][ + target_channel + ] = paths + + self._populate_tree_from_data() + else: + QtWidgets.QMessageBox.information( + self, + "No files found", + f"Could not find any supported dataset files in {folder}", + ) + def _set_widgets_enabled(self, enabled): """Enable or disable file and module widgets based on results folder selection.""" self.workflow_type.setEnabled(enabled) @@ -9446,13 +9564,17 @@ def on_results_folder_dropped(self, folder): def on_template_changed(self, template_name): """Load a template""" try: - template_folder = CONFIG["Templates"][template_name] - if template_folder[:2] == 'r"' or template_folder[:2] == "r'": - template_folder = template_folder[2:-1] + template_path = CONFIG["Templates"][template_name] + if template_path[:2] == 'r"' or template_path[:2] == "r'": + template_path = template_path[2:-1] + + # If path points to a file, use its directory + if template_path.endswith(".py"): + template_folder = os.path.dirname(template_path) + else: + template_folder = template_path except KeyError: return - # # Enable widgets when a folder is selected - # self._set_widgets_enabled(True) # Search for YAML files and load file list self._load_yaml_file_list(template_folder) @@ -9882,8 +10004,10 @@ def search_assignments(statements, variables_dict): # ) # Load aggregation workflow if found - if workflow_modules_agg is not None and isinstance( - workflow_modules_agg, list + if ( + workflow_modules_agg is not None + and isinstance(workflow_modules_agg, list) + and len(workflow_modules_agg) > 0 ): self._populate_workflow_from_definition( workflow_modules_agg, @@ -10024,8 +10148,10 @@ def _load_workflow_definition(self, folder): ) # Load aggregation workflow if present - if workflow_modules_agg is not None and isinstance( - workflow_modules_agg, list + if ( + workflow_modules_agg is not None + and isinstance(workflow_modules_agg, list) + and len(workflow_modules_agg) > 0 ): self._populate_workflow_from_definition( workflow_modules_agg, diff --git a/picasso_workflow/metaworkflow.py b/picasso_workflow/metaworkflow.py index fbd0dab..4095abd 100644 --- a/picasso_workflow/metaworkflow.py +++ b/picasso_workflow/metaworkflow.py @@ -358,19 +358,11 @@ def parse_source( def find_dnapaint_raw(working_folder): - datasets = {} - for root, dirs, files in os.walk(working_folder): - for file in files: - match = re.search(r"_MMStack_Pos(\d+).ome.tif", file) - if match is not None: - key = os.path.split(root)[-1] - datasets[key] = os.path.join(root, file) - continue - match = re.search(r"_NDTiffStack.tif", file) - if match is not None: - key = os.path.split(root)[-1] - datasets[key] = os.path.join(root, file) - continue + from picasso_workflow import util + from picasso import io + + datasets = util.find_raw_movies(working_folder) + dest_file = os.path.join(working_folder, "src_loc.yaml") io.save_info(dest_file, [datasets]) return datasets, dest_file @@ -552,17 +544,17 @@ def __init__( self.analysis_name = os.path.split(working_folder)[-1] super().__init__( - analysis_name, - working_folder, - confluence_url, - confluence_space, - confluence_token, - base_page, - confluence_username, - dest_machine, - always_save, - profile_space, - profile_basepage, + analysis_name=analysis_name, + working_folder=working_folder, + confluence_url=confluence_url, + confluence_space=confluence_space, + confluence_token=confluence_token, + confluence_username=confluence_username, + base_page=base_page, + dest_machine=dest_machine, + always_save=always_save, + profile_space=profile_space, + profile_basepage=profile_basepage, ) def prepare_analysis( @@ -683,17 +675,18 @@ def __init__( self.analysis_name = analysis_name super().__init__( - analysis_name, - working_folder, - confluence_url, - confluence_space, - confluence_token, - confluence_username, - base_page, - dest_machine, - always_save, - profile_space, - profile_basepage, + analysis_name=analysis_name, + working_folder=working_folder, + confluence_url=confluence_url, + confluence_space=confluence_space, + confluence_token=confluence_token, + confluence_username=confluence_username, + base_page=base_page, + investigation_description=investigation_description, + dest_machine=dest_machine, + always_save=always_save, + profile_space=profile_space, + profile_basepage=profile_basepage, ) def prepare_analysis( @@ -844,18 +837,18 @@ def __init__( self.analysis_name = analysis_name super().__init__( - analysis_name, - working_folder, - confluence_url, - confluence_space, - confluence_token, - base_page, - confluence_username, - investigation_description, - dest_machine, - always_save, - profile_space, - profile_basepage, + analysis_name=analysis_name, + working_folder=working_folder, + confluence_url=confluence_url, + confluence_space=confluence_space, + confluence_token=confluence_token, + confluence_username=confluence_username, + base_page=base_page, + investigation_description=investigation_description, + dest_machine=dest_machine, + always_save=always_save, + profile_space=profile_space, + profile_basepage=profile_basepage, ) def prepare_sglcell_analysis(self, get_workflow_modules): diff --git a/picasso_workflow/outpost_modules/render.py b/picasso_workflow/outpost_modules/render.py index edb73d1..549bc64 100644 --- a/picasso_workflow/outpost_modules/render.py +++ b/picasso_workflow/outpost_modules/render.py @@ -85,7 +85,11 @@ def render_single_channel(kwargs, locs, n_group_colors=8, cmap="magma"): group_colors = get_group_color(locs, n_group_colors) locs = [locs[group_colors == _] for _ in range(n_group_colors)] return render_multi_channel(kwargs, locs=locs) - n_locs, image = render.render(locs, **kwargs) + + render_args = kwargs.copy() + render_args.pop("cmap", None) + render_args.pop("n_group_colors", None) + n_locs, image = render.render(locs, **render_args) # adjust contrast and convert to 8 bits image = scale_contrast([image])[0] @@ -93,14 +97,14 @@ def render_single_channel(kwargs, locs, n_group_colors=8, cmap="magma"): # paint locs using the colormap of choice (Display Settings # Dialog) - cmap = np.uint8(np.round(255 * plt.get_cmap(cmap)(np.arange(256)))) + cmap = np.uint8(np.round(255 * plt.colormaps[cmap](np.arange(256)))) - # return a 4 channel (rgb and alpha) array + # return a 4 channel (rgba) array Y, X = image.shape bgra = np.zeros((Y, X, 4), dtype=np.uint8, order="C") - bgra[..., 0] = cmap[:, 2][image] - bgra[..., 1] = cmap[:, 1][image] - bgra[..., 2] = cmap[:, 0][image] + bgra[..., 0] = cmap[:, 0][image] # R + bgra[..., 1] = cmap[:, 1][image] # G + bgra[..., 2] = cmap[:, 2][image] # B return bgra @@ -150,13 +154,16 @@ def render_multi_channel( int(np.ceil(kwargs["oversampling"] * (y_max - y_min))), ) # if single channel is rendered + render_args = kwargs.copy() + render_args.pop("cmap", None) + render_args.pop("n_group_colors", None) if len(locs) == 1: - renderings = [render.render(_, **kwargs) for _ in locs] + renderings = [render.render(_, **render_args) for _ in locs] else: renderings = [ - render.render(_, **kwargs) for i, _ in enumerate(locs) + render.render(_, **render_args) for i, _ in enumerate(locs) ] # renders only channels that are checked in dataset dialog - # renderings = [render.render(_, **kwargs) for _ in locs] + # renderings = [render.render(_, **render_args) for _ in locs] # n_locs = sum([_[0] for _ in renderings]) image = np.array([_[1] for _ in renderings]) @@ -207,27 +214,21 @@ def scale_contrast(images): Parameters ---------- - images : list of np.arrays + images : list of np.arrays or np.ndarray Arrays with rendered locs (grayscale) Returns ------- - list of np.arrays - Scaled images + np.ndarray + Scaled images in [0, 1] """ - - upper = ( - min( - [ - _.max() - for _ in images # if no locs were clustered - if _.max() != 0 # the maximum value in image is 0.0 - ] - ) - / 4 - ) - # upper = INITIAL_REL_MAXIMUM * max_ - + images = np.asarray(images) + all_nonzero = images[images > 0].ravel() + if len(all_nonzero) == 0: + return images + # Map the 80th percentile of non-background pixels to full white, + # saturating the top 20% and scaling everything else proportionally brighter. + upper = np.percentile(all_nonzero, 80) images = images / upper images[~np.isfinite(images)] = 0 images = np.minimum(images, 1.0) @@ -296,6 +297,8 @@ def plot_scene( x_offset=0, y_offset=0, title="", + figsize=None, + dpi=100, ): """Plot a scene in the locs Args: @@ -303,6 +306,11 @@ def plot_scene( optional keys: oversampling, viewport, blur_method, min_blur_width, ang, n_group_colors, cmap + figsize : tuple, default None + (width, height) in inches; None uses matplotlib's default (6.4 x 4.8). + dpi : int, default 100 + Output pixel density. Output pixels = figsize * dpi. + Increase to get a larger saved image without changing layout. """ if not isinstance(channel_locs, list): channel_locs = [channel_locs] @@ -319,9 +327,9 @@ def plot_scene( for locs in channel_locs ] logger.debug(f"#locs: {nlocs}; #groups{ngroups}") - fig, ax = plt.subplots() + fig, ax = plt.subplots(figsize=figsize, dpi=dpi) if fp is not None: - fig.savefig(fp) + fig.savefig(fp, dpi=dpi) return fig, ax # overwrite default kwargs with input kwargs if render_kwargs is not None: @@ -332,22 +340,38 @@ def plot_scene( logger.debug(f"rendering locs with offset {(x_offset, y_offset)} nm") bgra = render_scene(kwargs, channel_locs) + H, W = bgra.shape[:2] + + # Auto-size the figure so every rendered pixel maps 1:1 to an output pixel. + # This avoids any resampling by matplotlib and gives true STORM resolution. + # An explicit figsize overrides this (useful for thumbnails / ROI images). + if figsize is None: + fig_w, fig_h = W / dpi, H / dpi + use_auto = True + else: + fig_w, fig_h = figsize + use_auto = False + + fig, ax = plt.subplots(figsize=(fig_w, fig_h), dpi=dpi) + if use_auto: + fig.subplots_adjust(left=0, right=1, bottom=0, top=1) - fig, ax = plt.subplots() ax.imshow( bgra, - aspect="equal", - origin="lower", + aspect="auto" if use_auto else "equal", + interpolation="none", + origin="upper", extent=[ x_offset / 1000, - (bgra.shape[1] * image_px_size + x_offset) / 1000, + (W * image_px_size + x_offset) / 1000, + (H * image_px_size + y_offset) / 1000, y_offset / 1000, - (bgra.shape[0] * image_px_size + y_offset) / 1000, ], ) - ax.set_xlabel("x [µm]") - ax.set_ylabel("y [µm]") - ax.set_title(title) + ax.axis("off") if fp is not None: - fig.savefig(fp) + if use_auto: + fig.savefig(fp, pad_inches=0, dpi=dpi) + else: + fig.savefig(fp, bbox_inches="tight", pad_inches=0, dpi=dpi) return fig, ax diff --git a/picasso_workflow/outpost_modules/undrift_rsso.py b/picasso_workflow/outpost_modules/undrift_rsso.py index e4a172b..f734ab6 100644 --- a/picasso_workflow/outpost_modules/undrift_rsso.py +++ b/picasso_workflow/outpost_modules/undrift_rsso.py @@ -1961,7 +1961,7 @@ def _plot_shift_trajectory_2d( """ import matplotlib.pyplot as plt from matplotlib.collections import LineCollection - from matplotlib import cm + from matplotlib import colormaps import matplotlib.colors as mcolors n_iterations = len(iteration_history) @@ -2010,7 +2010,7 @@ def _plot_shift_trajectory_2d( mincnt=1, linewidths=0.2, zorder=1) # Color map for iteration progression - cmap = cm.get_cmap('RdYlGn_r') # Red (start) -> Yellow -> Green (end) + cmap = colormaps['RdYlGn_r'] # Red (start) -> Yellow -> Green (end) # Plot sample frame trajectories with color gradient for idx in sample_indices: diff --git a/picasso_workflow/tests/test_confluence.py b/picasso_workflow/tests/test_confluence.py index e6299ea..be09bcf 100644 --- a/picasso_workflow/tests/test_confluence.py +++ b/picasso_workflow/tests/test_confluence.py @@ -732,12 +732,40 @@ def random_val(self): # @unittest.skip("") def render(self): parameters = {} - results = { - "start time": "now", - "duration": 4.12, - "success": True, - } - self.cr.render(0, parameters, results) + import tempfile + import shutil + import os + temp_dir = tempfile.mkdtemp() + try: + fp_fullfov = os.path.join(temp_dir, "fullfov.png") + with open(fp_fullfov, "wb") as f: + f.write(b"PNG mock data") + + fp_ctrmass = os.path.join(temp_dir, "ctrmass.png") + with open(fp_ctrmass, "wb") as f: + f.write(b"PNG mock data") + + fp_scene_rois = [] + for i in range(5): + fp_roi = os.path.join(temp_dir, f"roi_{i+1}.png") + with open(fp_roi, "wb") as f: + f.write(b"PNG mock data") + fp_scene_rois.append(fp_roi) + + results = { + "start time": "now", + "duration": 4.12, + "success": True, + "fp_scene_fullfov": fp_fullfov, + "fp_scene_ctrmass": fp_ctrmass, + "fp_scene_rois": fp_scene_rois, + } + self.cr.render(0, parameters, results) + finally: + try: + shutil.rmtree(temp_dir) + except Exception: + pass # clean up pgid, pgtitle = self.cr.ci.get_page_properties( diff --git a/picasso_workflow/tests/test_util.py b/picasso_workflow/tests/test_util.py index e7d3dd6..fa71539 100644 --- a/picasso_workflow/tests/test_util.py +++ b/picasso_workflow/tests/test_util.py @@ -215,3 +215,39 @@ def test_06_valid_expression(self): expression = "* 3.1415" val = util.is_valid_expression(expression) assert val + + def test_07_get_movie_groups(self): + from picasso_workflow.util import get_movie_groups + + paths = [ + "movie_1.ome.tif", + "movie_0.ome.tif", + "movie_2.ome.tif", + "other.ome.tif", + ] + groups = get_movie_groups(paths, ".ome.tif") + assert set(groups.keys()) == {"movie", "other"} + assert groups["movie"] == [ + "movie_0.ome.tif", + "movie_1.ome.tif", + "movie_2.ome.tif", + ] + assert groups["other"] == ["other.ome.tif"] + + def test_08_find_dnapaint_raw(self): + import tempfile + from pathlib import Path + from picasso_workflow.metaworkflow import find_dnapaint_raw + + with tempfile.TemporaryDirectory() as tempdir: + temp_path = Path(tempdir) + (temp_path / "cond1" / "pos10").mkdir(parents=True) + (temp_path / "cond1" / "pos10" / "movie_0.ome.tif").touch() + (temp_path / "cond1" / "pos2").mkdir(parents=True) + (temp_path / "cond1" / "pos2" / "movie_0.ome.tif").touch() + + datasets, _ = find_dnapaint_raw(tempdir) + keys = list(datasets.keys()) + assert "pos2" in keys + assert "pos10" in keys + assert keys.index("pos2") < keys.index("pos10") diff --git a/picasso_workflow/util.py b/picasso_workflow/util.py index 01fa72e..b7aa8ee 100644 --- a/picasso_workflow/util.py +++ b/picasso_workflow/util.py @@ -2731,3 +2731,121 @@ def stripplot(data, positions, jitter, ax, color, alpha=1): x = pos * np.ones(len(d)) x += np.random.uniform(-jitter / 2, jitter / 2, size=len(d)) ax.scatter(x, d, color=color, alpha=alpha) + + +def get_movie_groups(paths, extension): + """ + Groups files based on basename and index, supporting variable extensions. + + Args: + paths: list of filenames + extension: the extension to match (e.g. '.tif' or '.ome.tif') + Returns: + dict: mapping from base name to list of file paths (sorted by index) + """ + import re + + groups = {} + if not paths: + return groups + + ext_pattern = re.escape(extension) + pattern = re.compile(rf"(.*?)(?:_(\d+))?{ext_pattern}$") + + match_infos = [] + for path in paths: + match = pattern.match(path) + if match: + base, index = match.groups() + match_infos.append( + { + "path": path, + "base": base, + "index": int(index) if index else 0, + } + ) + + # Grouping logic + basenames = {m["base"] for m in match_infos} + for base in basenames: + group_items = [m for m in match_infos if m["base"] == base] + # Sort by index + group_items.sort(key=lambda x: x["index"]) + groups[base] = [item["path"] for item in group_items] + + return groups + + +def find_raw_movies(working_folder): + """ + Recursively finds raw movie files (.tif, .ome.tif, .nd2) in a folder. + + Args: + working_folder: path to search + Returns: + dict: mapping from dataset name to path or list of paths + """ + import os + import fnmatch + from pathlib import Path + + datasets = {} + + for root, dirs, files in os.walk(working_folder): + p = Path(root) + + # Check what extensions exist here + has_nd2 = list(p.glob("*.nd2")) + has_ometif = list(p.glob("*.ome.tif")) + # .tif check needs to exclude .ome.tif to be accurate + has_tif = [ + f for f in p.glob("*.tif") if not f.name.endswith(".ome.tif") + ] + + found_types = [] + if has_nd2: + found_types.append(".nd2") + if has_ometif: + found_types.append(".ome.tif") + if has_tif: + found_types.append(".tif") + + if not found_types: + continue + + # Priority: .nd2 > .ome.tif > .tif + ext = found_types[0] + + if ext == ".nd2": + nd2_files = sorted(fnmatch.filter(os.listdir(root), "*.nd2")) + for f in nd2_files: + stem = Path(f).stem + # Use parent folder name as key if unique, otherwise include stem + key = p.name if p.name else stem + if len(nd2_files) > 1: + key = f"{p.name}_{stem}" + datasets[key] = str(p / f) + else: + # tif or ome.tif + tif_files = sorted([f.name for f in p.glob(f"*{ext}")]) + groups = get_movie_groups(tif_files, ext) + for base, group_paths in groups.items(): + # Use directory name as key if it's the only group, otherwise use base + if len(groups) == 1: + key = p.name + else: + key = base + + full_paths = [str(p / fname) for fname in group_paths] + datasets[key] = full_paths[0] + + # Natural sort the datasets by key + import re + + def natsort_key(s): + return [ + int(t) if t.isdigit() else t.lower() for t in re.split(r"(\d+)", s) + ] + + sorted_keys = sorted(datasets.keys(), key=natsort_key) + return {k: datasets[k] for k in sorted_keys}