From 197b3cf675d5748fdd9a7ca64a644597fa8605ef Mon Sep 17 00:00:00 2001 From: Alexfebo Date: Wed, 2 Apr 2025 16:49:35 +0200 Subject: [PATCH 01/27] Alex B + C modif OK on Add basic MINFLUX colour stats plotting Alex B modif of MINFLUX.py (LocRate + LocError) Alex C modif of NPCcalcLM --- PYMEcs/Analysis/MINFLUX.py | 31 ++++++++++++++-- PYMEcs/experimental/MINFLUX.py | 62 +++++++++++++++++++++++++++++--- PYMEcs/experimental/NPCcalcLM.py | 51 ++++++++++++++++++++++---- 3 files changed, 131 insertions(+), 13 deletions(-) diff --git a/PYMEcs/Analysis/MINFLUX.py b/PYMEcs/Analysis/MINFLUX.py index 6e7bc93..a1c90db 100644 --- a/PYMEcs/Analysis/MINFLUX.py +++ b/PYMEcs/Analysis/MINFLUX.py @@ -3,6 +3,8 @@ import numpy as np import matplotlib.pyplot as plt from PYMEcs.pyme_warnings import warn +import pandas as pd + def propcheck_density_stats(ds,warning=True): for prop in ['clst_area','clst_vol','clst_density','clst_stdz']: @@ -51,7 +53,6 @@ def plot_density_stats(ds,objectID='dbscanClumpID',scatter=False): ax1[1].scattered_boxplot(sz,labels=['Stddev Z'],showmeans=True) plt.tight_layout() -import pandas as pd from PYMEcs.misc.matplotlib import boxswarmplot def plot_density_stats_sns(ds,objectID='dbscanClumpID'): if not propcheck_density_stats(ds): @@ -78,10 +79,17 @@ def plot_density_stats_sns(ds,objectID='dbscanClumpID'): return dens + +# copied from experimental>NPCcalcLM.py to try getting the filename +# this should be a backwards compatible way to access the main filename associated with the pipeline/datasource + def plot_stats_minflux(deltas, durations, tdiff, tdmedian, efo_or_dtovertime, times, showTimeAverages=False, dsKey=None, areaString=None): + # --- Create the figure (plot with 2x2 subplots) --- fig, (ax1, ax2) = plt.subplots(2, 2) + + # --- Compute the TBT Median --- h = ax1[0].hist(deltas,bins=40) dtmedian = np.median(deltas) ax1[0].plot([dtmedian,dtmedian],[0,h[0].max()]) @@ -93,6 +101,7 @@ def plot_stats_minflux(deltas, durations, tdiff, tdmedian, efo_or_dtovertime, ti ax1[0].text(0.95, 0.6, areaString, horizontalalignment='right', verticalalignment='bottom', transform=ax1[0].transAxes) + # --- Compute the Duration Median --- h = ax1[1].hist(durations,bins=40) durmedian = np.median(durations) ax1[1].plot([durmedian,durmedian],[0,h[0].max()]) @@ -100,6 +109,7 @@ def plot_stats_minflux(deltas, durations, tdiff, tdmedian, efo_or_dtovertime, ti ax1[1].text(0.95, 0.8, 'median %.0f ms' % (1e3*durmedian), horizontalalignment='right', verticalalignment='bottom', transform=ax1[1].transAxes) + # --- Compute the Time between Localisations Median --- h = ax2[0].hist(tdiff,bins=50,range=(0,0.1)) ax2[0].plot([tdmedian,tdmedian],[0,h[0].max()]) # these are times between repeated localisations of the same dye molecule @@ -107,7 +117,7 @@ def plot_stats_minflux(deltas, durations, tdiff, tdmedian, efo_or_dtovertime, ti ax2[0].text(0.95, 0.8, 'median %.0f ms' % (1e3*tdmedian), horizontalalignment='right', verticalalignment='bottom', transform=ax2[0].transAxes) - + # --- Compute the TBT running time average --- if showTimeAverages: ax2[1].plot(times,efo_or_dtovertime) ax2[1].set_xlabel('TBT running time average [s]') @@ -122,7 +132,22 @@ def plot_stats_minflux(deltas, durations, tdiff, tdmedian, efo_or_dtovertime, ti plt.suptitle('Location rate analysis from datasource %s' % dsKey) plt.tight_layout() - + # --- Calculate dimensions and area of the image --- + dimension = areaString.split(' ')[1] + area = float(dimension.split('x')[0])*float(dimension.split('x')[1]) + + # --- Save medians values to csv --- (Alex B addition) + df = pd.DataFrame({ + "Metric": ["TBT (Time Between Traces)", "dimension", "area", "Trace Duration", "Time Between Localizations"], + "Median": [dtmedian, dimension, area, durmedian * 1e3, tdmedian * 1e3], # Convert seconds -> milliseconds where needed + "Unit": ["s","um^2","um^2", "ms", "ms"] }) + # --- Show the head of df in the console --- (Alex B addition) + print(df.head()) + + # --- Save the dataframe to a csv file in the user specified path --- (Alex B addition) + saving_path = input("name the file to save (will be save in the current location):") + df.to_csv(saving_path + ".csv", index=False) + # this function assumes a pandas dataframe # the pandas frame should generally be generated via the function minflux_npy2pyme from PYMEcs.IO.MINFLUX def analyse_locrate_pdframe(datain,use_invalid=False,showTimeAverages=True): diff --git a/PYMEcs/experimental/MINFLUX.py b/PYMEcs/experimental/MINFLUX.py index 8133f79..b6ab531 100644 --- a/PYMEcs/experimental/MINFLUX.py +++ b/PYMEcs/experimental/MINFLUX.py @@ -3,6 +3,8 @@ import wx from PYMEcs.pyme_warnings import warn + +# --- Define the Localisation Error Analysis function --- def plot_errors(pipeline): if not 'coalesced_nz' in pipeline.dataSources: warn('no data source named "coalesced_nz" - check recipe and ensure this is MINFLUX data') @@ -11,23 +13,32 @@ def plot_errors(pipeline): pipeline.selectDataSource('coalesced_nz') p = pipeline clumpSize = p['clumpSize'] + + # --- Prepare the plot --- plt.figure() plt.subplot(221) + + # --- Plot the coalesced error in x, y (and z if available) --- if 'error_z' in pipeline.keys(): plt.boxplot([p['error_x'],p['error_y'],p['error_z']],labels=['error_x','error_y','error_z']) else: plt.boxplot([p['error_x'],p['error_y']],labels=['error_x','error_y']) plt.ylabel('loc error - coalesced (nm)') pipeline.selectDataSource('with_clumps') + + # --- Plot the Photon number and background rate --- plt.subplot(222) - bp_dict = plt.boxplot([p['nPhotons'],p['fbg']],labels=['photons','background rate']) - for line in bp_dict['medians']: + bp_dict1 = plt.boxplot([p['nPhotons'],p['fbg']],labels=['photons','background rate']) + for line in bp_dict1['medians']: # get position data for median line x, y = line.get_xydata()[0] # top of median line # overlay median value plt.text(x, y, '%.0f' % y, - horizontalalignment='right') # draw above, centered + horizontalalignment='right') # draw above, centered uids, idx = np.unique(p['clumpIndex'],return_index=True) + #print(f"bp_dict1: {bp_dict1['medians'][0].get_xydata()[0][1]}") + + # --- Plot the error in x, y, (and z if available) --- plt.subplot(223) if 'error_z' in pipeline.keys(): plt.boxplot([p['error_x'][idx],p['error_y'][idx],p['error_z'][idx]], @@ -35,6 +46,8 @@ def plot_errors(pipeline): else: plt.boxplot([p['error_x'][idx],p['error_y'][idx]],labels=['error_x','error_y']) plt.ylabel('loc error - raw (nm)') + + # --- Plot the clump size --- plt.subplot(224) bp_dict = plt.boxplot([clumpSize],labels=['clump size']) for line in bp_dict['medians']: @@ -42,10 +55,51 @@ def plot_errors(pipeline): x, y = line.get_xydata()[0] # top of median line # overlay median value plt.text(x, y, '%.0f' % y, - horizontalalignment='right') # draw above, centered + horizontalalignment='right') # draw above, centered + + # Display the plot plt.tight_layout() pipeline.selectDataSource(curds) + # --- Calculate the mean and median of the clump size --- (Alex B addition) + # mean_photon = np.mean(p['nPhotons']) + median_photon = np.median(p['nPhotons']) + + # mean_bg = np.mean(p['fbg']) + median_bg = np.median(p['fbg']) + + # mean_clump = np.mean(p['clumpSize']) + pipeline.selectDataSource('coalesced_nz') + median_clump = np.median(p['clumpSize']) + + median_error_x = np.median(p['error_x']) + median_error_y = np.median(p['error_y']) + median_error_z = np.median(p['error_z']) + + # median_error_x_coalesced = np.median(p['error_x']) + # median_error_y_coalesced = np.median(p['error_y']) + # median_error_z_coalesced = np.median(p['error_z']) + + # --- Print statements --- (Alex B addition) + print("Raw median:", np.median(p['clumpSize'])) + print("Number of data points:", len(p['clumpSize'])) + print("Boxplot median:", [line.get_xydata()[0][1] for line in bp_dict['medians']]) + + # --- Save the values to csv --- (Alex B addition) + df = pd.DataFrame({ + "Metric": ["Photons", "Background", "Clump Size", "Error X", "Error Y", "Error Z"], # "Error X", "Error Y", "Error Z" + "Median": [median_photon, median_bg, median_clump, median_error_x, median_error_y ,median_error_z ], + "Unit": ["","","","nm","nm","nm" ] }) + # --- Show the head of df in the console --- (Alex B addition) + print(df.head()) + + # --- Save the dataframe to a csv file in the user specified path --- (Alex B addition) + saving_name = input("filename to append to (will be save in the current location)?") + # append the data to the csv file + df.to_csv(f"{saving_name}.csv", mode='a', header=False, index=False) + # # --- Save as a standalone csv file --- (Alex B addition) + # df.to_csv(f"{saving_name}_standalone.csv", index=False) + from PYMEcs.misc.matplotlib import boxswarmplot import pandas as pd def _plot_clustersize_counts(cts, ctsgt1, xlabel='Cluster Size', wintitle=None, bigCfraction=None,bigcf_percluster=None, **kwargs): diff --git a/PYMEcs/experimental/NPCcalcLM.py b/PYMEcs/experimental/NPCcalcLM.py index 3d19d57..069f679 100644 --- a/PYMEcs/experimental/NPCcalcLM.py +++ b/PYMEcs/experimental/NPCcalcLM.py @@ -385,6 +385,7 @@ def On3DNPCaddTemplates(self, event=None): self.visFr.add_layer(layer) def OnNPC3DSaveMeasurements(self, event=None): + import pandas as pd pipeline = self.visFr.pipeline npcs = findNPCset(pipeline) if npcs is None or 'measurements' not in dir(npcs): @@ -398,14 +399,52 @@ def OnNPC3DSaveMeasurements(self, event=None): return fpath = fdialog.GetPath() - meas = np.array(npcs.measurements, dtype='i') import pandas as pd df = pd.DataFrame({'Ntop_NPC3D': meas[:, 0], 'Nbot_NPC3D': meas[:, 1]}) - - from pathlib import Path - with open(fpath, 'w') as f: - f.write('# threshold %d, source data file %s\n' % - (self.NPCsettings.SegmentThreshold_3D,Path(pipeline_filename(pipeline)).name)) + meas = np.array(npcs.measurements, dtype='i') + entries = len(np.unique(pipeline.objectID)) + MINFLUX_filename = [pipeline.mdh.getEntry('MINFLUX.Filename')]*entries + NPC_threshold = [self.NPCsettings.SegmentThreshold_3D]*entries + fshortening = [pipeline.mdh.getEntry('MINFLUX.Foreshortening')]*entries + t_min = [np.round(np.min(pipeline.tim))]*entries + t_max = [np.round(np.max(pipeline.tim))]*entries + t_maxhr = [np.round(ti/3600) for ti in t_max] + + duration_hours = [np.round((np.max(pipeline.tim)-np.min(pipeline.tim))/3600,2)]*entries + duration_hours_rounded = [np.round(duration) for duration in duration_hours] + dim_z_nm = [np.round(np.max(pipeline.z)-np.min(pipeline.z),2)]*entries + NPCLE = (meas[:,0]+meas[:,1])/16 + + unique_ids, counts = np.unique(pipeline.objectID, return_counts=True) + nEvents = counts.tolist() + + df = pd.DataFrame({'Ntop_NPC3D': meas[:, 0], 'Nbot_NPC3D': meas[:, 1], "NPC_LE": NPCLE, + 'objectID': np.unique(pipeline.objectID), + 'diameters': np.round(pipeline.npcs.diam(), 4), + 'heights': np.round(pipeline.npcs.height(), 4), + 't_min': t_min, + 't_max': t_max, + 't_maxhr': t_maxhr, + 'duration_hours': duration_hours, + 'duration_hours_rounded': duration_hours_rounded, + 'nEvents': nEvents, + 'dim_z_nm': dim_z_nm, + 'foreshortening': fshortening, + 'NPC_threshold': NPC_threshold, + 'filename': MINFLUX_filename}) + + pipeline.selectDataSource('Localizations') + dim_x = (np.max(pipeline.x)-np.min(pipeline.x))/1000 + dim_y = (np.max(pipeline.y)-np.min(pipeline.y))/1000 + area_xy = [np.round(dim_x*dim_y,2)]*entries + + df.insert(3, 'area_xy', area_xy) + + try: + pipeline.selectDataSource('valid_npcs') + print("Pipeline data source successfully changed") + except: + print("Unable to change the pipeline to 'valid_npcs'") df.to_csv(fpath,index=False, mode='a') From f77d5d2fa4572b3a3955f67d33cbd04506b6c500 Mon Sep 17 00:00:00 2001 From: Alexfebo Date: Tue, 6 May 2025 12:20:10 +0200 Subject: [PATCH 02/27] Fix bug about meas moved the line: meas = np.array(npcs.measurements, dtype='i') To its correct position --- PYMEcs/experimental/NPCcalcLM.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/PYMEcs/experimental/NPCcalcLM.py b/PYMEcs/experimental/NPCcalcLM.py index a67ed5b..b33f3db 100644 --- a/PYMEcs/experimental/NPCcalcLM.py +++ b/PYMEcs/experimental/NPCcalcLM.py @@ -365,9 +365,10 @@ def OnNPC3DSaveMeasurements(self, event=None): return fpath = fdialog.GetPath() + meas = np.array(npcs.measurements, dtype='i') + import pandas as pd df = pd.DataFrame({'Ntop_NPC3D': meas[:, 0], 'Nbot_NPC3D': meas[:, 1]}) - meas = np.array(npcs.measurements, dtype='i') entries = len(np.unique(pipeline.objectID)) MINFLUX_filename = [pipeline.mdh.getEntry('MINFLUX.Filename')]*entries NPC_threshold = [self.NPCsettings.SegmentThreshold_3D]*entries From 203a500ae02caf2e76cc1a86fa3d56cdcf4710bf Mon Sep 17 00:00:00 2001 From: Alexfebo Date: Wed, 28 May 2025 13:42:25 +0200 Subject: [PATCH 03/27] Save csv and images from MINFLUX stats using "pipeline.mdh.timestamp" --- PYMEcs/Analysis/MINFLUX.py | 11 ++++++++--- PYMEcs/experimental/MINFLUX.py | 14 ++++++++------ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/PYMEcs/Analysis/MINFLUX.py b/PYMEcs/Analysis/MINFLUX.py index a1c90db..af7cec6 100644 --- a/PYMEcs/Analysis/MINFLUX.py +++ b/PYMEcs/Analysis/MINFLUX.py @@ -141,12 +141,17 @@ def plot_stats_minflux(deltas, durations, tdiff, tdmedian, efo_or_dtovertime, ti "Metric": ["TBT (Time Between Traces)", "dimension", "area", "Trace Duration", "Time Between Localizations"], "Median": [dtmedian, dimension, area, durmedian * 1e3, tdmedian * 1e3], # Convert seconds -> milliseconds where needed "Unit": ["s","um^2","um^2", "ms", "ms"] }) + # --- Show the head of df in the console --- (Alex B addition) print(df.head()) - # --- Save the dataframe to a csv file in the user specified path --- (Alex B addition) - saving_path = input("name the file to save (will be save in the current location):") - df.to_csv(saving_path + ".csv", index=False) + # --- Append the dataframe to a csv file in the user specified path --- (Alex B addition) + saving_name = input("Use the timestamp as a filename for saving (will be save in the current location):") + df.to_csv(saving_name + ".csv", mode='a', index=False, header=False) + #df.to_csv(saving_name + ".csv", index=False) + + # --- Save the figure --- (Alex B addition) + plt.savefig(saving_name + '_loc_rate.png', dpi=300, bbox_inches='tight') # this function assumes a pandas dataframe # the pandas frame should generally be generated via the function minflux_npy2pyme from PYMEcs.IO.MINFLUX diff --git a/PYMEcs/experimental/MINFLUX.py b/PYMEcs/experimental/MINFLUX.py index f7bf4d2..6c8c456 100644 --- a/PYMEcs/experimental/MINFLUX.py +++ b/PYMEcs/experimental/MINFLUX.py @@ -61,6 +61,9 @@ def plot_errors(pipeline): plt.tight_layout() pipeline.selectDataSource(curds) + # Save the plot as a png file (Alex B addition) + plt.savefig(pipeline.mdh.get('MINFLUX.TimeStamp') + '_loc_error.png', dpi=300, bbox_inches='tight') + # --- Calculate the mean and median of the clump size --- (Alex B addition) # mean_photon = np.mean(p['nPhotons']) median_photon = np.median(p['nPhotons']) @@ -93,12 +96,11 @@ def plot_errors(pipeline): # --- Show the head of df in the console --- (Alex B addition) print(df.head()) - # --- Save the dataframe to a csv file in the user specified path --- (Alex B addition) - saving_name = input("filename to append to (will be save in the current location)?") - # append the data to the csv file - df.to_csv(f"{saving_name}.csv", mode='a', header=False, index=False) - # # --- Save as a standalone csv file --- (Alex B addition) - # df.to_csv(f"{saving_name}_standalone.csv", index=False) +# --- Save as a csv file --- (Alex B addition) + csv_name = pipeline.mdh.get('MINFLUX.TimeStamp') + print(f'\ncsv name is: {csv_name}\n') # Used as a reminder for LocRate csv saving + df.to_csv(csv_name + ".csv", index=False, header=True) + # By default the file is saved on the Desktop, if a session file is used, it is saved in the same directory as the session file. from PYMEcs.misc.matplotlib import boxswarmplot import pandas as pd From 79a0b49f5b513f737f705376739e2788ef6cf748 Mon Sep 17 00:00:00 2001 From: Alexfebo Date: Tue, 3 Jun 2025 17:57:16 +0200 Subject: [PATCH 04/27] Read old and new csv temp files (colnames/datetime) --- PYMEcs/misc/utils.py | 93 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 85 insertions(+), 8 deletions(-) diff --git a/PYMEcs/misc/utils.py b/PYMEcs/misc/utils.py index 4bf4728..8149cb6 100644 --- a/PYMEcs/misc/utils.py +++ b/PYMEcs/misc/utils.py @@ -39,19 +39,96 @@ def unique_name(stem,names): import pandas as pd -def read_temp_csv(filename,timeformat='%d/%m/%Y %H:%M'): - def remap_names(name): # for slightly more robust comlumn renaming - if 'Rack' in name: +# Working solution from ChatGPT +# import re + +# def read_temp_csv(filename, timeformat='%d/%m/%Y %H:%M'): +# def remap_names(name): +# # Search for known keywords inside the column name +# if re.search(r'\bRack\b', name, re.IGNORECASE): +# return 'Rack' +# elif re.search(r'\bBox\b', name, re.IGNORECASE): +# return 'Box' +# elif re.search(r'\bStativ\b', name, re.IGNORECASE): +# return 'Stand' +# else: +# return name + +# df = pd.read_csv(filename, encoding="ISO-8859-1") +# df.columns = [remap_names(col) for col in df.columns] + +# # Identify time column +# time_col = next((col for col in df.columns if 'time' in col.lower()), None) +# if time_col is None: +# raise ValueError("No column containing 'time' found.") + +# df['datetime'] = pd.to_datetime(df[time_col], format=timeformat) +# return df + +# Test +import re + +# def read_temp_csv(filename, timeformat='%d/%m/%Y %H:%M'): +# # The next function remaps the column names based on known keywords (Rack, Box, Stativ, Time) +# def remap_names(name): +# if re.search(r'\bRack\b', name, re.IGNORECASE): +# return 'Rack' +# elif re.search(r'\bBox\b', name, re.IGNORECASE): +# return 'Box' +# elif re.search(r'\bStativ\b', name, re.IGNORECASE): +# return 'Stand' +# elif re.search(r'\bTime\b', name, re.IGNORECASE): +# return 'Time' +# else: +# return name + +# trec = pd.read_csv(filename, encoding="ISO-8859-1") +# trec.columns = [remap_names(col) for col in trec.columns] +# # Transform the time column to datetime format +# trec['datetime'] = pd.to_datetime(trec['Time'], format=timeformat) + +# # Print statement to check if the renaming worked +# print("Renamed columns:", trec.columns.tolist()) + +# return trec.rename(columns=remap_names) + + +# Test for different datetime formats +import re + +def read_temp_csv(filename, timeformat): + def remap_names(name): + if re.search(r'\bRack\b', name, re.IGNORECASE): return 'Rack' - elif 'Box' in name: + elif re.search(r'\bBox\b', name, re.IGNORECASE): return 'Box' - elif 'Stativ' in name: + elif re.search(r'\bStativ\b', name, re.IGNORECASE): return 'Stand' + elif re.search(r'\bTime\b', name, re.IGNORECASE): + return 'Time' else: return name - trec = pd.read_csv(filename,encoding = "ISO-8859-1") - trec['datetime'] = pd.to_datetime(trec['Time'],format=timeformat) - return trec.rename(columns=remap_names) + + trec = pd.read_csv(filename, encoding="ISO-8859-1") + trec.columns = [remap_names(col) for col in trec.columns] + + # Ensure timeformat is a list (even if only one format is provided) + if isinstance(timeformat, str): + timeformat = [timeformat] + + # Try all provided time formats + for fmt in timeformat: + try: + trec['datetime'] = pd.to_datetime(trec['Time'], format=fmt) + break + except ValueError: + continue + else: + raise ValueError("None of the provided time formats matched the 'Time' column.") + + return trec + + def set_diff(trec,t0): trec['tdiff'] = trec['datetime'] - t0 From 31adc8b60b5750458c6f129dbfcd6333a6a26111 Mon Sep 17 00:00:00 2001 From: Alexfebo Date: Thu, 5 Jun 2025 08:33:19 +0200 Subject: [PATCH 05/27] change look for temp file to look for temp folder --- PYMEcs/experimental/MINFLUX.py | 110 +++++++++++++++++++++++++++------ 1 file changed, 92 insertions(+), 18 deletions(-) diff --git a/PYMEcs/experimental/MINFLUX.py b/PYMEcs/experimental/MINFLUX.py index 46eeabe..b87628f 100644 --- a/PYMEcs/experimental/MINFLUX.py +++ b/PYMEcs/experimental/MINFLUX.py @@ -1017,37 +1017,111 @@ def OnMBMaddTrackLabels(self, event): def OnMINFLUXsetTempDataFile(self, event): import PYME.config as config - with wx.FileDialog(self.visFr, "Choose Temperature data file", wildcard='CSV (*.csv)|*.csv', - style=wx.FD_OPEN) as dialog: + + # CS code to find a *.csv file in the config directory + # with wx.FileDialog(self.visFr, "Choose Temperature data file", wildcard='CSV (*.csv)|*.csv', + # style=wx.FD_OPEN) as dialog: + # if dialog.ShowModal() == wx.ID_CANCEL: + # return + # fname = dialog.GetPath() + + # if config.get('MINFLUX-temperature_file') == fname: + # warn("config option 'MINFLUX-temperature_file' already set to %s" % fname) + # return # already set to this value, return + + # config.update_config({'MINFLUX-temperature_file': fname}, + # config='user', create_backup=True) + + # Modif to look for a folder instead of a file + with wx.DirDialog(self.visFr, "Choose folder containing temperature CSV files") as dialog: if dialog.ShowModal() == wx.ID_CANCEL: return - fname = dialog.GetPath() - - if config.get('MINFLUX-temperature_file') == fname: - warn("config option 'MINFLUX-temperature_file' already set to %s" % fname) - return # already set to this value, return + folder = dialog.GetPath() + + if config.get('MINFLUX-temperature_file') == folder: + warn("config option 'MINFLUX-temperature_file' already set to %s" % folder) + return - config.update_config({'MINFLUX-temperature_file': fname}, - config='user', create_backup=True) + config.update_config({'MINFLUX-temperature_file': folder}, + config='user', create_backup=True) def OnMINFLUXplotTempData(self, event): import PYME.config as config - if config.get('MINFLUX-temperature_file') is None: - warn("Need to set Temperature file location first") - return + import os + from os.path import basename + from glob import glob from PYMEcs.misc.utils import read_temp_csv, set_diff, timestamp_to_datetime - mtemps = read_temp_csv(config.get('MINFLUX-temperature_file'), - timeformat=config.get('MINFLUX-temperature_time_format',['%d.%m.%Y %H:%M:%S', # Newest format - '%d/%m/%Y %H:%M:%S' # Original format - ])) - if len(self.visFr.pipeline.dataSources) == 0: - warn("no datasources, this is probably an empty pipeline, have you loaded any data?") + + # Below commented out original code from CS + ################ + # if config.get('MINFLUX-temperature_file') is None: + # warn("Need to set Temperature file location first") + # return + ################ + + # New code from ChatGPT to select directory instead of a file. + folder = config.get('MINFLUX-temperature_file') + if folder is None or not os.path.isdir(folder): # ✅ changed: now expects a folder + warn("Need to set temperature **folder** location first") return + + # CS code t0 = self.visFr.pipeline.mdh.get('MINFLUX.TimeStamp') if t0 is None: warn("no MINFLUX TimeStamp in metadata, giving up") return + # ChatGPT addition: convert t0 from timestamp for comparing it with timedates from csv file + t0_dt = timestamp_to_datetime(t0) + + # ChatGPT: Identify the correct temperature CSV files in the folder + # Loop over CSVs to find matching file + timeformat = config.get('MINFLUX-temperature_time_format', ['%d.%m.%Y %H:%M:%S', + '%d/%m/%Y %H:%M:%S']) + candidate_files = sorted(glob(os.path.join(folder, '*.csv'))) + + selected = None + for f in candidate_files: + try: + # print(f"\nChecking file: {basename(f)}\n") # Show which file is being checked + + df = read_temp_csv(f, timeformat=timeformat) + + if 'datetime' not in df.columns: + print(f"File {f} has no 'datetime' column after parsing, skipping.") + continue + + # print(f"\nMin timestamp in file: {df['datetime'].min()}, Max: {df['datetime'].max()}\n") + + if df['datetime'].min() <= t0_dt <= df['datetime'].max(): + selected = f + break + except Exception as e: + print(f"Error reading {f}: {e}") + continue + + # cs Code + if selected is None: + warn("No temperature file found that includes the MINFLUX TimeStamp") + return + + # Read temperature data from the correct CSV file + mtemps = read_temp_csv(selected, timeformat=timeformat) + + + # Original code from CS to read temperature data + ################ + # mtemps = read_temp_csv(config.get('MINFLUX-temperature_file'), + # timeformat=config.get('MINFLUX-temperature_time_format',['%d.%m.%Y %H:%M:%S', # Newest format + # '%d/%m/%Y %H:%M:%S' # Original format + # ])) + ################ + # CS code + if len(self.visFr.pipeline.dataSources) == 0: + warn("no datasources, this is probably an empty pipeline, have you loaded any data?") + return + + # Original CS code for processing temperature data set_diff(mtemps,timestamp_to_datetime(t0)) p = self.visFr.pipeline range = (1e-3*p['t'].min(),1e-3*p['t'].max()) From 729bdb6f9a5bf53e59d13b83cd128635f842c0dd Mon Sep 17 00:00:00 2001 From: Alexfebo Date: Thu, 5 Jun 2025 14:17:29 +0200 Subject: [PATCH 06/27] Get timestamp for saving LocRate 1- Define timestamp in OnLocalisationRate (experimental/MINFLUX.py) 2- Add timestamp parameter in analyse_locrat_pdframe (Analysis/MINFLUX.py 3- add timestamp argument in the function definition plot_stats_minflux (Analysis/MINFLUX.py) --- PYMEcs/Analysis/MINFLUX.py | 14 +++++++++----- PYMEcs/experimental/MINFLUX.py | 6 +++++- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/PYMEcs/Analysis/MINFLUX.py b/PYMEcs/Analysis/MINFLUX.py index af7cec6..3492c30 100644 --- a/PYMEcs/Analysis/MINFLUX.py +++ b/PYMEcs/Analysis/MINFLUX.py @@ -84,7 +84,7 @@ def plot_density_stats_sns(ds,objectID='dbscanClumpID'): # this should be a backwards compatible way to access the main filename associated with the pipeline/datasource def plot_stats_minflux(deltas, durations, tdiff, tdmedian, efo_or_dtovertime, times, - showTimeAverages=False, dsKey=None, areaString=None): + showTimeAverages=False, dsKey=None, areaString=None, timestamp=None): # --- Create the figure (plot with 2x2 subplots) --- fig, (ax1, ax2) = plt.subplots(2, 2) @@ -146,7 +146,11 @@ def plot_stats_minflux(deltas, durations, tdiff, tdmedian, efo_or_dtovertime, ti print(df.head()) # --- Append the dataframe to a csv file in the user specified path --- (Alex B addition) - saving_name = input("Use the timestamp as a filename for saving (will be save in the current location):") + + if timestamp is None: + saving_name = input("Use the timestamp as a filename for saving (will be save in the current location):") + else: + saving_name = timestamp df.to_csv(saving_name + ".csv", mode='a', index=False, header=False) #df.to_csv(saving_name + ".csv", index=False) @@ -209,7 +213,7 @@ def analyse_locrate_pdframe(datain,use_invalid=False,showTimeAverages=True): # similar version but now using a pipeline -def analyse_locrate(data,datasource='Localizations',showTimeAverages=True): +def analyse_locrate(data,datasource='Localizations',showTimeAverages=True, timestamp=None): curds = data.selectedDataSourceKey data.selectDataSource(datasource) bins = np.arange(int(data['clumpIndex'].max())+1) + 0.5 @@ -237,8 +241,8 @@ def analyse_locrate(data,datasource='Localizations',showTimeAverages=True): if showTimeAverages: delta_averages, bin_edges, binnumber = binned_statistic(starts[:-1],deltas,statistic='mean', bins=50) delta_av_times = 0.5*(bin_edges[:-1] + bin_edges[1:]) # bin centres - plot_stats_minflux(deltas, durations_proper, tdiff, tdmedian, delta_averages, delta_av_times, showTimeAverages=True, dsKey = datasource, areaString=area_string) + plot_stats_minflux(deltas, durations_proper, tdiff, tdmedian, delta_averages, delta_av_times, showTimeAverages=True, dsKey = datasource, areaString=area_string, timestamp=timestamp) else: - plot_stats_minflux(deltas, durations_proper, tdiff, tdmedian, data['efo'], None, dsKey = datasource, areaString=area_string) + plot_stats_minflux(deltas, durations_proper, tdiff, tdmedian, data['efo'], None, dsKey = datasource, areaString=area_string, timestamp=timestamp) return (starts,deltas) diff --git a/PYMEcs/experimental/MINFLUX.py b/PYMEcs/experimental/MINFLUX.py index b87628f..f07af08 100644 --- a/PYMEcs/experimental/MINFLUX.py +++ b/PYMEcs/experimental/MINFLUX.py @@ -1179,6 +1179,10 @@ def OnLocalisationRate(self, event): pipeline = self.visFr.pipeline curds = pipeline.selectedDataSourceKey pipeline.selectDataSource(self.analysisSettings.defaultDatasourceForAnalysis) + #Added by Alex B to get timestamp and send it to plot_stats_minflux + timestamp = pipeline.mdh.get('MINFLUX.TimeStamp') + print("Timestamp from OnLocalisationRate(experimental/MINFLUX.py): ", timestamp) + # if not 'cfr' in pipeline.keys(): Error(self.visFr,'no property called "cfr", likely no MINFLUX data - aborting') pipeline.selectDataSource(curds) @@ -1189,7 +1193,7 @@ def OnLocalisationRate(self, event): return pipeline.selectDataSource(curds) - analyse_locrate(pipeline,datasource=self.analysisSettings.defaultDatasourceForAnalysis,showTimeAverages=True) + analyse_locrate(pipeline,datasource=self.analysisSettings.defaultDatasourceForAnalysis,showTimeAverages=True, timestamp=timestamp) def OnEfoAnalysis(self, event): pipeline = self.visFr.pipeline From c1906a86cd8abd40133b937dcea0d8d13c9c593d Mon Sep 17 00:00:00 2001 From: Alexfebo Date: Thu, 5 Jun 2025 15:38:41 +0200 Subject: [PATCH 07/27] keeps the same format for save LocRate and LocError Add **if statements** in plot_errors (experimental/MINFLUX.py) and plot_stats_minflux (Analysis/MINFLUX.py), to: 1- create temporary files 2- Check is LocError / LocRate temp exists 3- If exists, merges them to always keep the same final csv structure --- PYMEcs/Analysis/MINFLUX.py | 39 ++++++++++++++++++++++----- PYMEcs/experimental/MINFLUX.py | 48 ++++++++++++++++++++++++++-------- 2 files changed, 69 insertions(+), 18 deletions(-) diff --git a/PYMEcs/Analysis/MINFLUX.py b/PYMEcs/Analysis/MINFLUX.py index 3492c30..9ef758b 100644 --- a/PYMEcs/Analysis/MINFLUX.py +++ b/PYMEcs/Analysis/MINFLUX.py @@ -4,6 +4,7 @@ import matplotlib.pyplot as plt from PYMEcs.pyme_warnings import warn import pandas as pd +import os def propcheck_density_stats(ds,warning=True): @@ -145,17 +146,41 @@ def plot_stats_minflux(deltas, durations, tdiff, tdmedian, efo_or_dtovertime, ti # --- Show the head of df in the console --- (Alex B addition) print(df.head()) - # --- Append the dataframe to a csv file in the user specified path --- (Alex B addition) - + # --- Save as a csv file in the user specified path --- (Alex B addition) + # Get the timeastamp for the filename if timestamp is None: - saving_name = input("Use the timestamp as a filename for saving (will be save in the current location):") + csv_name = input("Use the timestamp as a filename for saving (will be save in the current location):") else: - saving_name = timestamp - df.to_csv(saving_name + ".csv", mode='a', index=False, header=False) - #df.to_csv(saving_name + ".csv", index=False) + csv_name = timestamp + # --- Save as a csv file --- (Alex B addition) + # Here we want to save the LocRate, however if LocError already exists we want to combine both files + # and ensure that the final csv keeps the organization regrdless on which csv was generated first (locError or LocRate). + + # variables to check if locerror and locrate csv files already exist + locerror_file = csv_name + "temp_LocError.csv" + locrate_file = csv_name + "temp_LocRate.csv" + combined_file = csv_name + ".csv" + + # Save the LocRate file + df.to_csv(locrate_file, index=False, header=True) + + # If the LocError file already exists, merge: + if os.path.exists(locerror_file): + df_rate = pd.read_csv(locrate_file) + df_error = pd.read_csv(locerror_file) + + # Combine the two dataframes in the desired order + df_combined = pd.concat([df_error, df_rate], ignore_index=True) + df_combined.to_csv(combined_file, index=False, header=True) + + # Cleanup temp files + os.remove(locerror_file) + os.remove(locrate_file) + + print(f'\ncsv name is: {csv_name}\nIf you did not load a session, csv file and figures will be saved on the desktop') # Used as a reminder for LocRate csv saving # --- Save the figure --- (Alex B addition) - plt.savefig(saving_name + '_loc_rate.png', dpi=300, bbox_inches='tight') + plt.savefig(csv_name + '_loc_rate.png', dpi=300, bbox_inches='tight') # this function assumes a pandas dataframe # the pandas frame should generally be generated via the function minflux_npy2pyme from PYMEcs.IO.MINFLUX diff --git a/PYMEcs/experimental/MINFLUX.py b/PYMEcs/experimental/MINFLUX.py index f07af08..63bc410 100644 --- a/PYMEcs/experimental/MINFLUX.py +++ b/PYMEcs/experimental/MINFLUX.py @@ -1,6 +1,7 @@ import matplotlib.pyplot as plt import numpy as np import wx +import os from PYMEcs.pyme_warnings import warn @@ -38,7 +39,14 @@ def plot_errors(pipeline): uids, idx = np.unique(p['clumpIndex'],return_index=True) #print(f"bp_dict1: {bp_dict1['medians'][0].get_xydata()[0][1]}") - # --- Plot the error in x, y, (and z if available) --- + # Get the median values for photons and background + # mean_photon = np.mean(p['nPhotons']) + median_photon = np.median(p['nPhotons']) + + # mean_bg = np.mean(p['fbg']) + median_bg = np.median(p['fbg']) + + # --- Plot the error in x, y, (and z if available) before coalescing, this is done by using [idx] --- plt.subplot(223) if 'error_z' in pipeline.keys(): plt.boxplot([p['error_x'][idx],p['error_y'][idx],p['error_z'][idx]], @@ -64,12 +72,7 @@ def plot_errors(pipeline): # Save the plot as a png file (Alex B addition) plt.savefig(pipeline.mdh.get('MINFLUX.TimeStamp') + '_loc_error.png', dpi=300, bbox_inches='tight') - # --- Calculate the mean and median of the clump size --- (Alex B addition) - # mean_photon = np.mean(p['nPhotons']) - median_photon = np.median(p['nPhotons']) - - # mean_bg = np.mean(p['fbg']) - median_bg = np.median(p['fbg']) + # --- Get the median of the clump size and errors x, y, z coalesced --- (Alex B addition) # mean_clump = np.mean(p['clumpSize']) pipeline.selectDataSource('coalesced_nz') @@ -97,9 +100,32 @@ def plot_errors(pipeline): print(df.head()) # --- Save as a csv file --- (Alex B addition) +# Here we want to save the LocError, however if LocRate already exists we want to combine both files +# and ensure that the final csv keeps the organization regrdless on which csv was generated first (locError or LocRate). + + # variables to check if locerror and locrate csv files already exist csv_name = pipeline.mdh.get('MINFLUX.TimeStamp') - print(f'\ncsv name is: {csv_name}\n') # Used as a reminder for LocRate csv saving - df.to_csv(csv_name + ".csv", index=False, header=True) + locerror_file = csv_name + "temp_LocError.csv" + locrate_file = csv_name + "temp_LocRate.csv" + combined_file = csv_name + ".csv" + + # Save the locerror file + df.to_csv(locerror_file, index=False, header=True) + + # If the locrate file already exists, merge: + if os.path.exists(locrate_file): + df_rate = pd.read_csv(locrate_file) + df_error = pd.read_csv(locerror_file) + + # Combine the two dataframes in the desired order + df_combined = pd.concat([df_error, df_rate], ignore_index=True) + df_combined.to_csv(combined_file, index=False, header=True) + + # Cleanup temp files + os.remove(locerror_file) + os.remove(locrate_file) + + print(f'\ncsv name is: {csv_name}\nIf you did not load a session, csv file and figures will be saved on the desktop') # Used as a reminder for LocRate csv saving # By default the file is saved on the Desktop, if a session file is used, it is saved in the same directory as the session file. from PYMEcs.misc.matplotlib import boxswarmplot @@ -1181,8 +1207,8 @@ def OnLocalisationRate(self, event): pipeline.selectDataSource(self.analysisSettings.defaultDatasourceForAnalysis) #Added by Alex B to get timestamp and send it to plot_stats_minflux timestamp = pipeline.mdh.get('MINFLUX.TimeStamp') - print("Timestamp from OnLocalisationRate(experimental/MINFLUX.py): ", timestamp) - # + # print("Timestamp from OnLocalisationRate(experimental/MINFLUX.py): ", timestamp) + if not 'cfr' in pipeline.keys(): Error(self.visFr,'no property called "cfr", likely no MINFLUX data - aborting') pipeline.selectDataSource(curds) From def91b0e087ccc22ba971b00a229487f808b9148 Mon Sep 17 00:00:00 2001 From: AlexFEBo Date: Thu, 12 Jun 2025 17:23:24 +0200 Subject: [PATCH 08/27] Improve readability of MFX metadata (HTML) --- PYMEcs/experimental/MINFLUX.py | 79 ++++++++++++++++++++++++---------- 1 file changed, 57 insertions(+), 22 deletions(-) diff --git a/PYMEcs/experimental/MINFLUX.py b/PYMEcs/experimental/MINFLUX.py index 6c4ad9e..2b68377 100644 --- a/PYMEcs/experimental/MINFLUX.py +++ b/PYMEcs/experimental/MINFLUX.py @@ -715,34 +715,68 @@ def OnMFXAttributes(self, event): dlg.ShowModal() else: warn("could not find zarr attribute - is this a MFX zarr file?") - + def OnMFXInfo(self, event): import io - from wx.lib.dialogs import ScrolledMessageDialog + import wx.html fres = self.visFr.pipeline.dataSources['FitResults'] if 'zarr' not in dir(fres): warn("could not find zarr attribute - is this a MFX zarr file?") return - else: - try: - mfx_attrs = fres.zarr['mfx'].attrs.asdict() - except AttributeError: - warn("could not access MFX attributes - do we have MFX data in zarr?") - return - if '_legacy' in mfx_attrs: - warn("legacy data detected - no useful MFX metadata in legacy data") - return - from PYMEcs.IO.MINFLUX import get_metadata_from_mfx_attrs - md_itr_info, md_globals = get_metadata_from_mfx_attrs(mfx_attrs) - import pprint - with io.StringIO() as output: - print(md_itr_info.to_string(show_dimensions=False,index=True,line_width=80),file=output) - print('\nMFX Globals:',file=output) - print(pprint.pformat(md_globals,indent=4),file=output) - mfx_info_str = output.getvalue() - with ScrolledMessageDialog(self.visFr, mfx_info_str, "MFX info (tentative)", size=(900,400), - style=wx.RESIZE_BORDER | wx.DEFAULT_DIALOG_STYLE ) as dlg: - dlg.ShowModal() + + # Get the metadata as before + try: + mfx_attrs = fres.zarr['mfx'].attrs.asdict() + except AttributeError: + warn("could not access MFX attributes - do we have MFX data in zarr?") + return + if '_legacy' in mfx_attrs: + warn("legacy data detected - no useful MFX metadata in legacy data") + return + + from PYMEcs.IO.MINFLUX import get_metadata_from_mfx_attrs + md_itr_info, md_globals = get_metadata_from_mfx_attrs(mfx_attrs) + + # Create an HTML dialog + dlg = wx.Dialog(self.visFr, title="MINFLUX Metadata Information", + size=(950, 600), style=wx.DEFAULT_DIALOG_STYLE | wx.RESIZE_BORDER) + + # Create HTML content + html_window = wx.html.HtmlWindow(dlg, style=wx.html.HW_SCROLLBAR_AUTO) + + # Format the DataFrame as an HTML table + html_content = "" + html_content += "

MINFLUX Iteration Parameters

" + html_content += md_itr_info.to_html( + classes='table table-striped', + float_format=lambda x: f"{x:.2f}" if isinstance(x, float) else x + ) + + # Format global parameters + html_content += "

MINFLUX Global Parameters

" + html_content += "" + for key, value in sorted(md_globals.items()): + html_content += f"" + html_content += "
{key}{value}
" + html_content += "" + + html_window.SetPage(html_content) + + # Add OK button + btn_sizer = wx.StdDialogButtonSizer() + btn = wx.Button(dlg, wx.ID_OK) + btn.SetDefault() + btn_sizer.AddButton(btn) + btn_sizer.Realize() + + # Layout + sizer = wx.BoxSizer(wx.VERTICAL) + sizer.Add(html_window, 1, wx.EXPAND | wx.ALL, 5) + sizer.Add(btn_sizer, 0, wx.ALIGN_CENTER | wx.ALL, 5) + dlg.SetSizer(sizer) + + dlg.ShowModal() + dlg.Destroy() def OnZarrToZipStore(self, event): with wx.DirDialog(self.visFr, 'Zarr to convert ...', @@ -1300,6 +1334,7 @@ def OnOrigamiSiteRecipe(self, event=None): recipe.add_modules_and_execute(modules) + pipeline.selectDataSource(corrSiteClumps) def OnOrigamiSiteTrackPlot(self, event): From 6ee21301b0f4304da3a5d23720e05a95dc1d6d66 Mon Sep 17 00:00:00 2001 From: AlexFEBo Date: Fri, 13 Jun 2025 10:42:55 +0200 Subject: [PATCH 09/27] In OnEfoAnalysis: - Set bins size to 1000 Hz (1 kHz) - Include data timestamp in fig title --- PYMEcs/experimental/MINFLUX.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/PYMEcs/experimental/MINFLUX.py b/PYMEcs/experimental/MINFLUX.py index 2b68377..376d112 100644 --- a/PYMEcs/experimental/MINFLUX.py +++ b/PYMEcs/experimental/MINFLUX.py @@ -938,7 +938,9 @@ def plot_drift(p,ax,drift_1stpass, drift_2ndpass): naxes = 2 has_drift = 'driftx' in p.keys() has_drift_ori = 'driftx_ori' in p.keys() - mbm = findmbm(p,warnings=False) + mbm = findmbm(pipeline) + if mbm is None: + return has_mbm = mbm is not None has_mbm2 = 'mbmx' in p.keys() @@ -1226,14 +1228,20 @@ def OnEfoAnalysis(self, event): pipeline = self.visFr.pipeline curds = pipeline.selectedDataSourceKey pipeline.selectDataSource(self.analysisSettings.defaultDatasourceForAnalysis) + #Added by Alex B to get timestamp and send it to plot_stats_minflux + timestamp = pipeline.mdh.get('MINFLUX.TimeStamp') if not 'efo' in pipeline.keys(): Error(self.visFr,'no property called "efo", likely no MINFLUX data or wrong datasource (CHECK) - aborting') return plt.figure() - h = plt.hist(1e-3*pipeline['efo'],bins='auto',range=(0,200)) + # Convert efo to kHz and create bins of 5 kHz (5000 Hz) + efo_khz = 1e-3*pipeline['efo'] + bins = np.arange(0, 200, 5) # Bins from 0 to 200 kHz in 5 kHz steps + h = plt.hist(efo_khz, bins=bins, range=(0,200), edgecolor='white', linewidth=0.5) dskey = pipeline.selectedDataSourceKey plt.xlabel('efo (photon rate in kHz)') - plt.title("EFO stats, using datasource '%s'" % dskey) + plt.xlim(0, 200) # Limit X axis to 200 kHz (200,000 Hz) + plt.title(f"EFO stats for {timestamp}, using datasource '{dskey}'") pipeline.selectDataSource(curds) From f2bf5eca1d3e4f4b22c83bd19ae41da0059405b7 Mon Sep 17 00:00:00 2001 From: Alexfebo Date: Fri, 13 Jun 2025 10:51:57 +0200 Subject: [PATCH 10/27] Set bins size to 1 kHz --- PYMEcs/experimental/MINFLUX.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PYMEcs/experimental/MINFLUX.py b/PYMEcs/experimental/MINFLUX.py index 376d112..e3a9b33 100644 --- a/PYMEcs/experimental/MINFLUX.py +++ b/PYMEcs/experimental/MINFLUX.py @@ -1234,9 +1234,9 @@ def OnEfoAnalysis(self, event): Error(self.visFr,'no property called "efo", likely no MINFLUX data or wrong datasource (CHECK) - aborting') return plt.figure() - # Convert efo to kHz and create bins of 5 kHz (5000 Hz) + # Convert efo to kHz and create bins of 1 kHz (1000 Hz) efo_khz = 1e-3*pipeline['efo'] - bins = np.arange(0, 200, 5) # Bins from 0 to 200 kHz in 5 kHz steps + bins = np.arange(0, 200, 1) # Bins from 0 to 200 kHz in 1 kHz steps h = plt.hist(efo_khz, bins=bins, range=(0,200), edgecolor='white', linewidth=0.5) dskey = pipeline.selectedDataSourceKey plt.xlabel('efo (photon rate in kHz)') From 274820ffd39a6ca76c762164ada51d00ee204cdb Mon Sep 17 00:00:00 2001 From: Alexfebo Date: Tue, 26 Aug 2025 17:08:26 +0200 Subject: [PATCH 11/27] Now save LocError and LocRate as individual csv files Easier to work with individual files than cross checking which one exist and create a single one. Notebook for analysis updated to read from two individual csv --- PYMEcs/Analysis/MINFLUX.py | 62 +++++++++-------- PYMEcs/experimental/MINFLUX.py | 117 ++++++++++++++++++++------------- 2 files changed, 107 insertions(+), 72 deletions(-) diff --git a/PYMEcs/Analysis/MINFLUX.py b/PYMEcs/Analysis/MINFLUX.py index 9ef758b..1008a20 100644 --- a/PYMEcs/Analysis/MINFLUX.py +++ b/PYMEcs/Analysis/MINFLUX.py @@ -1,10 +1,10 @@ +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd from scipy.stats import binned_statistic + from PYMEcs.IO.MINFLUX import get_stddev_property -import numpy as np -import matplotlib.pyplot as plt from PYMEcs.pyme_warnings import warn -import pandas as pd -import os def propcheck_density_stats(ds,warning=True): @@ -55,6 +55,8 @@ def plot_density_stats(ds,objectID='dbscanClumpID',scatter=False): plt.tight_layout() from PYMEcs.misc.matplotlib import boxswarmplot + + def plot_density_stats_sns(ds,objectID='dbscanClumpID'): if not propcheck_density_stats(ds): return @@ -98,7 +100,7 @@ def plot_stats_minflux(deltas, durations, tdiff, tdmedian, efo_or_dtovertime, ti ax1[0].set_xlabel('time between traces (TBT) [s]') ax1[0].text(0.95, 0.8, 'median %.2f s' % dtmedian, horizontalalignment='right', verticalalignment='bottom', transform=ax1[0].transAxes) - if not areaString is None: + if areaString is not None: ax1[0].text(0.95, 0.6, areaString, horizontalalignment='right', verticalalignment='bottom', transform=ax1[0].transAxes) @@ -153,31 +155,39 @@ def plot_stats_minflux(deltas, durations, tdiff, tdmedian, efo_or_dtovertime, ti else: csv_name = timestamp # --- Save as a csv file --- (Alex B addition) - # Here we want to save the LocRate, however if LocError already exists we want to combine both files - # and ensure that the final csv keeps the organization regrdless on which csv was generated first (locError or LocRate). - - # variables to check if locerror and locrate csv files already exist - locerror_file = csv_name + "temp_LocError.csv" - locrate_file = csv_name + "temp_LocRate.csv" - combined_file = csv_name + ".csv" - # Save the LocRate file - df.to_csv(locrate_file, index=False, header=True) - - # If the LocError file already exists, merge: - if os.path.exists(locerror_file): - df_rate = pd.read_csv(locrate_file) - df_error = pd.read_csv(locerror_file) + # Below is old way of saving (before 2025-08-26) + # Replaced by saving each file individually (refering to plot error and plot stats from MFX) + # The merging of files can be done in the analysis workbook + # This avoid creating too many intermediate files and conditionals checks if other csv are being created later on for analysis + # Here we want to save the LocRate, however if LocError already exists we want to combine both files + # and ensure that the final csv keeps the organization regrdless on which csv was generated first (locError or LocRate). + + # variables to check if locerror and locrate csv files already exist + # locerror_file = csv_name + "temp_LocError.csv" + # locrate_file = csv_name + "temp_LocRate.csv" + # combined_file = csv_name + ".csv" - # Combine the two dataframes in the desired order - df_combined = pd.concat([df_error, df_rate], ignore_index=True) - df_combined.to_csv(combined_file, index=False, header=True) + # # Save the LocRate file + # df.to_csv(locrate_file, index=False, header=True) - # Cleanup temp files - os.remove(locerror_file) - os.remove(locrate_file) + # # If the LocError file already exists, merge: + # if os.path.exists(locerror_file): + # df_rate = pd.read_csv(locrate_file) + # df_error = pd.read_csv(locerror_file) + + # # Combine the two dataframes in the desired order + # df_combined = pd.concat([df_error, df_rate], ignore_index=True) + # df_combined.to_csv(combined_file, index=False, header=True) + + # # Cleanup temp files + # os.remove(locerror_file) + # os.remove(locrate_file) - print(f'\ncsv name is: {csv_name}\nIf you did not load a session, csv file and figures will be saved on the desktop') # Used as a reminder for LocRate csv saving + # df.to_csv(locrate_file, index=False, header=True) + + print(f'\ncsv name is: {csv_name}\nIf you did not load a session, csv file and figures will be saved on the desktop') # Used as a reminder for LocRate csv saving + df.to_csv(csv_name + '_LocRate.csv', index=False, header=True) # --- Save the figure --- (Alex B addition) plt.savefig(csv_name + '_loc_rate.png', dpi=300, bbox_inches='tight') diff --git a/PYMEcs/experimental/MINFLUX.py b/PYMEcs/experimental/MINFLUX.py index c09f388..c99be43 100644 --- a/PYMEcs/experimental/MINFLUX.py +++ b/PYMEcs/experimental/MINFLUX.py @@ -1,16 +1,18 @@ +import logging +import os + import matplotlib.pyplot as plt import numpy as np import wx -import os -import logging logger = logging.getLogger(__file__) from PYMEcs.pyme_warnings import warn + # --- Define the Localisation Error Analysis function --- def plot_errors(pipeline): - if not 'coalesced_nz' in pipeline.dataSources: + if 'coalesced_nz' not in pipeline.dataSources: warn('no data source named "coalesced_nz" - check recipe and ensure this is MINFLUX data') return curds = pipeline.selectedDataSourceKey @@ -103,36 +105,51 @@ def plot_errors(pipeline): print(df.head()) # --- Save as a csv file --- (Alex B addition) -# Here we want to save the LocError, however if LocRate already exists we want to combine both files -# and ensure that the final csv keeps the organization regrdless on which csv was generated first (locError or LocRate). - # variables to check if locerror and locrate csv files already exist - csv_name = pipeline.mdh.get('MINFLUX.TimeStamp') - locerror_file = csv_name + "temp_LocError.csv" - locrate_file = csv_name + "temp_LocRate.csv" - combined_file = csv_name + ".csv" + # Below is old way of saving (before 2025-08-26) + # Replaced by saving each file individually (refering to plot error and plot stats from MFX) + # The merging of files can be done in the analysis workbook + # This avoid creating too many intermediate files and conditionals checks if other csv are being created later on for analysis + + # # Here we want to save the LocError, however if LocRate already exists we want to combine both files + # # and ensure that the final csv keeps the organization regrdless on which csv was generated first (locError or LocRate). - # Save the locerror file - df.to_csv(locerror_file, index=False, header=True) + # # variables to check if locerror and locrate csv files already exist + # csv_name = pipeline.mdh.get('MINFLUX.TimeStamp') + # locerror_file = csv_name + "temp_LocError.csv" + # locrate_file = csv_name + "temp_LocRate.csv" + # combined_file = csv_name + ".csv" - # If the locrate file already exists, merge: - if os.path.exists(locrate_file): - df_rate = pd.read_csv(locrate_file) - df_error = pd.read_csv(locerror_file) + # # Save the locerror file + # df.to_csv(locerror_file, index=False, header=True) - # Combine the two dataframes in the desired order - df_combined = pd.concat([df_error, df_rate], ignore_index=True) - df_combined.to_csv(combined_file, index=False, header=True) + # # If the locrate file already exists, merge: + # if os.path.exists(locrate_file): + # df_rate = pd.read_csv(locrate_file) + # df_error = pd.read_csv(locerror_file) - # Cleanup temp files - os.remove(locerror_file) - os.remove(locrate_file) + # # Combine the two dataframes in the desired order + # df_combined = pd.concat([df_error, df_rate], ignore_index=True) + # df_combined.to_csv(combined_file, index=False, header=True) + + # # Cleanup temp files + # os.remove(locerror_file) + # os.remove(locrate_file) - print(f'\ncsv name is: {csv_name}\nIf you did not load a session, csv file and figures will be saved on the desktop') # Used as a reminder for LocRate csv saving + # Get the Timestamp (from pipeline) to create csv name + csv_name = pipeline.mdh.get('MINFLUX.TimeStamp') + + # Save the df as csv + df.to_csv(csv_name + "_LocError.csv", index=False, header=True) + + print(f'\ncsv name is: {csv_name}\nIf you did not load a session, csv file and figures will be saved on the desktop') # Used as a reminder for LocRate csv saving # By default the file is saved on the Desktop, if a session file is used, it is saved in the same directory as the session file. -from PYMEcs.misc.matplotlib import boxswarmplot import pandas as pd + +from PYMEcs.misc.matplotlib import boxswarmplot + + def _plot_clustersize_counts(cts, ctsgt1, xlabel='Cluster Size', wintitle=None, bigCfraction=None,bigcf_percluster=None, plotints=True, **kwargs): if 'range' in kwargs: enforce_xlims = True @@ -204,7 +221,7 @@ def _plot_clustersize_counts(cts, ctsgt1, xlabel='Cluster Size', wintitle=None, fig.canvas.manager.set_window_title(figtitle) def plot_cluster_analysis(pipeline, ds='dbscanClustered',showPlot=True, return_means=False, psu=None, bins=15, bigc_thresh=50, **kwargs): - if not ds in pipeline.dataSources: + if ds not in pipeline.dataSources: warn('no data source named "%s" - check recipe and ensure this is MINFLUX data' % ds) return curds = pipeline.selectedDataSourceKey @@ -248,7 +265,7 @@ def cluster_analysis(pipeline): return plot_cluster_analysis(pipeline, ds='dbscanClustered',showPlot=False,return_means=True) def plot_intra_clusters_dists(pipeline, ds='dbscanClustered',bins=15,NNs=1,**kwargs): - if not ds in pipeline.dataSources: + if ds not in pipeline.dataSources: warn('no data source named "%s" - check recipe and ensure this is MINFLUX data' % ds) return from scipy.spatial import KDTree @@ -348,8 +365,9 @@ def set_axis_style(ax, labels): plt.subplots_adjust(bottom=0.15, wspace=0.05) -from scipy.special import binom from scipy.optimize import curve_fit +from scipy.special import binom + def sigpn(p): return pn(1,p)+pn(2,p)+pn(3,p)+pn(4,p) @@ -509,13 +527,14 @@ def plot_site_tracking(pipeline,fignum=None,plotSmoothingCurve=True): axs[1, 1].set_ylabel('orig. corr [nm]') plt.tight_layout() +import PYME.config +from PYME.recipes.traits import Bool, CStr, Enum, Float, HasTraits + from PYMEcs.Analysis.MINFLUX import analyse_locrate +from PYMEcs.IO.MINFLUX import findmbm from PYMEcs.misc.guiMsgBoxes import Error from PYMEcs.misc.utils import unique_name -from PYMEcs.IO.MINFLUX import findmbm -from PYME.recipes.traits import HasTraits, Float, Enum, CStr, Bool, Int, List -import PYME.config class MINFLUXSettings(HasTraits): withOrigamiSmoothingCurves = Bool(True,label='Plot smoothing curves',desc="if overplotting smoothing curves " + @@ -604,6 +623,7 @@ def __init__(self, visFr): def OnClumpScatterPosPlot(self,event): from scipy.stats import binned_statistic + from PYMEcs.IO.MINFLUX import get_stddev_property def detect_coalesced(pipeline): # placeholder, to be implemented @@ -686,7 +706,7 @@ def OnMBMLowessCacheSave(self,event): mod.lowess_cachesave() def OnMBMAttributes(self, event): - from wx.lib.dialogs import ScrolledMessageDialog + from wx.lib.dialogs import ScrolledMessageDialog fres = self.visFr.pipeline.dataSources['FitResults'] if 'zarr' in dir(fres): try: @@ -703,7 +723,7 @@ def OnMBMAttributes(self, event): warn("could not find zarr attribute - is this a MFX zarr file?") def OnMFXAttributes(self, event): - from wx.lib.dialogs import ScrolledMessageDialog + from wx.lib.dialogs import ScrolledMessageDialog fres = self.visFr.pipeline.dataSources['FitResults'] if 'zarr' in dir(fres): try: @@ -720,7 +740,6 @@ def OnMFXAttributes(self, event): warn("could not find zarr attribute - is this a MFX zarr file?") def OnMFXInfo(self, event): - import io import wx.html fres = self.visFr.pipeline.dataSources['FitResults'] if 'zarr' not in dir(fres): @@ -787,8 +806,9 @@ def OnZarrToZipStore(self, event): if ddialog.ShowModal() != wx.ID_OK: return fpath = ddialog.GetPath() - from PYMEcs.misc.utils import zarrtozipstore from pathlib import Path + + from PYMEcs.misc.utils import zarrtozipstore zarr_root = Path(fpath) dest_dir = zarr_root.parent archive_name = dest_dir / zarr_root.with_suffix('.zarr').name # we make archive_name here in the calling routine so that we can check for existence etc @@ -835,19 +855,25 @@ def OnAlphaShapes(self, event): return # now we add a layer to render our alpha shape polygons - from PYME.LMVis.layers.tracks import TrackRenderLayer # NOTE: we may rename the clumpIndex variable in this layer to polyIndex or similar + from PYME.LMVis.layers.tracks import ( + TrackRenderLayer, # NOTE: we may rename the clumpIndex variable in this layer to polyIndex or similar + ) layer = TrackRenderLayer(self.visFr.pipeline, dsname='cluster_shapes', method='tracks', clump_key='polyIndex', line_width=2.0, alpha=0.5) self.visFr.add_layer(layer) def OnAddMINFLUXTracksCI(self, event): # now we add a track layer to render our traces - from PYME.LMVis.layers.tracks import TrackRenderLayer # NOTE: we may rename the clumpIndex variable in this layer to polyIndex or similar + from PYME.LMVis.layers.tracks import ( + TrackRenderLayer, # NOTE: we may rename the clumpIndex variable in this layer to polyIndex or similar + ) layer = TrackRenderLayer(self.visFr.pipeline, dsname='output', method='tracks', clump_key='clumpIndex', line_width=2.0, alpha=0.5) self.visFr.add_layer(layer) def OnAddMINFLUXTracksTid(self, event): # now we add a track layer to render our traces - from PYME.LMVis.layers.tracks import TrackRenderLayer # NOTE: we may rename the clumpIndex variable in this layer to polyIndex or similar + from PYME.LMVis.layers.tracks import ( + TrackRenderLayer, # NOTE: we may rename the clumpIndex variable in this layer to polyIndex or similar + ) layer = TrackRenderLayer(self.visFr.pipeline, dsname='output', method='tracks', clump_key='tid', line_width=2.0, alpha=0.5) self.visFr.add_layer(layer) @@ -856,7 +882,6 @@ def OnLoadCustom(self, event): def OnMBMSave(self,event): - from pathlib import Path pipeline = self.visFr.pipeline mbm = findmbm(pipeline) if mbm is None: @@ -1100,11 +1125,10 @@ def OnMINFLUXsetTempDataFolder(self, event): config='user', create_backup=True) def OnMINFLUXplotTempData(self, event): - import PYME.config as config - import os - from os.path import basename from glob import glob + import PYME.config as config + configvar = 'MINFLUX-temperature_folder' folder = config.get(configvar) if folder is None: @@ -1198,7 +1222,7 @@ def OnClumpIndexContig(self, event): pipeline = self.visFr.pipeline curds = pipeline.selectedDataSourceKey pipeline.selectDataSource(self.analysisSettings.defaultDatasourceForAnalysis) - if not 'clumpIndex' in pipeline.keys(): + if 'clumpIndex' not in pipeline.keys(): Error(self.visFr,'no property called "clumpIndex", cannot check') pipeline.selectDataSource(curds) return @@ -1221,11 +1245,11 @@ def OnLocalisationRate(self, event): timestamp = pipeline.mdh.get('MINFLUX.TimeStamp') # print("Timestamp from OnLocalisationRate(experimental/MINFLUX.py): ", timestamp) - if not 'cfr' in pipeline.keys(): + if 'cfr' not in pipeline.keys(): Error(self.visFr,'no property called "cfr", likely no MINFLUX data - aborting') pipeline.selectDataSource(curds) return - if not 'tim' in pipeline.keys(): + if 'tim' not in pipeline.keys(): Error(self.visFr,'no property called "tim", you need to convert to CSV with a more recent version of PYME-Extra - aborting') pipeline.selectDataSource(curds) return @@ -1239,7 +1263,7 @@ def OnEfoAnalysis(self, event): pipeline.selectDataSource(self.analysisSettings.defaultDatasourceForAnalysis) #Added by Alex B to get timestamp and send it to plot_stats_minflux timestamp = pipeline.mdh.get('MINFLUX.TimeStamp') - if not 'efo' in pipeline.keys(): + if 'efo' not in pipeline.keys(): Error(self.visFr,'no property called "efo", likely no MINFLUX data or wrong datasource (CHECK) - aborting') return plt.figure() @@ -1286,9 +1310,10 @@ def OnOrigamiFinalFilter(self, event=None): pipeline.selectDataSource(finalFiltered) def OnOrigamiSiteRecipe(self, event=None): - from PYMEcs.recipes.localisations import OrigamiSiteTrack, DBSCANClustering2 from PYME.recipes.localisations import MergeClumps from PYME.recipes.tablefilters import FilterTable, Mapping + + from PYMEcs.recipes.localisations import DBSCANClustering2, OrigamiSiteTrack pipeline = self.visFr.pipeline recipe = pipeline.recipe From a146a34cfdc6b20a2cea694c948326d97cc6b9db Mon Sep 17 00:00:00 2001 From: Alexfebo Date: Thu, 28 Aug 2025 17:54:12 +0200 Subject: [PATCH 12/27] Add Save All NPC 3D Analysis Actions Upon menu click, performs: - Save NPC set (pickle) - Save Measurement (CSV) - Show NPC Geom - Show NPC template fit - Plot Segments - Save NPC segment data --- PYMEcs/experimental/NPCcalcLM.py | 48 ++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/PYMEcs/experimental/NPCcalcLM.py b/PYMEcs/experimental/NPCcalcLM.py index f2b8bed..03b97f7 100644 --- a/PYMEcs/experimental/NPCcalcLM.py +++ b/PYMEcs/experimental/NPCcalcLM.py @@ -1,13 +1,15 @@ -from PYMEcs.pyme_warnings import warn -from PYMEcs.Analysis.NPC import estimate_nlabeled, npclabel_fit, plotcdf_npc3d -from PYME.recipes import tablefilters -import wx -from traits.api import HasTraits, Str, Int, CStr, List, Enum, Float, Bool -import numpy as np +import logging + import matplotlib.pyplot as plt -from PYMEcs.misc.utils import unique_name +import numpy as np +import wx +from PYME.recipes import tablefilters +from traits.api import Bool, Enum, Float, HasTraits, Int + +from PYMEcs.Analysis.NPC import estimate_nlabeled, npclabel_fit, plotcdf_npc3d from PYMEcs.IO.NPC import findNPCset -import logging +from PYMEcs.misc.utils import unique_name +from PYMEcs.pyme_warnings import warn logger = logging.getLogger(__name__) @@ -126,6 +128,28 @@ def __init__(self, visFr): self._npcsettings = None self.gallery_layer = None self.segment_layer = None + + # --- Alex B addition test for running several action at once --- + # Add combined MenuItem for all requested actions + visFr.AddMenuItem('Experimental>NPC3D', 'Save All NPC 3D Analysis Actions', self.OnNPC3DRunAllActions) + def OnNPC3DRunAllActions(self, event=None): + """Performs all key NPC 3D analysis actions in sequence.""" + # Save NPC Set with full fit analysis + self.OnNPC3DSaveNPCSet() + # Save Measurements Only (csv, no fit info saved) + self.OnNPC3DSaveMeasurements() + # Show NPC geometry statistics + self.OnNPC3DGeometryStats() + # Show NPC template fit statistics + self.OnNPC3DTemplateFitStats() + # Plot NPC by-segment data + self.OnNPC3DPlotBySegments() + # Save NPC by-segment data + self.OnNPC3DSaveBySegments() + + # --- End of Alex B addition test for running several action at once --- + + @property def NPCsettings(self): @@ -186,7 +210,6 @@ def OnAnalyse3DNPCsByID(self, event=None): npcs = findNPCset(pipeline) do_plot = False else: - from PYMEcs.IO.MINFLUX import foreshortening npcs = NPC3DSet(filename=pipeline_filename(pipeline), zclip=self.NPCsettings.Zclip_3D, offset_mode=self.NPCsettings.OffsetMode_3D, @@ -367,7 +390,9 @@ def On3DNPCaddTemplates(self, event=None): # now we add a track layer to render our template polygons # TODO - we may need to check if this happened before or not! - from PYME.LMVis.layers.tracks import TrackRenderLayer # NOTE: we may rename the clumpIndex variable in this layer to polyIndex or similar + from PYME.LMVis.layers.tracks import ( + TrackRenderLayer, # NOTE: we may rename the clumpIndex variable in this layer to polyIndex or similar + ) layer = TrackRenderLayer(pipeline, dsname=ds_template_name, method='tracks', clump_key='polyIndex', line_width=2.0, alpha=0.5) self.visFr.add_layer(layer) @@ -388,7 +413,6 @@ def OnNPC3DSaveMeasurements(self, event=None): fpath = fdialog.GetPath() meas = np.array(npcs.measurements, dtype='i') - import pandas as pd df = pd.DataFrame({'Ntop_NPC3D': meas[:, 0], 'Nbot_NPC3D': meas[:, 1]}) entries = len(np.unique(pipeline.objectID)) MINFLUX_filename = [pipeline.mdh.getEntry('MINFLUX.Filename')]*entries @@ -493,6 +517,7 @@ def OnNPC3DGeometryStats(self,event=None): warn('no valid NPC measurements found, thus no geometry info available...') return import pandas as pd + from PYMEcs.misc.matplotlib import boxswarmplot, figuredefaults diams = np.asarray(npcs.diam()) heights = np.asarray(npcs.height()) @@ -518,6 +543,7 @@ def OnNPC3DTemplateFitStats(self,event=None): warn('no valid NPC measurements found, thus no geometry info available...') return import pandas as pd + from PYMEcs.misc.matplotlib import boxswarmplot, figuredefaults id = [npc.objectID for npc in npcs.npcs] # not used right now llperloc = [npc.opt_result.fun/npc.npts.shape[0] for npc in npcs.npcs] From c9735d88afd949c0d6d21599b276aba803dfe1af Mon Sep 17 00:00:00 2001 From: AlexFEBo Date: Thu, 4 Sep 2025 11:17:52 +0200 Subject: [PATCH 13/27] Auto-save for all NPC action implemented as single menu-Items --- PYMEcs/Analysis/NPC.py | 34 ++- PYMEcs/experimental/NPCcalcLM.py | 422 +++++++++++++++++++++++++++++++ 2 files changed, 452 insertions(+), 4 deletions(-) diff --git a/PYMEcs/Analysis/NPC.py b/PYMEcs/Analysis/NPC.py index 0138aef..0bfca20 100644 --- a/PYMEcs/Analysis/NPC.py +++ b/PYMEcs/Analysis/NPC.py @@ -1,5 +1,6 @@ import matplotlib.pyplot as plt import numpy as np + from PYMEcs.pyme_warnings import warn piover4 = np.pi/4.0 @@ -31,6 +32,8 @@ # return (crot.T[:,0],crot.T[:,1]) from circle_fit import taubinSVD + + def fitcirc(x,y,sigma=None): pcs = np.vstack((x,y)).T xc, yc, r, sigma = taubinSVD(pcs) @@ -222,8 +225,9 @@ def segnotorad(sec): else: return Nlabeled -from scipy.special import binom from scipy.optimize import curve_fit +from scipy.special import binom + def pn(k,n,p): return (binom(n,k)*(np.power(p,k)*np.power((1-p),(n-k)))) @@ -272,6 +276,8 @@ def npclabel_fit(nphist,sigma=None): return (popt[0],n_labels_scaled,perr[0]) from PYMEcs.misc.utils import get_timestamp_from_filename + + def plotcdf_npc3d(nlab,plot_as_points=True,timestamp=None,thresh=None): pr = prangeNPC3D() for p in pr.keys(): @@ -325,9 +331,10 @@ def to3vecs(x,y,z): def xyzfrom3vec(v): return (v[:,0],v[:,1],v[:,2]) -from scipy.spatial.transform import Rotation as R from scipy.interpolate import RegularGridInterpolator from scipy.signal import fftconvolve +from scipy.spatial.transform import Rotation as R + def fpinterpolate(fp3d,x,y,z,method='linear', bounds_error=True, fill_value=np.nan): # V[i,j,k] = 100*x[i] + 10*y[j] + z[k] @@ -614,7 +621,7 @@ def normalize_points(self,zclip=None,mode='mean'): raise RuntimeError("unknown mode '%s', should be mean or median" % mode) npts = self.points - self.offset nt = self.t - if not zclip is None: + if zclip is not None: zgood = (npts[:,2] > -zclip)*(npts[:,2] < zclip) npts = npts[zgood,:] nt = nt[zgood] @@ -878,7 +885,6 @@ def measure_labeleff(self,nthresh=1,do_plot=False,printpars=False,refit=False): self.measurements.append([nt,nb]) def plot_labeleff(self,thresh=None): - from PYMEcs.misc.utils import get_timestamp_from_filename if len(self.measurements) < 10: raise RuntimeError("not enough measurements, need at least 10, got %d" % len(self.measurements)) @@ -892,6 +898,26 @@ def plot_labeleff(self,thresh=None): plt.figure() plotcdf_npc3d(nlab,timestamp=get_timestamp_from_filename(self.filename),thresh=thresh) + # --- Alex B addiiton for auto-saving of LE-figure --- + # Function is called in NPCcalcLM.py (PYMEcs>experimental>OnAnalyse3DNPCsByID_auto_save) + + def plot_labeleff_for_auto_save(self,thresh=None): + if len(self.measurements) < 10: + raise RuntimeError("not enough measurements, need at least 10, got %d" % + len(self.measurements)) + meas = np.array(self.measurements) + nlab = meas.sum(axis=1) + # fill with trailing zeros if we have a known number of NPCs but have fewer measurements + # the "missing NPCs" typically represent NPCs with no events + if int(self.known_number) > 0 and nlab.shape[0] < int(self.known_number): + nlab = np.pad(nlab,((0,int(self.known_number)-nlab.shape[0]))) + + LEfig = plt.figure() + plotcdf_npc3d(nlab,timestamp=get_timestamp_from_filename(self.filename),thresh=thresh) + return LEfig + + # --- Fin Alex B addition --- + def diam(self): diams = [] for npc in self.npcs: diff --git a/PYMEcs/experimental/NPCcalcLM.py b/PYMEcs/experimental/NPCcalcLM.py index 03b97f7..5e63d16 100644 --- a/PYMEcs/experimental/NPCcalcLM.py +++ b/PYMEcs/experimental/NPCcalcLM.py @@ -1,4 +1,5 @@ import logging +import os import matplotlib.pyplot as plt import numpy as np @@ -125,6 +126,19 @@ def __init__(self, visFr): visFr.AddMenuItem('Experimental>NPC3D', 'Plot NPC by-segment data', self.OnNPC3DPlotBySegments) visFr.AddMenuItem('Experimental>NPC3D', 'Save NPC by-segment data', self.OnNPC3DSaveBySegments) + # --- Alex B addition for auto-saving NPC sets --- + # TODO: When all are working, replace the items in 'OnNPC3DRunAllActions' + self.visFr.AddMenuItem('Experimental>NPC3D', "Auto-save Analyse 3D NPCs by ID", self.OnAnalyse3DNPCsByID_auto_save) + self.visFr.AddMenuItem('Experimental>NPC3D', "Auto-save NPC Set", self.OnNPC3DSaveNPCSet_auto_save) + self.visFr.AddMenuItem('Experimental>NPC3D', "Auto-save Measurements Only (csv, no fit info saved)", self.OnNPC3DSaveMeasurements_auto_save) + self.visFr.AddMenuItem('Experimental>NPC3D', "Auto-save NPC geometry statistics",self.OnNPC3DGeometryStats_auto_save) + self.visFr.AddMenuItem('Experimental>NPC3D', "Auto-save NPC template fit statistics",self.OnNPC3DTemplateFitStats_auto_save) + self.visFr.AddMenuItem('Experimental>NPC3D', "Auto-save NPC by-segment data",self.OnNPC3DSaveBySegments_auto_save) + self.visFr.AddMenuItem('Experimental>NPC3D', "Auto-save NPC by-segment plot",self.OnNPC3DPlotBySegments_auto_save) + + # --- End of Alex B addition for auto-saving NPC sets --- + + self._npcsettings = None self.gallery_layer = None self.segment_layer = None @@ -279,6 +293,118 @@ def OnAnalyse3DNPCsByID(self, event=None): npcs.plot_labeleff(thresh=self.NPCsettings.SegmentThreshold_3D) +# --- Alex B addition --- +# AIM: perform all actions 3D NPC actions and save output automatically +# Original function 'OnAnalyse3DNPCsByID', copied and modified for automatic saving. + + def OnAnalyse3DNPCsByID_auto_save(self, event=None): + from PYMEcs.Analysis.NPC import NPC3DSet + pipeline = self.visFr.pipeline + + # --- Alex B addition --- + # We define a few variables used for automatic saving later + + base_dir = os.getcwd() # Get the working directory + MINFLUXts = pipeline.mdh.get('MINFLUX.TimeStamp') # Get the timestamp and use it for naming the file to save + if MINFLUXts is not None: + fitplot_filename = f"{MINFLUXts}-NPC_fit_plot.png" + leplot_filename = f"{MINFLUXts}-LE_plot.png" + else: # If no timestamp is found, use default filenames + fitplot_filename = "NPC_fit_plot.png" + leplot_filename = "LE_plot.png" + fitplot_save_path = os.path.join(base_dir, fitplot_filename) # Save path for fit plot + leplot_save_path = os.path.join(base_dir, leplot_filename) # Save path for LE plot + + # --- End Alex B addition --- + + if findNPCset(pipeline,warnings=False) is not None: + npcs = findNPCset(pipeline) + do_plot = False # If NPC set exists, do not re-analyze / re-plot + else: # If NPC set does not exists, do analyze plot + npcs = NPC3DSet(filename=pipeline_filename(pipeline), + zclip=self.NPCsettings.Zclip_3D, + offset_mode=self.NPCsettings.OffsetMode_3D, + NPCdiam=self.NPCsettings.StartDiam_3D, + NPCheight=self.NPCsettings.StartHeight_3D, + foreshortening=pipeline.mdh.get('MINFLUX.Foreshortening',1.0), + known_number=self.NPCsettings.KnownNumber_3D, + templatemode=self.NPCsettings.TemplateMode_3D, + sigma=self.NPCsettings.TemplateSigma_3D) + do_plot = True # If NPC set does not exists, do analyze plot (lines 326) + for oid in np.unique(pipeline['objectID']): + npcs.addNPCfromPipeline(pipeline,oid) + + # for example use of ProgressDialog see also + # https://github.com/Metallicow/wxPython-Sample-Apps-and-Demos/blob/master/101_Common_Dialogs/ProgressDialog/ProgressDialog_extended.py + progress = wx.ProgressDialog("NPC analysis in progress", "please wait", maximum=len(npcs.npcs), + parent=self.visFr, + style=wx.PD_SMOOTH + | wx.PD_AUTO_HIDE + | wx.PD_CAN_ABORT + | wx.PD_ESTIMATED_TIME + | wx.PD_REMAINING_TIME) + if do_plot: # IInitialize plots (only creates canvas and axes) + fig, axes=plt.subplots(2,3) + if 'templatemode' in dir(npcs) and npcs.templatemode == 'twostage': + figpre, axespre=plt.subplots(2,3,label='pre-llm') + cancelled = False + npcs.measurements = [] + if 'templatemode' in dir(npcs) and npcs.templatemode == 'detailed': + rotation = 22.5 # this value may need adjustment + else: + rotation = None + + + # keep track if any fits were performed + anyfits = False + for i,npc in enumerate(npcs.npcs): + if not npc.fitted: + if 'templatemode' in dir(npcs) and npcs.templatemode == 'twostage': + npc.fitbymll(npcs.llm,plot=True,printpars=False,axes=axes,preminimizer=npcs.llmpre,axespre=axespre) # Plot generated here + else: + npc.fitbymll(npcs.llm,plot=True,printpars=False,axes=axes) # Plot generated here + anyfits = True + nt,nb = npc.nlabeled(nthresh=self.NPCsettings.SegmentThreshold_3D, + dr=self.NPCsettings.RadiusUncertainty_3D, + rotlocked=self.NPCsettings.RotationLocked_3D, + zrange=self.NPCsettings.Zclip_3D, + rotation=rotation) + if self.NPCsettings.SkipEmptyTopOrBottom_3D and (nt == 0 or nb == 0): + pass # we skip NPCs with empty rings in this case + else: + npcs.measurements.append([nt,nb]) + (keepGoing, skip) = progress.Update(i+1) + if not keepGoing: + logger.info('OnAnalyse3DNPCsByID: progress cancelled, aborting NPC analysis') + cancelled = True + progress.Destroy() + wx.Yield() + # Cancelled by user. + break + wx.Yield() + else: + if anyfits: + pipeline.npcs = npcs # we update the pipeline npcs attribute only if the for loop completed normally and we fitted + + if cancelled: + return + + #--- Alex B modif addition--- + + # Save the main NPC fit plot + # Uses the variables (filename+path defined earlier) + # Note: Save only the last fit plot + if do_plot and not cancelled: + fig.savefig(fitplot_save_path) + print(f"NPC fit plot automatically saved as: {fitplot_save_path}") + + # Create the labeling efficiency plot + fig = npcs.plot_labeleff_for_auto_save(thresh=self.NPCsettings.SegmentThreshold_3D) # We are calling the alternative function: plot_labeleff_for_auto_save + # which returns the fig object, we then can use fig for automatic saving + fig.savefig(leplot_save_path) + print(f"Labeling efficiency plot automatically saved as: {leplot_save_path}") + # --- End of Alex B addition --- + def OnNPC3DSaveBySegments(self, event=None): pipeline = self.visFr.pipeline @@ -300,6 +426,46 @@ def OnNPC3DSaveBySegments(self, event=None): df.to_csv(fpath,index=False) else: warn("could not find valid NPC set, have you carried out fitting?") + + # --- Alex B addition --- + + def OnNPC3DSaveBySegments_auto_save(self, event=None): + pipeline = self.visFr.pipeline + + # --- Alex B addition --- + # We define a few variables used for automatic saving later + base_dir = os.getcwd() # Get the working directory + MINFLUXts = pipeline.mdh.get('MINFLUX.TimeStamp') # Get the timestamp and use it for naming the file to save + if MINFLUXts is not None: + NPC_segments_stats = f"{MINFLUXts}-NPC_segments.csv" + else: + NPC_segments_stats = "NPC_segments.csv" + NPC_segments_stats_save_path = os.path.join(base_dir, NPC_segments_stats) # Save path for csv file + # --- End of Alex B addition --- + + if findNPCset(pipeline) is not None: + nbs = findNPCset(pipeline).n_bysegments() + if nbs is None: + warn("could not find npcs with by-segment fitting info, have you carried out fitting with recent code?") + return + # # Original code with dialog for manual saving + # with wx.FileDialog(self.visFr, 'Save NPC by-segment data as ...', + # wildcard='CSV (*.csv)|*.csv', + # style=wx.FD_SAVE) as fdialog: + # if fdialog.ShowModal() != wx.ID_OK: + # return + # else: + # fpath = fdialog.GetPath() + + import pandas as pd + df = pd.DataFrame.from_dict(dict(top=nbs['top'].flatten(),bottom=nbs['bottom'].flatten())) + df.to_csv(NPC_segments_stats_save_path,index=False) # Alex B automatic saving + print(f"NPC by-segment data automatically saved as: {NPC_segments_stats_save_path}") + + else: + warn("could not find valid NPC set, have you carried out fitting?") + + # --- End of Alex B addition --- def OnNPC3DPlotBySegments(self, event=None): pipeline = self.visFr.pipeline @@ -326,6 +492,55 @@ def OnNPC3DPlotBySegments(self, event=None): verticalalignment='center', color='r', transform=fig.transFigure) else: warn("could not find valid NPC set, have you carried out fitting?") + + # --- Alex B addition --- + + def OnNPC3DPlotBySegments_auto_save(self, event=None): + pipeline = self.visFr.pipeline + + # --- Alex B addition --- + # We define a few variables used for automatic saving later + + base_dir = os.getcwd() # Get the working directory + MINFLUXts = pipeline.mdh.get('MINFLUX.TimeStamp') # Get the timestamp and use it for naming the file to save + if MINFLUXts is not None: + NPC_plot_segments = f"{MINFLUXts}-NPC_segments.png" + else: + NPC_plot_segments = "NPC_segments.png" + NPC_plot_segments_save_path = os.path.join(base_dir, NPC_plot_segments) # Save path for csv file + + # --- End of Alex B addition --- + + if findNPCset(pipeline) is not None: + nbs = findNPCset(pipeline).n_bysegments() + if nbs is None: + warn("could not find npcs with by-segment fitting info, have you carried out fitting with recent code?") + return + fig = plt.figure() + plt.hist(nbs['bottom'].flatten(),bins=np.arange(nbs['bottom'].max()+2)-0.5,alpha=0.5,label='bottom',density=True,histtype='step') + plt.plot([nbs['bottom'].mean(),nbs['bottom'].mean()],[0,0.2],'b--') + plt.hist(nbs['top'].flatten(),bins=np.arange(nbs['top'].max()+2)-0.5,alpha=0.5,label='top',density=True,histtype='step') + plt.plot([nbs['top'].mean(),nbs['top'].mean()],[0,0.2],'r--') + plt.legend() + nbsflat = nbs['bottom'].flatten() + b_meanall = 0.5*nbsflat.mean() # 0.5 to get per site because we have two sites per segment + b_meannz = 0.5*nbsflat[nbsflat>0].mean() # 0.5 to get per site because we have two sites per segment + plt.text(0.65,0.5,'cytop per site: %.1f (%.1f per labeled)' % (b_meanall,b_meannz), horizontalalignment='center', + verticalalignment='center', color='b', transform=fig.transFigure) + nbsflat = nbs['top'].flatten() + t_meanall = 0.5*nbsflat.mean() # 0.5 to get per site because we have two sites per segment + t_meannz = 0.5*nbsflat[nbsflat>0].mean() # 0.5 to get per site because we have two sites per segment + plt.text(0.65,0.44,'nucleop per site: %.1f (%.1f per labeled)' % (t_meanall,t_meannz), horizontalalignment='center', + verticalalignment='center', color='r', transform=fig.transFigure) + # --- Alex B addition --- + # Save the NPC geometry stats plot + fig.savefig(NPC_plot_segments_save_path) + print(f"NPC geometry stats plot automatically saved as: {NPC_plot_segments_save_path}") + # --- End of Alex B addition --- + else: + warn("could not find valid NPC set, have you carried out fitting?") + + # --- End of Alex B addition --- def On3DNPCaddGallery(self, event=None): pipeline = self.visFr.pipeline @@ -460,6 +675,89 @@ def OnNPC3DSaveMeasurements(self, event=None): df.to_csv(fpath,index=False, mode='a') + # --- Alex B addition for auto-save of measurements --- + + def OnNPC3DSaveMeasurements_auto_save(self, event=None): + import pandas as pd + pipeline = self.visFr.pipeline + npcs = findNPCset(pipeline) + if npcs is None or 'measurements' not in dir(npcs): + warn('no valid NPC measurements found, therefore cannot save...') + return + # Dialog for saving --> Commented-out + # fdialog = wx.FileDialog(self.visFr, 'Save NPC measurements as ...', + # wildcard='CSV (*.csv)|*.csv', + # style=wx.FD_SAVE) + # if fdialog.ShowModal() != wx.ID_OK: + # return + + # fpath = fdialog.GetPath() + + # --- Alex B addition --- + # We define a few variables used for automatic saving later + + base_dir = os.getcwd() # Get the working directory + MINFLUXts = pipeline.mdh.get('MINFLUX.TimeStamp') # Get the timestamp and use it for naming the file to save + if MINFLUXts is not None: + csv_filename = f"{MINFLUXts}-LE_stats.csv" + else: + csv_filename = "LE_stats.csv" + csv_save_path = os.path.join(base_dir, csv_filename) # Save path for csv file + + # --- End of Alex B addition --- + + + meas = np.array(npcs.measurements, dtype='i') + + df = pd.DataFrame({'Ntop_NPC3D': meas[:, 0], 'Nbot_NPC3D': meas[:, 1]}) + entries = len(np.unique(pipeline.objectID)) + MINFLUX_filename = [pipeline.mdh.getEntry('MINFLUX.Filename')]*entries + NPC_threshold = [self.NPCsettings.SegmentThreshold_3D]*entries + fshortening = [pipeline.mdh.getEntry('MINFLUX.Foreshortening')]*entries + t_min = [np.round(np.min(pipeline.tim))]*entries + t_max = [np.round(np.max(pipeline.tim))]*entries + t_maxhr = [np.round(ti/3600) for ti in t_max] + + duration_hours = [np.round((np.max(pipeline.tim)-np.min(pipeline.tim))/3600,2)]*entries + duration_hours_rounded = [np.round(duration) for duration in duration_hours] + dim_z_nm = [np.round(np.max(pipeline.z)-np.min(pipeline.z),2)]*entries + NPCLE = (meas[:,0]+meas[:,1])/16 + + unique_ids, counts = np.unique(pipeline.objectID, return_counts=True) + nEvents = counts.tolist() + + df = pd.DataFrame({'Ntop_NPC3D': meas[:, 0], 'Nbot_NPC3D': meas[:, 1], "NPC_LE": NPCLE, + 'objectID': np.unique(pipeline.objectID), + 'diameters': np.round(pipeline.npcs.diam(), 4), + 'heights': np.round(pipeline.npcs.height(), 4), + 't_min': t_min, + 't_max': t_max, + 't_maxhr': t_maxhr, + 'duration_hours': duration_hours, + 'duration_hours_rounded': duration_hours_rounded, + 'nEvents': nEvents, + 'dim_z_nm': dim_z_nm, + 'foreshortening': fshortening, + 'NPC_threshold': NPC_threshold, + 'filename': MINFLUX_filename}) + + pipeline.selectDataSource('Localizations') + dim_x = (np.max(pipeline.x)-np.min(pipeline.x))/1000 + dim_y = (np.max(pipeline.y)-np.min(pipeline.y))/1000 + area_xy = [np.round(dim_x*dim_y,2)]*entries + + df.insert(3, 'area_xy', area_xy) + + try: + pipeline.selectDataSource('valid_npcs') + print("Pipeline data source successfully changed") + except: + print("Unable to change the pipeline to 'valid_npcs'") + + df.to_csv(csv_save_path,index=False, mode='a') + +# --- End of Alex B addition for auto-save --- + def OnNPC3DLoadMeasurements(self, event=None): from PYMEcs.misc.utils import get_timestamp_from_filename @@ -510,6 +808,41 @@ def OnNPC3DSaveNPCSet(self, event=None): save_NPC_set(npcs,fdialog.GetPath()) + +# --- Alex B addition --- +# AIM: perform all actions 3D NPC actions and save output automatically +# Original function 'OnNPC3DSaveNPCSet', copied and modified for automatic saving. +# Works fine as single action (new button: OnNPC3DSaveNPCSet_auto_save) line 130 + + def OnNPC3DSaveNPCSet_auto_save(self, event=None): + """Automatically save the NPC set to a default file path without user dialog.""" + + from PYMEcs.IO.NPC import save_NPC_set + + pipeline = self.visFr.pipeline + + # Get the MINFLUX timestamp from the pipeline metadata, if available + MINFLUXts = pipeline.mdh.get('MINFLUX.TimeStamp') + # Construct the default filename using the timestamp if present + if MINFLUXts is not None: + defaultFile = f"{MINFLUXts}-NPCset.pickle" + else: + defaultFile = "NPCset.pickle" + # Save in the current directory (same as the session file + base_dir = os.getcwd() + save_path = os.path.join(base_dir, defaultFile) + # Find the current NPC set in the pipeline + npcs = findNPCset(pipeline) + print(f"Attempting to automatically save NPC Set to: {save_path}.") + # If no NPC set is found, warn and exit + if npcs is None: + warn('no valid NPC Set found, therefore cannot save...') + return + # Save the NPC set to the constructed path + save_NPC_set(npcs, save_path) + +# --- End of Alex B addition --- + def OnNPC3DGeometryStats(self,event=None): pipeline = self.visFr.pipeline npcs = findNPCset(pipeline) @@ -536,6 +869,54 @@ def OnNPC3DGeometryStats(self,event=None): plt.title("%d NPCs, mean diam %.0f nm, mean ring spacing %.0f nm" % (heights.size,diams.mean(),heights.mean()), fontsize=11) plt.ylim(0,150) +# --- Alex B addition --- +# Origimnal function 'OnNPC3DGeometryStats', copied and modified for automatic saving. + + def OnNPC3DGeometryStats_auto_save(self,event=None): + pipeline = self.visFr.pipeline + npcs = findNPCset(pipeline) + if npcs is None: + warn('no valid NPC measurements found, thus no geometry info available...') + return + import pandas as pd + + # --- Alex B addition --- + # We define a few variables used for automatic saving later + + base_dir = os.getcwd() # Get the working directory + MINFLUXts = pipeline.mdh.get('MINFLUX.TimeStamp') # Get the timestamp and use it for naming the file to save + if MINFLUXts is not None: + geom_stats_fig = f"{MINFLUXts}-Geom_stats.png" + else: + geom_stats_fig = "Geom_stats.png" + geom_stats_fig_save_path = os.path.join(base_dir, geom_stats_fig) # Save path for csv file + + # --- End of Alex B addition --- + + from PYMEcs.misc.matplotlib import boxswarmplot, figuredefaults + diams = np.asarray(npcs.diam()) + heights = np.asarray(npcs.height()) + geo_df = pd.DataFrame.from_dict(dict(diameter=diams,height=heights)) + figuredefaults(fontsize=12) + plt.figure() + from scipy.stats import iqr + iqrh = iqr(heights) + sdh = np.std(heights) + iqrd = iqr(diams) + sdd = np.std(diams) + bp = boxswarmplot(geo_df,width=0.35,annotate_medians=True,annotate_means=True,showmeans=True,swarmalpha=0.4,swarmsize=4) + plt.text(0.0,50,"IQR %.1f\nSD %.1f" % (iqrd,sdd), horizontalalignment='center') + plt.text(1.0,120,"IQR %.1f\nSD %.1f" % (iqrh,sdh), horizontalalignment='center') + # res = plt.boxplot([diams,heights],showmeans=True,labels=['diameter','height']) + plt.title("%d NPCs, mean diam %.0f nm, mean ring spacing %.0f nm" % (heights.size,diams.mean(),heights.mean()), fontsize=11) + plt.ylim(0,150) + # --- Alex B addition --- + # Save the geometry stats plot + plt.savefig(geom_stats_fig_save_path) + print(f"Geometry stats plot automatically saved as: {geom_stats_fig_save_path}") + +# --- End of Alex B addition --- + def OnNPC3DTemplateFitStats(self,event=None): pipeline = self.visFr.pipeline npcs = findNPCset(pipeline) @@ -552,7 +933,48 @@ def OnNPC3DTemplateFitStats(self,event=None): plt.figure() bp = boxswarmplot(ll_df,width=0.35,annotate_medians=True,annotate_means=True,showmeans=True,swarmalpha=0.6,swarmsize=5) plt.title("NPC neg-log-likelihood per localization") + +# --- Alex B addition --- +# Original function 'OnNPC3DTemplateFitStats', copied and modified for automatic saving. + + def OnNPC3DTemplateFitStats_auto_save(self,event=None): + pipeline = self.visFr.pipeline + npcs = findNPCset(pipeline) + if npcs is None: + warn('no valid NPC measurements found, thus no geometry info available...') + return + import pandas as pd + + # --- Alex B addition --- + # We define a few variables used for automatic saving later + base_dir = os.getcwd() # Get the working directory + MINFLUXts = pipeline.mdh.get('MINFLUX.TimeStamp') # Get the timestamp and use it for naming the file to save + if MINFLUXts is not None: + template_fit_stats_fig = f"{MINFLUXts}-Template_fit_stats.png" + else: + template_fit_stats_fig = "Template_fit_stats.png" + template_fit_stats_fig_save_path = os.path.join(base_dir, template_fit_stats_fig) # Save path for csv file + + # --- End of Alex B addition --- + + from PYMEcs.misc.matplotlib import boxswarmplot, figuredefaults + id = [npc.objectID for npc in npcs.npcs] # not used right now + llperloc = [npc.opt_result.fun/npc.npts.shape[0] for npc in npcs.npcs] + ll_df = pd.DataFrame.from_dict(dict(llperloc=llperloc)) + figuredefaults(fontsize=12) + plt.figure() + bp = boxswarmplot(ll_df,width=0.35,annotate_medians=True,annotate_means=True,showmeans=True,swarmalpha=0.6,swarmsize=5) + plt.title("NPC neg-log-likelihood per localization") + + # --- Alex B addition --- + # Save the template fit stats plot + plt.savefig(template_fit_stats_fig_save_path) + print(f"Template fit stats plot automatically saved as: {template_fit_stats_fig_save_path}") + +# --- End of Alex B addition --- + + def OnSelectNPCsByMask(self,event=None): from PYME.DSView import dsviewer From b5ce3fe1b3d064ad9feae0877132a2208eda93de Mon Sep 17 00:00:00 2001 From: AlexFEBo Date: Thu, 4 Sep 2025 11:30:44 +0200 Subject: [PATCH 14/27] Perform all NPC3D actions and automatically saves all outputs --- PYMEcs/experimental/NPCcalcLM.py | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/PYMEcs/experimental/NPCcalcLM.py b/PYMEcs/experimental/NPCcalcLM.py index 5e63d16..3e03f0e 100644 --- a/PYMEcs/experimental/NPCcalcLM.py +++ b/PYMEcs/experimental/NPCcalcLM.py @@ -125,18 +125,7 @@ def __init__(self, visFr): visFr.AddMenuItem('Experimental>NPC3D', 'Add NPC Gallery', self.On3DNPCaddGallery) visFr.AddMenuItem('Experimental>NPC3D', 'Plot NPC by-segment data', self.OnNPC3DPlotBySegments) visFr.AddMenuItem('Experimental>NPC3D', 'Save NPC by-segment data', self.OnNPC3DSaveBySegments) - - # --- Alex B addition for auto-saving NPC sets --- - # TODO: When all are working, replace the items in 'OnNPC3DRunAllActions' - self.visFr.AddMenuItem('Experimental>NPC3D', "Auto-save Analyse 3D NPCs by ID", self.OnAnalyse3DNPCsByID_auto_save) - self.visFr.AddMenuItem('Experimental>NPC3D', "Auto-save NPC Set", self.OnNPC3DSaveNPCSet_auto_save) - self.visFr.AddMenuItem('Experimental>NPC3D', "Auto-save Measurements Only (csv, no fit info saved)", self.OnNPC3DSaveMeasurements_auto_save) - self.visFr.AddMenuItem('Experimental>NPC3D', "Auto-save NPC geometry statistics",self.OnNPC3DGeometryStats_auto_save) - self.visFr.AddMenuItem('Experimental>NPC3D', "Auto-save NPC template fit statistics",self.OnNPC3DTemplateFitStats_auto_save) - self.visFr.AddMenuItem('Experimental>NPC3D', "Auto-save NPC by-segment data",self.OnNPC3DSaveBySegments_auto_save) - self.visFr.AddMenuItem('Experimental>NPC3D', "Auto-save NPC by-segment plot",self.OnNPC3DPlotBySegments_auto_save) - - # --- End of Alex B addition for auto-saving NPC sets --- + visFr.AddMenuItem('Experimental>NPC3D', 'Analysis 3D NPCs by ID and save all outputs', self.OnNPC3DRunAllActions) # Add combined MenuItem for all requested actions self._npcsettings = None @@ -144,27 +133,24 @@ def __init__(self, visFr): self.segment_layer = None # --- Alex B addition test for running several action at once --- - # Add combined MenuItem for all requested actions - visFr.AddMenuItem('Experimental>NPC3D', 'Save All NPC 3D Analysis Actions', self.OnNPC3DRunAllActions) def OnNPC3DRunAllActions(self, event=None): """Performs all key NPC 3D analysis actions in sequence.""" + # Analyse NPC by mask and auto-save + self.OnAnalyse3DNPCsByID_auto_save() # Save NPC Set with full fit analysis - self.OnNPC3DSaveNPCSet() + self.OnNPC3DSaveNPCSet_auto_save() # Save Measurements Only (csv, no fit info saved) - self.OnNPC3DSaveMeasurements() + self.OnNPC3DSaveMeasurements_auto_save() # Show NPC geometry statistics - self.OnNPC3DGeometryStats() + self.OnNPC3DGeometryStats_auto_save() # Show NPC template fit statistics - self.OnNPC3DTemplateFitStats() + self.OnNPC3DTemplateFitStats_auto_save() # Plot NPC by-segment data - self.OnNPC3DPlotBySegments() + self.OnNPC3DPlotBySegments_auto_save() # Save NPC by-segment data - self.OnNPC3DSaveBySegments() - + self.OnNPC3DSaveBySegments_auto_save() # --- End of Alex B addition test for running several action at once --- - - @property def NPCsettings(self): if self._npcsettings is None: From 912c522f82eaa19b719541776bc8b89748d47c52 Mon Sep 17 00:00:00 2001 From: AlexFEBo Date: Tue, 9 Sep 2025 10:42:10 +0200 Subject: [PATCH 15/27] ChatGPT suggestion for prompting user for a saving directory --- PYMEcs/experimental/NPCcalcLM.py | 104 ++++++++++++++++--------------- 1 file changed, 55 insertions(+), 49 deletions(-) diff --git a/PYMEcs/experimental/NPCcalcLM.py b/PYMEcs/experimental/NPCcalcLM.py index 3e03f0e..e51a0a7 100644 --- a/PYMEcs/experimental/NPCcalcLM.py +++ b/PYMEcs/experimental/NPCcalcLM.py @@ -5,12 +5,11 @@ import numpy as np import wx from PYME.recipes import tablefilters -from traits.api import Bool, Enum, Float, HasTraits, Int - from PYMEcs.Analysis.NPC import estimate_nlabeled, npclabel_fit, plotcdf_npc3d from PYMEcs.IO.NPC import findNPCset from PYMEcs.misc.utils import unique_name from PYMEcs.pyme_warnings import warn +from traits.api import Bool, Enum, Float, HasTraits, Int logger = logging.getLogger(__name__) @@ -63,7 +62,7 @@ class NPCsettings(HasTraits): SecondPass_2D = Bool(False,label='Second pass for NPC fitting (2D)', desc="a second pass for 2D fitting should be run; we have experimented with a second pass rotation "+ "estimate and fitting hoping to improve on the first estimate; still experimental") - StartHeight_3D = Float(70.0,label='Starting ring spacing for 3D fitting', + StartHeight_3D = Float(50.0,label='Starting ring spacing for 3D fitting', desc="starting ring spacing value for the 3D fit; note only considered when doing the initial full fit; "+ "not considered when re-evaluating existing fit") StartDiam_3D = Float(107.0,label='Starting ring diameter for 3D fitting', @@ -119,6 +118,7 @@ def __init__(self, visFr): visFr.AddMenuItem('Experimental>NPC3D', "Save Measurements Only (csv, no fit info saved)",self.OnNPC3DSaveMeasurements) visFr.AddMenuItem('Experimental>NPC3D', "Load and display saved Measurements (from csv)",self.OnNPC3DLoadMeasurements) visFr.AddMenuItem('Experimental>NPC3D', "Show NPC geometry statistics",self.OnNPC3DGeometryStats) + visFr.AddMenuItem('Experimental>NPC3D', "Save NPC geometry statistics as CSV",self.OnNPC3DSaveGeometryStats) visFr.AddMenuItem('Experimental>NPC3D', "Show NPC template fit statistics",self.OnNPC3DTemplateFitStats) visFr.AddMenuItem('Experimental>NPC2D', 'NPC Analysis settings', self.OnNPCsettings) visFr.AddMenuItem('Experimental>NPC3D', 'NPC Analysis settings', self.OnNPCsettings) @@ -134,28 +134,32 @@ def __init__(self, visFr): # --- Alex B addition test for running several action at once --- def OnNPC3DRunAllActions(self, event=None): - """Performs all key NPC 3D analysis actions in sequence.""" - # Analyse NPC by mask and auto-save - self.OnAnalyse3DNPCsByID_auto_save() - # Save NPC Set with full fit analysis - self.OnNPC3DSaveNPCSet_auto_save() - # Save Measurements Only (csv, no fit info saved) - self.OnNPC3DSaveMeasurements_auto_save() - # Show NPC geometry statistics - self.OnNPC3DGeometryStats_auto_save() - # Show NPC template fit statistics - self.OnNPC3DTemplateFitStats_auto_save() - # Plot NPC by-segment data - self.OnNPC3DPlotBySegments_auto_save() - # Save NPC by-segment data - self.OnNPC3DSaveBySegments_auto_save() + """Performs all key NPC 3D analysis actions in sequence, prompting user for output folder.""" + # Prompt user for save directory + with wx.DirDialog(self.visFr, "Select folder to save all NPC outputs", style=wx.DD_DEFAULT_STYLE | wx.DD_NEW_DIR_BUTTON) as dirdialog: + if dirdialog.ShowModal() != wx.ID_OK: + return # User cancelled + else: + save_dir = dirdialog.GetPath() + + # Pass save_dir to all auto-save functions + self.OnAnalyse3DNPCsByID_auto_save(save_dir=save_dir) + self.OnNPC3DSaveNPCSet_auto_save(save_dir=save_dir) + self.OnNPC3DSaveMeasurements_auto_save(save_dir=save_dir) + self.OnNPC3DGeometryStats_auto_save(save_dir=save_dir) + self.OnNPC3DTemplateFitStats_auto_save(save_dir=save_dir) + self.OnNPC3DPlotBySegments_auto_save(save_dir=save_dir) + self.OnNPC3DSaveBySegments_auto_save(save_dir=save_dir) # --- End of Alex B addition test for running several action at once --- @property def NPCsettings(self): if self._npcsettings is None: foreshortening=self.visFr.pipeline.mdh.get('MINFLUX.Foreshortening',1.0) - self._npcsettings = NPCsettings(StartHeight_3D=70.0*foreshortening,Zclip_3D=75.0*foreshortening) + if foreshortening < 1.0: + self._npcsettings = NPCsettings(StartHeight_3D=70.0*foreshortening,Zclip_3D=75.0*foreshortening) + else: + self._npcsettings = NPCsettings(StartHeight_3D=50.0,Zclip_3D=55.0) return self._npcsettings def OnNPCsettings(self, event=None): @@ -283,14 +287,14 @@ def OnAnalyse3DNPCsByID(self, event=None): # AIM: perform all actions 3D NPC actions and save output automatically # Original function 'OnAnalyse3DNPCsByID', copied and modified for automatic saving. - def OnAnalyse3DNPCsByID_auto_save(self, event=None): + def OnAnalyse3DNPCsByID_auto_save(self, event=None, save_dir=None): from PYMEcs.Analysis.NPC import NPC3DSet pipeline = self.visFr.pipeline # --- Alex B addition --- # We define a few variables used for automatic saving later - - base_dir = os.getcwd() # Get the working directory + if save_dir is None: + save_dir = os.getcwd() # Default to current working directory MINFLUXts = pipeline.mdh.get('MINFLUX.TimeStamp') # Get the timestamp and use it for naming the file to save if MINFLUXts is not None: fitplot_filename = f"{MINFLUXts}-NPC_fit_plot.png" @@ -298,9 +302,8 @@ def OnAnalyse3DNPCsByID_auto_save(self, event=None): else: # If no timestamp is found, use default filenames fitplot_filename = "NPC_fit_plot.png" leplot_filename = "LE_plot.png" - fitplot_save_path = os.path.join(base_dir, fitplot_filename) # Save path for fit plot - leplot_save_path = os.path.join(base_dir, leplot_filename) # Save path for LE plot - + fitplot_save_path = os.path.join(save_dir, fitplot_filename) # Save path for fit plot + leplot_save_path = os.path.join(save_dir, leplot_filename) # Save path for LE plot # --- End Alex B addition --- if findNPCset(pipeline,warnings=False) is not None: @@ -415,18 +418,19 @@ def OnNPC3DSaveBySegments(self, event=None): # --- Alex B addition --- - def OnNPC3DSaveBySegments_auto_save(self, event=None): + def OnNPC3DSaveBySegments_auto_save(self, event=None, save_dir=None): pipeline = self.visFr.pipeline # --- Alex B addition --- # We define a few variables used for automatic saving later - base_dir = os.getcwd() # Get the working directory + if save_dir is None: + save_dir = os.getcwd() MINFLUXts = pipeline.mdh.get('MINFLUX.TimeStamp') # Get the timestamp and use it for naming the file to save if MINFLUXts is not None: NPC_segments_stats = f"{MINFLUXts}-NPC_segments.csv" else: NPC_segments_stats = "NPC_segments.csv" - NPC_segments_stats_save_path = os.path.join(base_dir, NPC_segments_stats) # Save path for csv file + NPC_segments_stats_save_path = os.path.join(save_dir, NPC_segments_stats) # Save path for csv file # --- End of Alex B addition --- if findNPCset(pipeline) is not None: @@ -481,19 +485,20 @@ def OnNPC3DPlotBySegments(self, event=None): # --- Alex B addition --- - def OnNPC3DPlotBySegments_auto_save(self, event=None): + def OnNPC3DPlotBySegments_auto_save(self, event=None, save_dir=None): pipeline = self.visFr.pipeline # --- Alex B addition --- # We define a few variables used for automatic saving later - base_dir = os.getcwd() # Get the working directory + if save_dir is None: + save_dir = os.getcwd() MINFLUXts = pipeline.mdh.get('MINFLUX.TimeStamp') # Get the timestamp and use it for naming the file to save if MINFLUXts is not None: NPC_plot_segments = f"{MINFLUXts}-NPC_segments.png" else: NPC_plot_segments = "NPC_segments.png" - NPC_plot_segments_save_path = os.path.join(base_dir, NPC_plot_segments) # Save path for csv file + NPC_plot_segments_save_path = os.path.join(save_dir, NPC_plot_segments) # Save path for csv file # --- End of Alex B addition --- @@ -591,9 +596,8 @@ def On3DNPCaddTemplates(self, event=None): # now we add a track layer to render our template polygons # TODO - we may need to check if this happened before or not! - from PYME.LMVis.layers.tracks import ( - TrackRenderLayer, # NOTE: we may rename the clumpIndex variable in this layer to polyIndex or similar - ) + from PYME.LMVis.layers.tracks import \ + TrackRenderLayer # NOTE: we may rename the clumpIndex variable in this layer to polyIndex or similar layer = TrackRenderLayer(pipeline, dsname=ds_template_name, method='tracks', clump_key='polyIndex', line_width=2.0, alpha=0.5) self.visFr.add_layer(layer) @@ -663,7 +667,7 @@ def OnNPC3DSaveMeasurements(self, event=None): # --- Alex B addition for auto-save of measurements --- - def OnNPC3DSaveMeasurements_auto_save(self, event=None): + def OnNPC3DSaveMeasurements_auto_save(self, event=None, save_dir=None): import pandas as pd pipeline = self.visFr.pipeline npcs = findNPCset(pipeline) @@ -682,13 +686,14 @@ def OnNPC3DSaveMeasurements_auto_save(self, event=None): # --- Alex B addition --- # We define a few variables used for automatic saving later - base_dir = os.getcwd() # Get the working directory + if save_dir is None: + save_dir = os.getcwd() MINFLUXts = pipeline.mdh.get('MINFLUX.TimeStamp') # Get the timestamp and use it for naming the file to save if MINFLUXts is not None: csv_filename = f"{MINFLUXts}-LE_stats.csv" else: csv_filename = "LE_stats.csv" - csv_save_path = os.path.join(base_dir, csv_filename) # Save path for csv file + csv_save_path = os.path.join(save_dir, csv_filename) # Save path for csv file # --- End of Alex B addition --- @@ -800,7 +805,7 @@ def OnNPC3DSaveNPCSet(self, event=None): # Original function 'OnNPC3DSaveNPCSet', copied and modified for automatic saving. # Works fine as single action (new button: OnNPC3DSaveNPCSet_auto_save) line 130 - def OnNPC3DSaveNPCSet_auto_save(self, event=None): + def OnNPC3DSaveNPCSet_auto_save(self, event=None, save_dir=None): """Automatically save the NPC set to a default file path without user dialog.""" from PYMEcs.IO.NPC import save_NPC_set @@ -814,9 +819,10 @@ def OnNPC3DSaveNPCSet_auto_save(self, event=None): defaultFile = f"{MINFLUXts}-NPCset.pickle" else: defaultFile = "NPCset.pickle" - # Save in the current directory (same as the session file - base_dir = os.getcwd() - save_path = os.path.join(base_dir, defaultFile) + # Save in the selected directory (or current directory if not provided) + if save_dir is None: + save_dir = os.getcwd() + save_path = os.path.join(save_dir, defaultFile) # Find the current NPC set in the pipeline npcs = findNPCset(pipeline) print(f"Attempting to automatically save NPC Set to: {save_path}.") @@ -836,7 +842,6 @@ def OnNPC3DGeometryStats(self,event=None): warn('no valid NPC measurements found, thus no geometry info available...') return import pandas as pd - from PYMEcs.misc.matplotlib import boxswarmplot, figuredefaults diams = np.asarray(npcs.diam()) heights = np.asarray(npcs.height()) @@ -858,7 +863,7 @@ def OnNPC3DGeometryStats(self,event=None): # --- Alex B addition --- # Origimnal function 'OnNPC3DGeometryStats', copied and modified for automatic saving. - def OnNPC3DGeometryStats_auto_save(self,event=None): + def OnNPC3DGeometryStats_auto_save(self,event=None, save_dir=None): pipeline = self.visFr.pipeline npcs = findNPCset(pipeline) if npcs is None: @@ -869,13 +874,14 @@ def OnNPC3DGeometryStats_auto_save(self,event=None): # --- Alex B addition --- # We define a few variables used for automatic saving later - base_dir = os.getcwd() # Get the working directory + if save_dir is None: + save_dir = os.getcwd() MINFLUXts = pipeline.mdh.get('MINFLUX.TimeStamp') # Get the timestamp and use it for naming the file to save if MINFLUXts is not None: geom_stats_fig = f"{MINFLUXts}-Geom_stats.png" else: geom_stats_fig = "Geom_stats.png" - geom_stats_fig_save_path = os.path.join(base_dir, geom_stats_fig) # Save path for csv file + geom_stats_fig_save_path = os.path.join(save_dir, geom_stats_fig) # Save path for csv file # --- End of Alex B addition --- @@ -910,7 +916,6 @@ def OnNPC3DTemplateFitStats(self,event=None): warn('no valid NPC measurements found, thus no geometry info available...') return import pandas as pd - from PYMEcs.misc.matplotlib import boxswarmplot, figuredefaults id = [npc.objectID for npc in npcs.npcs] # not used right now llperloc = [npc.opt_result.fun/npc.npts.shape[0] for npc in npcs.npcs] @@ -923,7 +928,7 @@ def OnNPC3DTemplateFitStats(self,event=None): # --- Alex B addition --- # Original function 'OnNPC3DTemplateFitStats', copied and modified for automatic saving. - def OnNPC3DTemplateFitStats_auto_save(self,event=None): + def OnNPC3DTemplateFitStats_auto_save(self,event=None, save_dir=None): pipeline = self.visFr.pipeline npcs = findNPCset(pipeline) if npcs is None: @@ -934,13 +939,14 @@ def OnNPC3DTemplateFitStats_auto_save(self,event=None): # --- Alex B addition --- # We define a few variables used for automatic saving later - base_dir = os.getcwd() # Get the working directory + if save_dir is None: + save_dir = os.getcwd() MINFLUXts = pipeline.mdh.get('MINFLUX.TimeStamp') # Get the timestamp and use it for naming the file to save if MINFLUXts is not None: template_fit_stats_fig = f"{MINFLUXts}-Template_fit_stats.png" else: template_fit_stats_fig = "Template_fit_stats.png" - template_fit_stats_fig_save_path = os.path.join(base_dir, template_fit_stats_fig) # Save path for csv file + template_fit_stats_fig_save_path = os.path.join(save_dir, template_fit_stats_fig) # Save path for csv file # --- End of Alex B addition --- From 5112247a3b9196124102b6e4ccd41a84af446d5f Mon Sep 17 00:00:00 2001 From: Alexfebo Date: Tue, 9 Sep 2025 11:54:55 +0200 Subject: [PATCH 16/27] Restore autosave of NPC3Dset --- PYMEcs/experimental/NPCcalcLM.py | 34 ++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/PYMEcs/experimental/NPCcalcLM.py b/PYMEcs/experimental/NPCcalcLM.py index e1def48..35938a9 100644 --- a/PYMEcs/experimental/NPCcalcLM.py +++ b/PYMEcs/experimental/NPCcalcLM.py @@ -798,6 +798,40 @@ def OnNPC3DSaveNPCSet(self, event=None): return save_NPC_set(npcs,fdialog.GetPath()) + +# --- Alex B addition --- +# AIM: perform all actions 3D NPC actions and save output automatically +# Original function 'OnNPC3DSaveNPCSet', copied and modified for automatic saving. +# Works fine as single action (new button: OnNPC3DSaveNPCSet_auto_save) line 130 + + def OnNPC3DSaveNPCSet_auto_save(self, event=None): + """Automatically save the NPC set to a default file path without user dialog.""" + + from PYMEcs.IO.NPC import save_NPC_set + + pipeline = self.visFr.pipeline + + # Get the MINFLUX timestamp from the pipeline metadata, if available + MINFLUXts = pipeline.mdh.get('MINFLUX.TimeStamp') + # Construct the default filename using the timestamp if present + if MINFLUXts is not None: + defaultFile = f"{MINFLUXts}-NPCset.pickle" + else: + defaultFile = "NPCset.pickle" + # Save in the current directory (same as the session file + base_dir = os.getcwd() + save_path = os.path.join(base_dir, defaultFile) + # Find the current NPC set in the pipeline + npcs = findNPCset(pipeline) + print(f"Attempting to automatically save NPC Set to: {save_path}.") + # If no NPC set is found, warn and exit + if npcs is None: + warn('no valid NPC Set found, therefore cannot save...') + return + # Save the NPC set to the constructed path + save_NPC_set(npcs, save_path) + +# --- End of Alex B addition --- def OnNPC3DSaveGeometryStats(self,event=None): pipeline = self.visFr.pipeline From d2603420243cfd3521cbaed2d9a26cb3e8020bb0 Mon Sep 17 00:00:00 2001 From: Alexfebo Date: Tue, 9 Sep 2025 12:02:09 +0200 Subject: [PATCH 17/27] Correct NPC3Dset autosave function Add the save_dir as an argument --- PYMEcs/experimental/NPCcalcLM.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/PYMEcs/experimental/NPCcalcLM.py b/PYMEcs/experimental/NPCcalcLM.py index 35938a9..15fda15 100644 --- a/PYMEcs/experimental/NPCcalcLM.py +++ b/PYMEcs/experimental/NPCcalcLM.py @@ -804,7 +804,7 @@ def OnNPC3DSaveNPCSet(self, event=None): # Original function 'OnNPC3DSaveNPCSet', copied and modified for automatic saving. # Works fine as single action (new button: OnNPC3DSaveNPCSet_auto_save) line 130 - def OnNPC3DSaveNPCSet_auto_save(self, event=None): + def OnNPC3DSaveNPCSet_auto_save(self, event=None, save_dir=None): """Automatically save the NPC set to a default file path without user dialog.""" from PYMEcs.IO.NPC import save_NPC_set @@ -819,8 +819,9 @@ def OnNPC3DSaveNPCSet_auto_save(self, event=None): else: defaultFile = "NPCset.pickle" # Save in the current directory (same as the session file - base_dir = os.getcwd() - save_path = os.path.join(base_dir, defaultFile) + if save_dir is None: + save_dir = os.getcwd() + save_path = os.path.join(save_dir, defaultFile) # Find the current NPC set in the pipeline npcs = findNPCset(pipeline) print(f"Attempting to automatically save NPC Set to: {save_path}.") From e3ce3ebb9387af05f5d00718e04625bfc8fbe5d4 Mon Sep 17 00:00:00 2001 From: Alexfebo Date: Fri, 12 Sep 2025 16:54:49 +0200 Subject: [PATCH 18/27] Add SaveGeomStats to autosave NPC3D function --- PYMEcs/experimental/NPCcalcLM.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/PYMEcs/experimental/NPCcalcLM.py b/PYMEcs/experimental/NPCcalcLM.py index 15fda15..40fc0de 100644 --- a/PYMEcs/experimental/NPCcalcLM.py +++ b/PYMEcs/experimental/NPCcalcLM.py @@ -150,6 +150,7 @@ def OnNPC3DRunAllActions(self, event=None): self.OnNPC3DTemplateFitStats_auto_save(save_dir=save_dir) self.OnNPC3DPlotBySegments_auto_save(save_dir=save_dir) self.OnNPC3DSaveBySegments_auto_save(save_dir=save_dir) + self.OnNPC3DSaveGeometryStats_auto_save(save_dir=save_dir) # --- End of Alex B addition test for running several action at once --- @property @@ -852,6 +853,37 @@ def OnNPC3DSaveGeometryStats(self,event=None): fpath = fdialog.GetPath() geo_df.to_csv(fpath,index=False) + +# --- Alex B addition --- +# Origimnal function 'OnNPC3DSaveGeometryStats', copied and modified for automatic saving. + + def OnNPC3DSaveGeometryStats_auto_save(self,event=None, save_dir=None): + pipeline = self.visFr.pipeline + npcs = findNPCset(pipeline) + if npcs is None: + warn('no valid NPC measurements found, thus no geometry info available...') + return + diams = np.asarray(npcs.diam()) + heights = np.asarray(npcs.height()) + import pandas as pd + geo_df = pd.DataFrame.from_dict(dict(diameter=diams,height=heights)) + # with wx.FileDialog(self.visFr, 'Save NPC measurements as ...', + # wildcard='CSV (*.csv)|*.csv', + # style=wx.FD_SAVE) as fdialog: + # if fdialog.ShowModal() != wx.ID_OK: + # return + # fpath = fdialog.GetPath() + if save_dir is None: + save_dir = os.getcwd() + MINFLUXts = pipeline.mdh.get('MINFLUX.TimeStamp') # Get the timestamp and use it for naming the file to save + if MINFLUXts is not None: + geo_stats_csv = f"{MINFLUXts}-NPC_geometry_stats.csv" + else: + geo_stats_csv = "NPC_geometry_stats.csv" + geo_df.to_csv(os.path.join(save_dir, geo_stats_csv), index=False) + +# --- End of Alex B addition --- + def OnNPC3DGeometryStats(self,event=None): pipeline = self.visFr.pipeline From 4ca025c588c02f93553e0bcfad3d32fb31e70fdf Mon Sep 17 00:00:00 2001 From: Alexfebo Date: Tue, 23 Sep 2025 09:22:31 +0200 Subject: [PATCH 19/27] Saving stats from Paraflux Itr working version --- PYMEcs/experimental/MINFLUX.py | 173 ++++++++++++++++++++++++++++++++- 1 file changed, 172 insertions(+), 1 deletion(-) diff --git a/PYMEcs/experimental/MINFLUX.py b/PYMEcs/experimental/MINFLUX.py index cf9cff0..5765149 100644 --- a/PYMEcs/experimental/MINFLUX.py +++ b/PYMEcs/experimental/MINFLUX.py @@ -222,7 +222,7 @@ def _plot_clustersize_counts(cts, ctsgt1, xlabel='Cluster Size', wintitle=None, def plot_cluster_analysis(pipeline, ds='dbscanClustered',showPlot=True, return_means=False, return_data=False, psu=None, bins=15, bigc_thresh=50, **kwargs): - if not ds in pipeline.dataSources: + if ds not in pipeline.dataSources: warn('no data source named "%s" - check recipe and ensure this is MINFLUX data' % ds) return curds = pipeline.selectedDataSourceKey @@ -611,6 +611,9 @@ def __init__(self, visFr): visFr.AddMenuItem('MINFLUX>Tracking', "Add traces as tracks (from clumpIndex)", self.OnAddMINFLUXTracksCI) visFr.AddMenuItem('MINFLUX>Tracking', "Add traces as tracks (from tid)", self.OnAddMINFLUXTracksTid) visFr.AddMenuItem('MINFLUX>Colour', "Plot colour stats", self.OnPlotColourStats) + # --- Alex B test addition --- + visFr.AddMenuItem('MINFLUX>Paraflux', "Run Paraflux Analysis", self.OnRunParafluxAnalysis) + # --- End of Alex B test addition --- # this section establishes Menu entries for loading MINFLUX recipes in one click # these recipes should be MINFLUX processing recipes of general interest @@ -624,6 +627,174 @@ def __init__(self, visFr): ID = visFr.AddMenuItem('MINFLUX>Recipes', r, self.OnLoadCustom).GetId() self.minfluxRIDs[ID] = minfluxRecipes[r] + # --- Alex B test addition funciton --- + def OnRunParafluxAnalysis(self, event): + import os + from pathlib import Path + + import numpy as np + import pandas as pd + import wx + import zarr + + # Add if statement to check if pipeline is available + # Access the pipeline from the visFr object and get the timestamp (used for saving) + pipeline = self.visFr.pipeline + timestamp = pipeline.mdh.get('MINFLUX.TimeStamp') + # TODO: Get the zarr.zip file path from the pipeline if possible + # For now, we will prompt the user to select the file + + # ===================================================== + # --- Step 1: Select and load Zarr.zip file --- + # ===================================================== + if store_path is None: + with wx.FileDialog( + self.visFr, + 'Choose a Zarr.zip to open ...', + wildcard='ZIP (*.zip)|*.zip', + style=wx.FD_OPEN | wx.FD_FILE_MUST_EXIST + ) as fdialog: + if fdialog.ShowModal() != wx.ID_OK: + return + store_path = Path(fdialog.GetPath()) + + # --- Open the archive --- + # defaultDir = str(Path(store_path).parent) + # store = zarr.storage.ZipStore(str(store_path), mode='r') + # archz = zarr.open(store, mode='r') + + # Load mfx data + mfxdata = archz['mfx'][:] + + # ===================================================== + # --- Step 2: Convert to DataFrame --- + # ===================================================== + df_mfx = pd.DataFrame() + for name in mfxdata.dtype.names: + col = mfxdata[name] + if col.ndim == 1: + df_mfx[name] = col + else: + n_dim = col.shape[1] + expanded = pd.DataFrame(col.tolist(), + index=df_mfx.index if not df_mfx.empty else None) + if name in ['loc', 'lnc']: + labels = ["x", "y", "z"] + expanded.columns = [f"{name}_{labels[i]}" for i in range(n_dim)] + elif name == 'dcr': + expanded.columns = [f"{name}_{i+1}" for i in range(n_dim)] + else: + expanded.columns = [f"{name}_{i}" for i in range(n_dim)] + df_mfx = pd.concat([df_mfx, expanded], axis=1) + + # ===================================================== + # --- Step 3: Run analysis pipeline --- + # ===================================================== + failure_map = { + 1: "Valid final", 2: "Valid not final", + 4: "Derived iteration", 5: "Reserved", + 6: "CFR failure", 8: "No signal", + 9: "DAC out of range", 11: "Background measurement" + } + pair_mapping = {1: 0, 2: 1, 3: 1, 4: 2, 5: 2, 6: 3, 7: 3, 8: 4, 9: 4} + pairs = [(0,1), (1,2), (2,3), (3,4), (4,5), + (5,6), (6,7), (7,8), (8,9)] + + def build_valid_tid_table(df): + vld = df[df['vld']].groupby('itr')['tid'].apply(lambda x: list(set(x))).reset_index() + vld['vld_loc_counts'] = vld['tid'].apply(len) + vld['failed_loc'] = vld['vld_loc_counts'].shift(1) - vld['vld_loc_counts'] + vld['Axis'] = np.where(vld['itr'] % 2 == 0, 'x,y', 'z') + vld.insert(1, 'Axis', vld.pop('Axis')) + vld['Cumulative sum of failed_loc'] = vld['failed_loc'].cumsum() + return vld + + def compute_passed_itr(vld): + initial_count = vld['vld_loc_counts'].iloc[0] + vld['passed_itr %'] = (vld['vld_loc_counts'] * 100 / initial_count).round(1) + return vld + + def compute_failed_sums(vld, pair_mapping): + vld['pair_group'] = vld['itr'].map(pair_mapping) + failed_sum_map = vld.groupby('pair_group')['failed_loc'].transform('sum') + vld['failed_sum'] = np.where( + vld['pair_group'].notna() & (vld['itr'] % 2 == 1), + failed_sum_map, + np.nan + ) + vld.drop(columns='pair_group', inplace=True) + initial_count = vld['vld_loc_counts'].iloc[0] + vld['failed %'] = (vld['failed_sum'] / initial_count * 100).round(1) + return vld + + def analyze_failures(vld, df, itr_from, itr_to, failure_map): + tids_from = set(vld.loc[vld['itr'] == itr_from, 'tid'].values[0]) + tids_to = set(vld.loc[vld['itr'] == itr_to, 'tid'].values[0]) + failed_tids = tids_from - tids_to + failed_df = df[df['tid'].isin(failed_tids) & (df['itr'] == itr_to)] + counts = failed_df['sta'].value_counts().rename_axis("sta").reset_index(name="count") + counts["reason"] = counts["sta"].map(failure_map).fillna("Other") + counts.insert(0, "itr", itr_to) + return counts + + def compute_failure_pivot(vld, df, pairs, failure_map): + failure_results = pd.concat( + [analyze_failures(vld, df, i_from, i_to, failure_map) for i_from, i_to in pairs], + ignore_index=True + ) + failure_pivot = failure_results.pivot_table( + index="itr", columns="reason", values="count", fill_value=0 + ).reset_index() + return vld.merge(failure_pivot, on="itr", how="left") + + def compute_cfr_and_no_signal(vld): + initial_count = vld['vld_loc_counts'].iloc[0] + vld['CFR failure %'] = np.where( + vld['itr'].isin([4, 6]), + (vld['CFR failure'] / initial_count * 100).round(1), + np.nan + ) + no_signal_groups = {5: [4, 5], 7: [6, 7]} + no_signal_pct = {} + for target_itr, group in no_signal_groups.items(): + total_no_signal = vld.loc[vld['itr'].isin(group), 'No signal'].sum() + no_signal_pct[target_itr] = (total_no_signal / initial_count * 100).round(1) + vld['No signal %'] = vld['itr'].map(no_signal_pct) + return vld + + # Run pipeline + vld = build_valid_tid_table(df_mfx) + vld = compute_passed_itr(vld) + vld = compute_failed_sums(vld, pair_mapping) + vld = compute_failure_pivot(vld, df_mfx, pairs, failure_map) + vld = compute_cfr_and_no_signal(vld) + + # ===================================================== + # --- Step 4: Save results --- + # ===================================================== + vld_full = vld.drop(columns='tid', errors='ignore') + default_dir = str(store_path.parent) + full_path = os.path.join(default_dir, "paraflux_stats_full.csv") + clean_path = os.path.join(default_dir, "paraflux_stats_clean.csv") + + vld_full.to_csv(full_path, index=False) + + keep_cols = ["itr", "Axis", "vld_loc_counts", + "failed_loc", "failed %", "CFR failure %", "No signal %"] + vld_clean = vld_full[keep_cols] + vld_clean.to_csv(clean_path, index=False) + + print(f"✔ Saved full results to: {full_path}") + print(f"✔ Saved cleaned results to: {clean_path}") + + # ===================================================== + # --- Step 5: Return results --- + # ===================================================== + return vld + + # --- end of Alex B addition --- + + def OnClumpScatterPosPlot(self,event): from scipy.stats import binned_statistic From b9393cf1fb245045f4e8e2ee5ae507754b1ec86b Mon Sep 17 00:00:00 2001 From: Alexfebo Date: Tue, 23 Sep 2025 11:46:22 +0200 Subject: [PATCH 20/27] get the filename and path from pipeline --- PYMEcs/experimental/MINFLUX.py | 48 ++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/PYMEcs/experimental/MINFLUX.py b/PYMEcs/experimental/MINFLUX.py index 5765149..5ddca85 100644 --- a/PYMEcs/experimental/MINFLUX.py +++ b/PYMEcs/experimental/MINFLUX.py @@ -4,6 +4,7 @@ import matplotlib.pyplot as plt import numpy as np import wx +import zarr logger = logging.getLogger(__file__) @@ -635,33 +636,40 @@ def OnRunParafluxAnalysis(self, event): import numpy as np import pandas as pd import wx - import zarr - - # Add if statement to check if pipeline is available - # Access the pipeline from the visFr object and get the timestamp (used for saving) - pipeline = self.visFr.pipeline - timestamp = pipeline.mdh.get('MINFLUX.TimeStamp') - # TODO: Get the zarr.zip file path from the pipeline if possible - # For now, we will prompt the user to select the file # ===================================================== # --- Step 1: Select and load Zarr.zip file --- # ===================================================== + # Access the pipeline from the visFr object + pipeline = self.visFr.pipeline + + # Check for pipeline existence + if pipeline is None: + Error(self.visFr, "No active pipeline found. Please load a MINFLUX dataset first.") + return + + # Try to get the FitResults path from the pipeline's datasources + datasources = pipeline._get_session_datasources() + store_path = datasources.get('FitResults') + + # Fallback to file dialog if no FitResults path is found if store_path is None: + print("FitResults path not found. Asking user to select a Zarr.zip file.") with wx.FileDialog( self.visFr, - 'Choose a Zarr.zip to open ...', + 'Select a Zarr.zip to open ...', wildcard='ZIP (*.zip)|*.zip', style=wx.FD_OPEN | wx.FD_FILE_MUST_EXIST ) as fdialog: if fdialog.ShowModal() != wx.ID_OK: return store_path = Path(fdialog.GetPath()) - - # --- Open the archive --- - # defaultDir = str(Path(store_path).parent) - # store = zarr.storage.ZipStore(str(store_path), mode='r') - # archz = zarr.open(store, mode='r') + else: + store_path = Path(store_path) + + # Open the Zarr file from .zip + store = zarr.storage.ZipStore(str(store_path), mode='r') + archz = zarr.open(store, mode='r') # Load mfx data mfxdata = archz['mfx'][:] @@ -772,10 +780,18 @@ def compute_cfr_and_no_signal(vld): # ===================================================== # --- Step 4: Save results --- # ===================================================== + + # Get the timestamp to append to the filename + try: + timestamp = pipeline.mdh.get('MINFLUX.TimeStamp') + except Exception: + print("No timestamp found in metadata, saving as default.") + timestamp = "default" + vld_full = vld.drop(columns='tid', errors='ignore') default_dir = str(store_path.parent) - full_path = os.path.join(default_dir, "paraflux_stats_full.csv") - clean_path = os.path.join(default_dir, "paraflux_stats_clean.csv") + full_path = os.path.join(default_dir, f"paraflux_stats_full_{timestamp}.csv") + clean_path = os.path.join(default_dir, f"paraflux_stats_clean_{timestamp}.csv") vld_full.to_csv(full_path, index=False) From 1d9e1e0d4521b57f5fec8001e2e9109fd10919f4 Mon Sep 17 00:00:00 2001 From: Alexfebo Date: Thu, 2 Oct 2025 11:41:04 +0200 Subject: [PATCH 21/27] Update OnRunParafluxAnalysis Now saves 2 csv of Stats from all Iteration: 1. Identical to Paraflux data 2. For all Itr --- PYMEcs/experimental/MINFLUX.py | 351 ++++++++++++++++++++++----------- 1 file changed, 238 insertions(+), 113 deletions(-) diff --git a/PYMEcs/experimental/MINFLUX.py b/PYMEcs/experimental/MINFLUX.py index 5ddca85..e1f52f1 100644 --- a/PYMEcs/experimental/MINFLUX.py +++ b/PYMEcs/experimental/MINFLUX.py @@ -628,31 +628,22 @@ def __init__(self, visFr): ID = visFr.AddMenuItem('MINFLUX>Recipes', r, self.OnLoadCustom).GetId() self.minfluxRIDs[ID] = minfluxRecipes[r] - # --- Alex B test addition funciton --- +# --- Alex B test addition function --- def OnRunParafluxAnalysis(self, event): - import os from pathlib import Path - import numpy as np - import pandas as pd - import wx - # ===================================================== # --- Step 1: Select and load Zarr.zip file --- # ===================================================== - # Access the pipeline from the visFr object - pipeline = self.visFr.pipeline + pipeline = self.visFr.pipeline # Get the pipeline from the GUI - # Check for pipeline existence if pipeline is None: - Error(self.visFr, "No active pipeline found. Please load a MINFLUX dataset first.") + Error(self.visFr, "No data found. Please load a MINFLUX dataset first.") return - # Try to get the FitResults path from the pipeline's datasources datasources = pipeline._get_session_datasources() store_path = datasources.get('FitResults') - # Fallback to file dialog if no FitResults path is found if store_path is None: print("FitResults path not found. Asking user to select a Zarr.zip file.") with wx.FileDialog( @@ -667,16 +658,11 @@ def OnRunParafluxAnalysis(self, event): else: store_path = Path(store_path) - # Open the Zarr file from .zip store = zarr.storage.ZipStore(str(store_path), mode='r') archz = zarr.open(store, mode='r') - - # Load mfx data mfxdata = archz['mfx'][:] - # ===================================================== - # --- Step 2: Convert to DataFrame --- - # ===================================================== + # Convert structured array (original mfx data) to DataFrame (containing all data from MFX experiment) df_mfx = pd.DataFrame() for name in mfxdata.dtype.names: col = mfxdata[name] @@ -684,8 +670,7 @@ def OnRunParafluxAnalysis(self, event): df_mfx[name] = col else: n_dim = col.shape[1] - expanded = pd.DataFrame(col.tolist(), - index=df_mfx.index if not df_mfx.empty else None) + expanded = pd.DataFrame(col.tolist(), index=df_mfx.index if not df_mfx.empty else None) if name in ['loc', 'lnc']: labels = ["x", "y", "z"] expanded.columns = [f"{name}_{labels[i]}" for i in range(n_dim)] @@ -695,120 +680,260 @@ def OnRunParafluxAnalysis(self, event): expanded.columns = [f"{name}_{i}" for i in range(n_dim)] df_mfx = pd.concat([df_mfx, expanded], axis=1) - # ===================================================== - # --- Step 3: Run analysis pipeline --- - # ===================================================== + # Create failure_map (used to interpret failure reasons in analyze_failures) failure_map = { 1: "Valid final", 2: "Valid not final", 4: "Derived iteration", 5: "Reserved", 6: "CFR failure", 8: "No signal", 9: "DAC out of range", 11: "Background measurement" } - pair_mapping = {1: 0, 2: 1, 3: 1, 4: 2, 5: 2, 6: 3, 7: 3, 8: 4, 9: 4} - pairs = [(0,1), (1,2), (2,3), (3,4), (4,5), - (5,6), (6,7), (7,8), (8,9)] - - def build_valid_tid_table(df): - vld = df[df['vld']].groupby('itr')['tid'].apply(lambda x: list(set(x))).reset_index() - vld['vld_loc_counts'] = vld['tid'].apply(len) - vld['failed_loc'] = vld['vld_loc_counts'].shift(1) - vld['vld_loc_counts'] - vld['Axis'] = np.where(vld['itr'] % 2 == 0, 'x,y', 'z') - vld.insert(1, 'Axis', vld.pop('Axis')) - vld['Cumulative sum of failed_loc'] = vld['failed_loc'].cumsum() - return vld - - def compute_passed_itr(vld): - initial_count = vld['vld_loc_counts'].iloc[0] - vld['passed_itr %'] = (vld['vld_loc_counts'] * 100 / initial_count).round(1) - return vld - - def compute_failed_sums(vld, pair_mapping): - vld['pair_group'] = vld['itr'].map(pair_mapping) - failed_sum_map = vld.groupby('pair_group')['failed_loc'].transform('sum') - vld['failed_sum'] = np.where( - vld['pair_group'].notna() & (vld['itr'] % 2 == 1), - failed_sum_map, - np.nan - ) - vld.drop(columns='pair_group', inplace=True) - initial_count = vld['vld_loc_counts'].iloc[0] - vld['failed %'] = (vld['failed_sum'] / initial_count * 100).round(1) - return vld - - def analyze_failures(vld, df, itr_from, itr_to, failure_map): - tids_from = set(vld.loc[vld['itr'] == itr_from, 'tid'].values[0]) - tids_to = set(vld.loc[vld['itr'] == itr_to, 'tid'].values[0]) - failed_tids = tids_from - tids_to + + # Run analysis pipeline + vld = self.compute_vld_stats(df_mfx, failure_map, pipeline, store_path) + return vld + + # ============================================================== + + # Creates a df with valid and failed localizations per iteration + def build_valid_df(self, df_mfx): + if not isinstance(df_mfx, pd.DataFrame): + df = pd.DataFrame(df_mfx) + else: + df = df_mfx.copy() + df_valid = df[df['vld']] + vld_itr = df_valid.groupby('itr')['tid'].apply(lambda x: list(set(x))).reset_index() # Get list of unique tids per iteration + vld_itr['Axis'] = np.where(vld_itr['itr'] % 2 == 0, 'x,y', 'z') # Define axis of each iteration + vld_itr['vld loc count'] = vld_itr['tid'].apply(len) + vld_itr['failed loc count'] = vld_itr['vld loc count'].shift(1, fill_value=vld_itr['vld loc count'].iloc[0]) - vld_itr['vld loc count'] # Calculate failed loc count per iteration + vld_itr.loc[0, 'failed loc count'] = 0 # Set failed loc count of first iteration to 0 (instead of NaN) + vld_itr['failed loc cum sum'] = vld_itr['failed loc count'].cumsum() + return vld_itr + + # Compute percentages of passed and failed localizations (from build_valid_df) + def compute_percentages(self, vld_itr): + initial_count = vld_itr['vld loc count'].iloc[0] # Percentage calculations are based on initial count of valid locs + vld_itr['passed itr %'] = (vld_itr['vld loc count'] * 100 / initial_count).round(1) + vld_itr['failed % per itr'] = (vld_itr['failed loc count'] * 100 / initial_count).round(1) + pair_sums = {} # This is done to mimic results from Paraflux + for i in range(1, len(vld_itr), 2): + pair_sums[i] = vld_itr.loc[i-1:i, 'failed % per itr'].sum().round(1) + vld_itr['failed % per itr pairs'] = vld_itr.index.map(pair_sums) + vld_itr['failed cum sum %'] = vld_itr['failed % per itr'].cumsum().round(1) + return vld_itr + + # Analyze failures between iterations and categorize them based on failure_map + def analyze_failures(self, vld_itr, df_mfx, failure_map): + def analyze_failures_single_steps(vld_itr, df, itr_from, itr_to, failure_map): + tids_from = set(vld_itr.loc[vld_itr['itr'] == itr_from, 'tid'].iloc[0]) # Select valid tids of the previous iteration + tids_to = set(vld_itr.loc[vld_itr['itr'] == itr_to, 'tid'].iloc[0]) # Select valid tids of the current iteration + failed_tids = tids_from - tids_to # Determine tids that failed in the current iteration failed_df = df[df['tid'].isin(failed_tids) & (df['itr'] == itr_to)] counts = failed_df['sta'].value_counts().rename_axis("sta").reset_index(name="count") counts["reason"] = counts["sta"].map(failure_map).fillna("Other") counts.insert(0, "itr", itr_to) return counts - def compute_failure_pivot(vld, df, pairs, failure_map): - failure_results = pd.concat( - [analyze_failures(vld, df, i_from, i_to, failure_map) for i_from, i_to in pairs], - ignore_index=True - ) - failure_pivot = failure_results.pivot_table( - index="itr", columns="reason", values="count", fill_value=0 - ).reset_index() - return vld.merge(failure_pivot, on="itr", how="left") - - def compute_cfr_and_no_signal(vld): - initial_count = vld['vld_loc_counts'].iloc[0] - vld['CFR failure %'] = np.where( - vld['itr'].isin([4, 6]), - (vld['CFR failure'] / initial_count * 100).round(1), - np.nan - ) - no_signal_groups = {5: [4, 5], 7: [6, 7]} - no_signal_pct = {} - for target_itr, group in no_signal_groups.items(): - total_no_signal = vld.loc[vld['itr'].isin(group), 'No signal'].sum() - no_signal_pct[target_itr] = (total_no_signal / initial_count * 100).round(1) - vld['No signal %'] = vld['itr'].map(no_signal_pct) - return vld - - # Run pipeline - vld = build_valid_tid_table(df_mfx) - vld = compute_passed_itr(vld) - vld = compute_failed_sums(vld, pair_mapping) - vld = compute_failure_pivot(vld, df_mfx, pairs, failure_map) - vld = compute_cfr_and_no_signal(vld) - - # ===================================================== - # --- Step 4: Save results --- - # ===================================================== - - # Get the timestamp to append to the filename + pairs = [(i, i+1) for i in range(vld_itr['itr'].max())] + failure_results = pd.concat( + [analyze_failures_single_steps(vld_itr, df_mfx, i_from, i_to, failure_map) for i_from, i_to in pairs], + ignore_index=True + ) + failure_pivot = failure_results.pivot_table( + index="itr", columns="reason", values="count", fill_value=0 + ).reset_index() + return vld_itr.merge(failure_pivot, on="itr", how="left") + + # Compute percentages for failure reasons + def add_failure_metrics(self, vld_itr, initial_count): + cfr_map = {5: 4, 7: 6} + vld_itr['CFR failure %'] = np.nan + for target_itr, source_itr in cfr_map.items(): + if not vld_itr.loc[vld_itr['itr'] == source_itr, 'CFR failure'].empty: + val = vld_itr.loc[vld_itr['itr'] == source_itr, 'CFR failure'].values[0] + vld_itr.loc[vld_itr['itr'] == target_itr, 'CFR failure %'] = (val / initial_count * 100).round(1) + vld_itr['No signal %'] = (vld_itr['No signal'] * 100 / initial_count).round(1) + no_signal_groups = {1: [0, 1],3: [2, 3], 5: [4, 5], 7: [6, 7], 9: [8, 9]} + no_signal_pct = { + target_itr: (vld_itr.loc[vld_itr['itr'].isin(group), 'No signal'].sum() / initial_count * 100).round(1) + for target_itr, group in no_signal_groups.items() + } + vld_itr['No signal % per itr pairs'] = vld_itr['itr'].map(no_signal_pct) + return vld_itr + + # Main function to compute stats of failed and valid localizations and save results + def compute_vld_stats(self, df_mfx, failure_map, pipeline, store_path): + # Run the analysis steps + vld_itr = self.build_valid_df(df_mfx) + vld_itr = self.compute_percentages(vld_itr) + vld_itr = self.analyze_failures(vld_itr, df_mfx, failure_map) + initial_count = vld_itr['vld loc count'].iloc[0] + vld_itr = self.add_failure_metrics(vld_itr, initial_count) + + # Save results to CSV files try: timestamp = pipeline.mdh.get('MINFLUX.TimeStamp') except Exception: print("No timestamp found in metadata, saving as default.") - timestamp = "default" + timestamp = "no_ts" - vld_full = vld.drop(columns='tid', errors='ignore') + vld_itr = vld_itr.drop(columns='tid', errors='ignore') default_dir = str(store_path.parent) - full_path = os.path.join(default_dir, f"paraflux_stats_full_{timestamp}.csv") - clean_path = os.path.join(default_dir, f"paraflux_stats_clean_{timestamp}.csv") - - vld_full.to_csv(full_path, index=False) + full_path = os.path.join(default_dir, f"{timestamp}_iteration_stats_full.csv") + paraflux_path = os.path.join(default_dir, f"{timestamp}_iteration_stats_Paraflux_only.csv") - keep_cols = ["itr", "Axis", "vld_loc_counts", - "failed_loc", "failed %", "CFR failure %", "No signal %"] - vld_clean = vld_full[keep_cols] - vld_clean.to_csv(clean_path, index=False) + vld_itr.to_csv(full_path, index=False) + keep_cols = ["itr", 'passed itr %', 'CFR failure %', 'No signal % per itr pairs'] + vld_paraflux = vld_itr[keep_cols] + vld_paraflux.to_csv(paraflux_path, index=False) print(f"✔ Saved full results to: {full_path}") - print(f"✔ Saved cleaned results to: {clean_path}") - - # ===================================================== - # --- Step 5: Return results --- - # ===================================================== - return vld - - # --- end of Alex B addition --- + print(f"✔ Saved cleaned results to: {paraflux_path}") + + return vld_itr + +############################################################## +############################################################## +############################################################## + + # # ===================================================== + # # --- Step 2: Convert to DataFrame --- + # # ===================================================== + # df_mfx = pd.DataFrame() + # for name in mfxdata.dtype.names: + # col = mfxdata[name] + # if col.ndim == 1: + # df_mfx[name] = col + # else: + # n_dim = col.shape[1] + # expanded = pd.DataFrame(col.tolist(), + # index=df_mfx.index if not df_mfx.empty else None) + # if name in ['loc', 'lnc']: + # labels = ["x", "y", "z"] + # expanded.columns = [f"{name}_{labels[i]}" for i in range(n_dim)] + # elif name == 'dcr': + # expanded.columns = [f"{name}_{i+1}" for i in range(n_dim)] + # else: + # expanded.columns = [f"{name}_{i}" for i in range(n_dim)] + # df_mfx = pd.concat([df_mfx, expanded], axis=1) + + # # ===================================================== + # # --- Step 3: Run analysis pipeline --- + # # ===================================================== + # failure_map = { + # 1: "Valid final", 2: "Valid not final", + # 4: "Derived iteration", 5: "Reserved", + # 6: "CFR failure", 8: "No signal", + # 9: "DAC out of range", 11: "Background measurement" + # } + # pair_mapping = {1: 0, 2: 1, 3: 1, 4: 2, 5: 2, 6: 3, 7: 3, 8: 4, 9: 4} + # pairs = [(0,1), (1,2), (2,3), (3,4), (4,5), + # (5,6), (6,7), (7,8), (8,9)] + + # def build_valid_tid_table(df): + # vld = df[df['vld']].groupby('itr')['tid'].apply(lambda x: list(set(x))).reset_index() + # vld['vld_loc_counts'] = vld['tid'].apply(len) + # vld['failed_loc'] = vld['vld_loc_counts'].shift(1) - vld['vld_loc_counts'] + # vld['Axis'] = np.where(vld['itr'] % 2 == 0, 'x,y', 'z') + # vld.insert(1, 'Axis', vld.pop('Axis')) + # vld['Cumulative sum of failed_loc'] = vld['failed_loc'].cumsum() + # return vld + + # def compute_passed_itr(vld): + # initial_count = vld['vld_loc_counts'].iloc[0] + # vld['passed_itr %'] = (vld['vld_loc_counts'] * 100 / initial_count).round(1) + # return vld + + # def compute_failed_sums(vld, pair_mapping): + # vld['pair_group'] = vld['itr'].map(pair_mapping) + # failed_sum_map = vld.groupby('pair_group')['failed_loc'].transform('sum') + # vld['failed_sum'] = np.where( + # vld['pair_group'].notna() & (vld['itr'] % 2 == 1), + # failed_sum_map, + # np.nan + # ) + # vld.drop(columns='pair_group', inplace=True) + # initial_count = vld['vld_loc_counts'].iloc[0] + # vld['failed %'] = (vld['failed_sum'] / initial_count * 100).round(1) + # return vld + + # def analyze_failures(vld, df, itr_from, itr_to, failure_map): + # tids_from = set(vld.loc[vld['itr'] == itr_from, 'tid'].values[0]) + # tids_to = set(vld.loc[vld['itr'] == itr_to, 'tid'].values[0]) + # failed_tids = tids_from - tids_to + # failed_df = df[df['tid'].isin(failed_tids) & (df['itr'] == itr_to)] + # counts = failed_df['sta'].value_counts().rename_axis("sta").reset_index(name="count") + # counts["reason"] = counts["sta"].map(failure_map).fillna("Other") + # counts.insert(0, "itr", itr_to) + # return counts + + # def compute_failure_pivot(vld, df, pairs, failure_map): + # failure_results = pd.concat( + # [analyze_failures(vld, df, i_from, i_to, failure_map) for i_from, i_to in pairs], + # ignore_index=True + # ) + # failure_pivot = failure_results.pivot_table( + # index="itr", columns="reason", values="count", fill_value=0 + # ).reset_index() + # return vld.merge(failure_pivot, on="itr", how="left") + + # def compute_cfr_and_no_signal(vld): + # initial_count = vld['vld_loc_counts'].iloc[0] + # vld['CFR failure %'] = np.where( + # vld['itr'].isin([4, 6]), + # (vld['CFR failure'] / initial_count * 100).round(1), + # np.nan + # ) + # no_signal_groups = {5: [4, 5], 7: [6, 7]} + # no_signal_pct = {} + # for target_itr, group in no_signal_groups.items(): + # total_no_signal = vld.loc[vld['itr'].isin(group), 'No signal'].sum() + # no_signal_pct[target_itr] = (total_no_signal / initial_count * 100).round(1) + # vld['No signal %'] = vld['itr'].map(no_signal_pct) + # return vld + + # # Run pipeline + # vld = build_valid_tid_table(df_mfx) + # vld = compute_passed_itr(vld) + # vld = compute_failed_sums(vld, pair_mapping) + # vld = compute_failure_pivot(vld, df_mfx, pairs, failure_map) + # vld = compute_cfr_and_no_signal(vld) + + # # ===================================================== + # # --- Step 4: Save results --- + # # ===================================================== + + # # Get the timestamp to append to the filename + # try: + # timestamp = pipeline.mdh.get('MINFLUX.TimeStamp') + # except Exception: + # print("No timestamp found in metadata, saving as default.") + # timestamp = "default" + + # vld_full = vld.drop(columns='tid', errors='ignore') + # default_dir = str(store_path.parent) + # full_path = os.path.join(default_dir, f"paraflux_stats_full_{timestamp}.csv") + # clean_path = os.path.join(default_dir, f"paraflux_stats_clean_{timestamp}.csv") + + # vld_full.to_csv(full_path, index=False) + + # keep_cols = ["itr", "Axis", "vld_loc_counts", + # "failed_loc", "failed %", "CFR failure %", "No signal %"] + # vld_clean = vld_full[keep_cols] + # vld_clean.to_csv(clean_path, index=False) + + # print(f"✔ Saved full results to: {full_path}") + # print(f"✔ Saved cleaned results to: {clean_path}") + + # # ===================================================== + # # --- Step 5: Return results --- + # # ===================================================== + # return vld + +############################################################## +############################################################## +############################################################## + + # # --- end of Alex B addition --- def OnClumpScatterPosPlot(self,event): From 26ffd6a64062d812d5985aa3e89b52696b319956 Mon Sep 17 00:00:00 2001 From: Alexfebo Date: Thu, 2 Oct 2025 11:53:25 +0200 Subject: [PATCH 22/27] Update OnRunParafluxAnalysis Add a graph output for itr 1, 3, 5, 7, 9 --- PYMEcs/experimental/MINFLUX.py | 208 ++++++++++----------------------- 1 file changed, 64 insertions(+), 144 deletions(-) diff --git a/PYMEcs/experimental/MINFLUX.py b/PYMEcs/experimental/MINFLUX.py index e1f52f1..7cdfd3e 100644 --- a/PYMEcs/experimental/MINFLUX.py +++ b/PYMEcs/experimental/MINFLUX.py @@ -760,6 +760,68 @@ def add_failure_metrics(self, vld_itr, initial_count): vld_itr['No signal % per itr pairs'] = vld_itr['itr'].map(no_signal_pct) return vld_itr + # Plot like in Paraflux + def paraflux_itr_plot(self, vld_paraflux): + # Mapping rules + label_map = { + "passed": "Passed", + "CFR": "CFR-filtered", + "No signal": "Dark", + "DAC": "Out of range", + "Other": "Other", + } + + def pretty_label(colname): + """Map colname to user-friendly label based on substring rules.""" + for key, label in label_map.items(): + if key.lower() in colname.lower(): + return label + return colname # fallback: keep original name + + # Keep only odd iterations (Paraflux style, i.e., 1, 3, 5, 7, 9) + vld_paraflux = vld_paraflux[vld_paraflux['itr'] % 2 == 1] + + # Base positions + r1 = np.arange(len(vld_paraflux)) # Define the position of bars on x-axis + names = vld_paraflux['itr'] # Names of group + barWidth = 0.85 # Bar width + + plt.figure(figsize=(8, 6)) # Set figure size + + # Colors (extendable if more cols are added) + colors = ["#0072B2", "#009E73", "#D55E00", "#E69F00", "#CC79A7"] + + bottom = np.zeros(len(vld_paraflux)) # track the stack height + for i, col in enumerate(vld_paraflux.columns[1:]): # Enumerate over all columns except 'itr' + vals = vld_paraflux[col].fillna(0) # Get the values for the current column, filling NaNs with 0 + label = pretty_label(col) # Get the pretty label for the legend + + plt.bar( + r1, vals, bottom=bottom, + color=colors[i % len(colors)], + edgecolor="white", width=barWidth, label=label + ) # Create the bar + + # Add labels inside each bar + for j, v in enumerate(vals): + if v > 0: + plt.text(r1[j], bottom[j] + v / 2, f"{v:.1f}%", # Only add text if value > 0 + ha="center", va="center", + color="black", + fontsize=9) + + bottom += vals.values + + # X/Y labels + plt.xticks(r1, names) + plt.xlabel("Iteration") + plt.ylabel("Events (%)") + plt.axhline(y=100, color="gray", linestyle="--", linewidth=1) + + plt.legend(loc="upper right", fontsize=9) + plt.tight_layout() + plt.show() + # Main function to compute stats of failed and valid localizations and save results def compute_vld_stats(self, df_mfx, failure_map, pipeline, store_path): # Run the analysis steps @@ -768,6 +830,7 @@ def compute_vld_stats(self, df_mfx, failure_map, pipeline, store_path): vld_itr = self.analyze_failures(vld_itr, df_mfx, failure_map) initial_count = vld_itr['vld loc count'].iloc[0] vld_itr = self.add_failure_metrics(vld_itr, initial_count) + vld_paraflux = self.paraflux_itr_plot(vld_itr[['itr', 'passed itr %', 'CFR failure %', 'No signal % per itr pairs']]) # Save results to CSV files try: @@ -791,150 +854,7 @@ def compute_vld_stats(self, df_mfx, failure_map, pipeline, store_path): return vld_itr -############################################################## -############################################################## -############################################################## - - # # ===================================================== - # # --- Step 2: Convert to DataFrame --- - # # ===================================================== - # df_mfx = pd.DataFrame() - # for name in mfxdata.dtype.names: - # col = mfxdata[name] - # if col.ndim == 1: - # df_mfx[name] = col - # else: - # n_dim = col.shape[1] - # expanded = pd.DataFrame(col.tolist(), - # index=df_mfx.index if not df_mfx.empty else None) - # if name in ['loc', 'lnc']: - # labels = ["x", "y", "z"] - # expanded.columns = [f"{name}_{labels[i]}" for i in range(n_dim)] - # elif name == 'dcr': - # expanded.columns = [f"{name}_{i+1}" for i in range(n_dim)] - # else: - # expanded.columns = [f"{name}_{i}" for i in range(n_dim)] - # df_mfx = pd.concat([df_mfx, expanded], axis=1) - - # # ===================================================== - # # --- Step 3: Run analysis pipeline --- - # # ===================================================== - # failure_map = { - # 1: "Valid final", 2: "Valid not final", - # 4: "Derived iteration", 5: "Reserved", - # 6: "CFR failure", 8: "No signal", - # 9: "DAC out of range", 11: "Background measurement" - # } - # pair_mapping = {1: 0, 2: 1, 3: 1, 4: 2, 5: 2, 6: 3, 7: 3, 8: 4, 9: 4} - # pairs = [(0,1), (1,2), (2,3), (3,4), (4,5), - # (5,6), (6,7), (7,8), (8,9)] - - # def build_valid_tid_table(df): - # vld = df[df['vld']].groupby('itr')['tid'].apply(lambda x: list(set(x))).reset_index() - # vld['vld_loc_counts'] = vld['tid'].apply(len) - # vld['failed_loc'] = vld['vld_loc_counts'].shift(1) - vld['vld_loc_counts'] - # vld['Axis'] = np.where(vld['itr'] % 2 == 0, 'x,y', 'z') - # vld.insert(1, 'Axis', vld.pop('Axis')) - # vld['Cumulative sum of failed_loc'] = vld['failed_loc'].cumsum() - # return vld - - # def compute_passed_itr(vld): - # initial_count = vld['vld_loc_counts'].iloc[0] - # vld['passed_itr %'] = (vld['vld_loc_counts'] * 100 / initial_count).round(1) - # return vld - - # def compute_failed_sums(vld, pair_mapping): - # vld['pair_group'] = vld['itr'].map(pair_mapping) - # failed_sum_map = vld.groupby('pair_group')['failed_loc'].transform('sum') - # vld['failed_sum'] = np.where( - # vld['pair_group'].notna() & (vld['itr'] % 2 == 1), - # failed_sum_map, - # np.nan - # ) - # vld.drop(columns='pair_group', inplace=True) - # initial_count = vld['vld_loc_counts'].iloc[0] - # vld['failed %'] = (vld['failed_sum'] / initial_count * 100).round(1) - # return vld - - # def analyze_failures(vld, df, itr_from, itr_to, failure_map): - # tids_from = set(vld.loc[vld['itr'] == itr_from, 'tid'].values[0]) - # tids_to = set(vld.loc[vld['itr'] == itr_to, 'tid'].values[0]) - # failed_tids = tids_from - tids_to - # failed_df = df[df['tid'].isin(failed_tids) & (df['itr'] == itr_to)] - # counts = failed_df['sta'].value_counts().rename_axis("sta").reset_index(name="count") - # counts["reason"] = counts["sta"].map(failure_map).fillna("Other") - # counts.insert(0, "itr", itr_to) - # return counts - - # def compute_failure_pivot(vld, df, pairs, failure_map): - # failure_results = pd.concat( - # [analyze_failures(vld, df, i_from, i_to, failure_map) for i_from, i_to in pairs], - # ignore_index=True - # ) - # failure_pivot = failure_results.pivot_table( - # index="itr", columns="reason", values="count", fill_value=0 - # ).reset_index() - # return vld.merge(failure_pivot, on="itr", how="left") - - # def compute_cfr_and_no_signal(vld): - # initial_count = vld['vld_loc_counts'].iloc[0] - # vld['CFR failure %'] = np.where( - # vld['itr'].isin([4, 6]), - # (vld['CFR failure'] / initial_count * 100).round(1), - # np.nan - # ) - # no_signal_groups = {5: [4, 5], 7: [6, 7]} - # no_signal_pct = {} - # for target_itr, group in no_signal_groups.items(): - # total_no_signal = vld.loc[vld['itr'].isin(group), 'No signal'].sum() - # no_signal_pct[target_itr] = (total_no_signal / initial_count * 100).round(1) - # vld['No signal %'] = vld['itr'].map(no_signal_pct) - # return vld - - # # Run pipeline - # vld = build_valid_tid_table(df_mfx) - # vld = compute_passed_itr(vld) - # vld = compute_failed_sums(vld, pair_mapping) - # vld = compute_failure_pivot(vld, df_mfx, pairs, failure_map) - # vld = compute_cfr_and_no_signal(vld) - - # # ===================================================== - # # --- Step 4: Save results --- - # # ===================================================== - - # # Get the timestamp to append to the filename - # try: - # timestamp = pipeline.mdh.get('MINFLUX.TimeStamp') - # except Exception: - # print("No timestamp found in metadata, saving as default.") - # timestamp = "default" - - # vld_full = vld.drop(columns='tid', errors='ignore') - # default_dir = str(store_path.parent) - # full_path = os.path.join(default_dir, f"paraflux_stats_full_{timestamp}.csv") - # clean_path = os.path.join(default_dir, f"paraflux_stats_clean_{timestamp}.csv") - - # vld_full.to_csv(full_path, index=False) - - # keep_cols = ["itr", "Axis", "vld_loc_counts", - # "failed_loc", "failed %", "CFR failure %", "No signal %"] - # vld_clean = vld_full[keep_cols] - # vld_clean.to_csv(clean_path, index=False) - - # print(f"✔ Saved full results to: {full_path}") - # print(f"✔ Saved cleaned results to: {clean_path}") - - # # ===================================================== - # # --- Step 5: Return results --- - # # ===================================================== - # return vld - -############################################################## -############################################################## -############################################################## - - # # --- end of Alex B addition --- - +### --- End of Alex B test addition function --- def OnClumpScatterPosPlot(self,event): from scipy.stats import binned_statistic From aa369a2c10a5994dfabde1626eb204261e1d7364 Mon Sep 17 00:00:00 2001 From: Alexfebo Date: Thu, 2 Oct 2025 12:11:36 +0200 Subject: [PATCH 23/27] Minor update of OnRunParafluxAnalysis --- PYMEcs/experimental/MINFLUX.py | 81 ++++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 34 deletions(-) diff --git a/PYMEcs/experimental/MINFLUX.py b/PYMEcs/experimental/MINFLUX.py index 7cdfd3e..8f9001a 100644 --- a/PYMEcs/experimental/MINFLUX.py +++ b/PYMEcs/experimental/MINFLUX.py @@ -628,13 +628,13 @@ def __init__(self, visFr): ID = visFr.AddMenuItem('MINFLUX>Recipes', r, self.OnLoadCustom).GetId() self.minfluxRIDs[ID] = minfluxRecipes[r] -# --- Alex B test addition function --- +# --- Alex B addition function to save and plot ITR stats (Paraflux like) --- def OnRunParafluxAnalysis(self, event): from pathlib import Path - # ===================================================== - # --- Step 1: Select and load Zarr.zip file --- - # ===================================================== + # ====================================================================================== + # --- Select, load Zarr.zip file, convert into DataFrame, Run the analysis functions --- + # ====================================================================================== pipeline = self.visFr.pipeline # Get the pipeline from the GUI if pipeline is None: @@ -692,21 +692,23 @@ def OnRunParafluxAnalysis(self, event): vld = self.compute_vld_stats(df_mfx, failure_map, pipeline, store_path) return vld - # ============================================================== + # ================================================== + # --- Analysis functions ( Paraflux-like) --- + # ================================================== - # Creates a df with valid and failed localizations per iteration + # Create a df with list of vld tids per iteration + additional basic stats def build_valid_df(self, df_mfx): - if not isinstance(df_mfx, pd.DataFrame): + if not isinstance(df_mfx, pd.DataFrame): # Convert to DataFrame if input is structured array df = pd.DataFrame(df_mfx) else: df = df_mfx.copy() - df_valid = df[df['vld']] + df_valid = df[df['vld']] # Select only valid localizations vld_itr = df_valid.groupby('itr')['tid'].apply(lambda x: list(set(x))).reset_index() # Get list of unique tids per iteration - vld_itr['Axis'] = np.where(vld_itr['itr'] % 2 == 0, 'x,y', 'z') # Define axis of each iteration - vld_itr['vld loc count'] = vld_itr['tid'].apply(len) + vld_itr['Axis'] = np.where(vld_itr['itr'] % 2 == 0, 'x,y', 'z') # Add a col with axis of each iteration + vld_itr['vld loc count'] = vld_itr['tid'].apply(len) # Count valid locs per iteration vld_itr['failed loc count'] = vld_itr['vld loc count'].shift(1, fill_value=vld_itr['vld loc count'].iloc[0]) - vld_itr['vld loc count'] # Calculate failed loc count per iteration vld_itr.loc[0, 'failed loc count'] = 0 # Set failed loc count of first iteration to 0 (instead of NaN) - vld_itr['failed loc cum sum'] = vld_itr['failed loc count'].cumsum() + vld_itr['failed loc cum sum'] = vld_itr['failed loc count'].cumsum() # Cumulative sum of failed locs return vld_itr # Compute percentages of passed and failed localizations (from build_valid_df) @@ -721,38 +723,44 @@ def compute_percentages(self, vld_itr): vld_itr['failed cum sum %'] = vld_itr['failed % per itr'].cumsum().round(1) return vld_itr - # Analyze failures between iterations and categorize them based on failure_map + # Analyze failures between consecutive iterations and categorize them based on failure_map (found on wiki from Abberior) def analyze_failures(self, vld_itr, df_mfx, failure_map): def analyze_failures_single_steps(vld_itr, df, itr_from, itr_to, failure_map): tids_from = set(vld_itr.loc[vld_itr['itr'] == itr_from, 'tid'].iloc[0]) # Select valid tids of the previous iteration tids_to = set(vld_itr.loc[vld_itr['itr'] == itr_to, 'tid'].iloc[0]) # Select valid tids of the current iteration failed_tids = tids_from - tids_to # Determine tids that failed in the current iteration - failed_df = df[df['tid'].isin(failed_tids) & (df['itr'] == itr_to)] - counts = failed_df['sta'].value_counts().rename_axis("sta").reset_index(name="count") - counts["reason"] = counts["sta"].map(failure_map).fillna("Other") - counts.insert(0, "itr", itr_to) + failed_df = df[df['tid'].isin(failed_tids) & (df['itr'] == itr_to)] # Create a df with only failed tids in the current iteration + counts = failed_df['sta'].value_counts().rename_axis("sta").reset_index(name="count") # Count failure reasons + counts["reason"] = counts["sta"].map(failure_map).fillna("Other") # Map failure reasons using failure_map + counts.insert(0, "itr", itr_to) # Add iteration column return counts - pairs = [(i, i+1) for i in range(vld_itr['itr'].max())] + pairs = [(i, i+1) for i in range(vld_itr['itr'].max())] # Create pairs of consecutive iterations + # Analyze failures for each pair and concatenate results failure_results = pd.concat( [analyze_failures_single_steps(vld_itr, df_mfx, i_from, i_to, failure_map) for i_from, i_to in pairs], ignore_index=True - ) - failure_pivot = failure_results.pivot_table( + ) + # Pivot the results to have failure reasons as columns + failure_pivot = failure_results.pivot_table( index="itr", columns="reason", values="count", fill_value=0 - ).reset_index() + ).reset_index() return vld_itr.merge(failure_pivot, on="itr", how="left") # Compute percentages for failure reasons def add_failure_metrics(self, vld_itr, initial_count): - cfr_map = {5: 4, 7: 6} - vld_itr['CFR failure %'] = np.nan + cfr_map = {5: 4, 7: 6} #map ITR where CFR failures occurs + vld_itr['CFR failure %'] = np.nan # Initialize column with NaNs + # Calculate CFR failure percentages based on cfr_map for target_itr, source_itr in cfr_map.items(): - if not vld_itr.loc[vld_itr['itr'] == source_itr, 'CFR failure'].empty: - val = vld_itr.loc[vld_itr['itr'] == source_itr, 'CFR failure'].values[0] - vld_itr.loc[vld_itr['itr'] == target_itr, 'CFR failure %'] = (val / initial_count * 100).round(1) + if not vld_itr.loc[vld_itr['itr'] == source_itr, 'CFR failure'].empty: # Check if CFR failure data exists for the source iteration + val = vld_itr.loc[vld_itr['itr'] == source_itr, 'CFR failure'].values[0] # Get the CFR failure count + vld_itr.loc[vld_itr['itr'] == target_itr, 'CFR failure %'] = (val / initial_count * 100).round(1) # Calculate percentage and assign to target iteration + # Calculate No signal percentage for each iteration vld_itr['No signal %'] = (vld_itr['No signal'] * 100 / initial_count).round(1) + # Define groups of iterations for No signal percentage calculation no_signal_groups = {1: [0, 1],3: [2, 3], 5: [4, 5], 7: [6, 7], 9: [8, 9]} + # Calculate No signal percentage for each group and map to iterations no_signal_pct = { target_itr: (vld_itr.loc[vld_itr['itr'].isin(group), 'No signal'].sum() / initial_count * 100).round(1) for target_itr, group in no_signal_groups.items() @@ -771,6 +779,7 @@ def paraflux_itr_plot(self, vld_paraflux): "Other": "Other", } + # Function to get pretty label based on the label map (substring matching from vld_paraflux col names) def pretty_label(colname): """Map colname to user-friendly label based on substring rules.""" for key, label in label_map.items(): @@ -781,36 +790,40 @@ def pretty_label(colname): # Keep only odd iterations (Paraflux style, i.e., 1, 3, 5, 7, 9) vld_paraflux = vld_paraflux[vld_paraflux['itr'] % 2 == 1] + # Set figure size + plt.figure(figsize=(8, 6)) + # Base positions - r1 = np.arange(len(vld_paraflux)) # Define the position of bars on x-axis + r1 = np.arange(len(vld_paraflux)) # Define the positions for each bar names = vld_paraflux['itr'] # Names of group barWidth = 0.85 # Bar width - plt.figure(figsize=(8, 6)) # Set figure size - # Colors (extendable if more cols are added) colors = ["#0072B2", "#009E73", "#D55E00", "#E69F00", "#CC79A7"] - bottom = np.zeros(len(vld_paraflux)) # track the stack height + # Define the bottom position for stacking + bottompos = np.zeros(len(vld_paraflux)) + + # Plot each column as a stacked bar for i, col in enumerate(vld_paraflux.columns[1:]): # Enumerate over all columns except 'itr' vals = vld_paraflux[col].fillna(0) # Get the values for the current column, filling NaNs with 0 - label = pretty_label(col) # Get the pretty label for the legend + labels = pretty_label(col) # Get the pretty label for the legend plt.bar( - r1, vals, bottom=bottom, + r1, vals, bottom=bottompos, color=colors[i % len(colors)], - edgecolor="white", width=barWidth, label=label + edgecolor="white", width=barWidth, label=labels ) # Create the bar # Add labels inside each bar for j, v in enumerate(vals): if v > 0: - plt.text(r1[j], bottom[j] + v / 2, f"{v:.1f}%", # Only add text if value > 0 + plt.text(r1[j], bottompos[j] + v / 2, f"{v:.1f}%", # Only add text if value > 0 ha="center", va="center", color="black", fontsize=9) - bottom += vals.values + bottompos += vals.values # X/Y labels plt.xticks(r1, names) From 2ad62457aee7e891653070fe209965fca1288bfc Mon Sep 17 00:00:00 2001 From: Alexfebo Date: Thu, 2 Oct 2025 13:04:39 +0200 Subject: [PATCH 24/27] Remove comments --- PYMEcs/Analysis/MINFLUX.py | 33 ------------------------- PYMEcs/experimental/MINFLUX.py | 35 -------------------------- PYMEcs/experimental/NPCcalcLM.py | 42 ++++++++------------------------ 3 files changed, 10 insertions(+), 100 deletions(-) diff --git a/PYMEcs/Analysis/MINFLUX.py b/PYMEcs/Analysis/MINFLUX.py index 1008a20..b08a087 100644 --- a/PYMEcs/Analysis/MINFLUX.py +++ b/PYMEcs/Analysis/MINFLUX.py @@ -83,9 +83,6 @@ def plot_density_stats_sns(ds,objectID='dbscanClumpID'): return dens -# copied from experimental>NPCcalcLM.py to try getting the filename -# this should be a backwards compatible way to access the main filename associated with the pipeline/datasource - def plot_stats_minflux(deltas, durations, tdiff, tdmedian, efo_or_dtovertime, times, showTimeAverages=False, dsKey=None, areaString=None, timestamp=None): @@ -156,36 +153,6 @@ def plot_stats_minflux(deltas, durations, tdiff, tdmedian, efo_or_dtovertime, ti csv_name = timestamp # --- Save as a csv file --- (Alex B addition) - # Below is old way of saving (before 2025-08-26) - # Replaced by saving each file individually (refering to plot error and plot stats from MFX) - # The merging of files can be done in the analysis workbook - # This avoid creating too many intermediate files and conditionals checks if other csv are being created later on for analysis - # Here we want to save the LocRate, however if LocError already exists we want to combine both files - # and ensure that the final csv keeps the organization regrdless on which csv was generated first (locError or LocRate). - - # variables to check if locerror and locrate csv files already exist - # locerror_file = csv_name + "temp_LocError.csv" - # locrate_file = csv_name + "temp_LocRate.csv" - # combined_file = csv_name + ".csv" - - # # Save the LocRate file - # df.to_csv(locrate_file, index=False, header=True) - - # # If the LocError file already exists, merge: - # if os.path.exists(locerror_file): - # df_rate = pd.read_csv(locrate_file) - # df_error = pd.read_csv(locerror_file) - - # # Combine the two dataframes in the desired order - # df_combined = pd.concat([df_error, df_rate], ignore_index=True) - # df_combined.to_csv(combined_file, index=False, header=True) - - # # Cleanup temp files - # os.remove(locerror_file) - # os.remove(locrate_file) - - # df.to_csv(locrate_file, index=False, header=True) - print(f'\ncsv name is: {csv_name}\nIf you did not load a session, csv file and figures will be saved on the desktop') # Used as a reminder for LocRate csv saving df.to_csv(csv_name + '_LocRate.csv', index=False, header=True) diff --git a/PYMEcs/experimental/MINFLUX.py b/PYMEcs/experimental/MINFLUX.py index 8f9001a..57a7a34 100644 --- a/PYMEcs/experimental/MINFLUX.py +++ b/PYMEcs/experimental/MINFLUX.py @@ -43,10 +43,8 @@ def plot_errors(pipeline): plt.text(x, y, '%.0f' % y, horizontalalignment='right') # draw above, centered uids, idx = np.unique(p['clumpIndex'],return_index=True) - #print(f"bp_dict1: {bp_dict1['medians'][0].get_xydata()[0][1]}") # Get the median values for photons and background - # mean_photon = np.mean(p['nPhotons']) median_photon = np.median(p['nPhotons']) # mean_bg = np.mean(p['fbg']) @@ -105,38 +103,6 @@ def plot_errors(pipeline): # --- Show the head of df in the console --- (Alex B addition) print(df.head()) -# --- Save as a csv file --- (Alex B addition) - - # Below is old way of saving (before 2025-08-26) - # Replaced by saving each file individually (refering to plot error and plot stats from MFX) - # The merging of files can be done in the analysis workbook - # This avoid creating too many intermediate files and conditionals checks if other csv are being created later on for analysis - - # # Here we want to save the LocError, however if LocRate already exists we want to combine both files - # # and ensure that the final csv keeps the organization regrdless on which csv was generated first (locError or LocRate). - - # # variables to check if locerror and locrate csv files already exist - # csv_name = pipeline.mdh.get('MINFLUX.TimeStamp') - # locerror_file = csv_name + "temp_LocError.csv" - # locrate_file = csv_name + "temp_LocRate.csv" - # combined_file = csv_name + ".csv" - - # # Save the locerror file - # df.to_csv(locerror_file, index=False, header=True) - - # # If the locrate file already exists, merge: - # if os.path.exists(locrate_file): - # df_rate = pd.read_csv(locrate_file) - # df_error = pd.read_csv(locerror_file) - - # # Combine the two dataframes in the desired order - # df_combined = pd.concat([df_error, df_rate], ignore_index=True) - # df_combined.to_csv(combined_file, index=False, header=True) - - # # Cleanup temp files - # os.remove(locerror_file) - # os.remove(locrate_file) - # Get the Timestamp (from pipeline) to create csv name csv_name = pipeline.mdh.get('MINFLUX.TimeStamp') @@ -1491,7 +1457,6 @@ def OnLocalisationRate(self, event): pipeline.selectDataSource(self.analysisSettings.defaultDatasourceForAnalysis) #Added by Alex B to get timestamp and send it to plot_stats_minflux timestamp = pipeline.mdh.get('MINFLUX.TimeStamp') - # print("Timestamp from OnLocalisationRate(experimental/MINFLUX.py): ", timestamp) if 'cfr' not in pipeline.keys(): Error(self.visFr,'no property called "cfr", likely no MINFLUX data - aborting') diff --git a/PYMEcs/experimental/NPCcalcLM.py b/PYMEcs/experimental/NPCcalcLM.py index 40fc0de..5c8c1e3 100644 --- a/PYMEcs/experimental/NPCcalcLM.py +++ b/PYMEcs/experimental/NPCcalcLM.py @@ -5,11 +5,12 @@ import numpy as np import wx from PYME.recipes import tablefilters +from traits.api import Bool, Enum, Float, HasTraits, Int + from PYMEcs.Analysis.NPC import estimate_nlabeled, npclabel_fit, plotcdf_npc3d from PYMEcs.IO.NPC import findNPCset from PYMEcs.misc.utils import unique_name from PYMEcs.pyme_warnings import warn -from traits.api import Bool, Enum, Float, HasTraits, Int logger = logging.getLogger(__name__) @@ -131,8 +132,8 @@ def __init__(self, visFr): self._npcsettings = None self.gallery_layer = None self.segment_layer = None - - # --- Alex B addition test for running several action at once --- + + # --- Alex B addition for running several NPC calc action at once --- def OnNPC3DRunAllActions(self, event=None): """Performs all key NPC 3D analysis actions in sequence, prompting user for output folder.""" # Prompt user for save directory @@ -286,7 +287,6 @@ def OnAnalyse3DNPCsByID(self, event=None): # --- Alex B addition --- # AIM: perform all actions 3D NPC actions and save output automatically -# Original function 'OnAnalyse3DNPCsByID', copied and modified for automatic saving. def OnAnalyse3DNPCsByID_auto_save(self, event=None, save_dir=None): from PYMEcs.Analysis.NPC import NPC3DSet @@ -439,14 +439,6 @@ def OnNPC3DSaveBySegments_auto_save(self, event=None, save_dir=None): if nbs is None: warn("could not find npcs with by-segment fitting info, have you carried out fitting with recent code?") return - # # Original code with dialog for manual saving - # with wx.FileDialog(self.visFr, 'Save NPC by-segment data as ...', - # wildcard='CSV (*.csv)|*.csv', - # style=wx.FD_SAVE) as fdialog: - # if fdialog.ShowModal() != wx.ID_OK: - # return - # else: - # fpath = fdialog.GetPath() import pandas as pd df = pd.DataFrame.from_dict(dict(top=nbs['top'].flatten(),bottom=nbs['bottom'].flatten())) @@ -597,8 +589,9 @@ def On3DNPCaddTemplates(self, event=None): # now we add a track layer to render our template polygons # TODO - we may need to check if this happened before or not! - from PYME.LMVis.layers.tracks import \ - TrackRenderLayer # NOTE: we may rename the clumpIndex variable in this layer to polyIndex or similar + from PYME.LMVis.layers.tracks import ( + TrackRenderLayer, # NOTE: we may rename the clumpIndex variable in this layer to polyIndex or similar + ) layer = TrackRenderLayer(pipeline, dsname=ds_template_name, method='tracks', clump_key='polyIndex', line_width=2.0, alpha=0.5) self.visFr.add_layer(layer) @@ -675,14 +668,6 @@ def OnNPC3DSaveMeasurements_auto_save(self, event=None, save_dir=None): if npcs is None or 'measurements' not in dir(npcs): warn('no valid NPC measurements found, therefore cannot save...') return - # Dialog for saving --> Commented-out - # fdialog = wx.FileDialog(self.visFr, 'Save NPC measurements as ...', - # wildcard='CSV (*.csv)|*.csv', - # style=wx.FD_SAVE) - # if fdialog.ShowModal() != wx.ID_OK: - # return - - # fpath = fdialog.GetPath() # --- Alex B addition --- # We define a few variables used for automatic saving later @@ -802,8 +787,6 @@ def OnNPC3DSaveNPCSet(self, event=None): # --- Alex B addition --- # AIM: perform all actions 3D NPC actions and save output automatically -# Original function 'OnNPC3DSaveNPCSet', copied and modified for automatic saving. -# Works fine as single action (new button: OnNPC3DSaveNPCSet_auto_save) line 130 def OnNPC3DSaveNPCSet_auto_save(self, event=None, save_dir=None): """Automatically save the NPC set to a default file path without user dialog.""" @@ -855,7 +838,6 @@ def OnNPC3DSaveGeometryStats(self,event=None): geo_df.to_csv(fpath,index=False) # --- Alex B addition --- -# Origimnal function 'OnNPC3DSaveGeometryStats', copied and modified for automatic saving. def OnNPC3DSaveGeometryStats_auto_save(self,event=None, save_dir=None): pipeline = self.visFr.pipeline @@ -867,12 +849,7 @@ def OnNPC3DSaveGeometryStats_auto_save(self,event=None, save_dir=None): heights = np.asarray(npcs.height()) import pandas as pd geo_df = pd.DataFrame.from_dict(dict(diameter=diams,height=heights)) - # with wx.FileDialog(self.visFr, 'Save NPC measurements as ...', - # wildcard='CSV (*.csv)|*.csv', - # style=wx.FD_SAVE) as fdialog: - # if fdialog.ShowModal() != wx.ID_OK: - # return - # fpath = fdialog.GetPath() + if save_dir is None: save_dir = os.getcwd() MINFLUXts = pipeline.mdh.get('MINFLUX.TimeStamp') # Get the timestamp and use it for naming the file to save @@ -892,6 +869,7 @@ def OnNPC3DGeometryStats(self,event=None): warn('no valid NPC measurements found, thus no geometry info available...') return import pandas as pd + from PYMEcs.misc.matplotlib import boxswarmplot, figuredefaults diams = np.asarray(npcs.diam()) heights = np.asarray(npcs.height()) @@ -911,7 +889,6 @@ def OnNPC3DGeometryStats(self,event=None): plt.ylim(0,150) # --- Alex B addition --- -# Origimnal function 'OnNPC3DGeometryStats', copied and modified for automatic saving. def OnNPC3DGeometryStats_auto_save(self,event=None, save_dir=None): pipeline = self.visFr.pipeline @@ -966,6 +943,7 @@ def OnNPC3DTemplateFitStats(self,event=None): warn('no valid NPC measurements found, thus no geometry info available...') return import pandas as pd + from PYMEcs.misc.matplotlib import boxswarmplot, figuredefaults id = [npc.objectID for npc in npcs.npcs] # not used right now llperloc = [npc.opt_result.fun/npc.npts.shape[0] for npc in npcs.npcs] From ecd9d981388f6499a4a701bcdfb850336a33aa19 Mon Sep 17 00:00:00 2001 From: Alexfebo Date: Wed, 8 Oct 2025 14:59:48 +0200 Subject: [PATCH 25/27] Improve handling of saving for LocError and LocRate --- PYMEcs/Analysis/MINFLUX.py | 120 +++++++++++++++++---------------- PYMEcs/experimental/MINFLUX.py | 50 ++++++-------- 2 files changed, 83 insertions(+), 87 deletions(-) diff --git a/PYMEcs/Analysis/MINFLUX.py b/PYMEcs/Analysis/MINFLUX.py index d1cbee0..bda22c7 100644 --- a/PYMEcs/Analysis/MINFLUX.py +++ b/PYMEcs/Analysis/MINFLUX.py @@ -1,49 +1,12 @@ +import os + import matplotlib.pyplot as plt import numpy as np import pandas as pd -from scipy.stats import binned_statistic - from PYMEcs.IO.MINFLUX import get_stddev_property from PYMEcs.pyme_warnings import warn +from scipy.stats import binned_statistic -import pandas as pd -from PYMEcs.misc.matplotlib import boxswarmplot - -def plot_stats(ds,ax,errdict,sdmax=None,swarmsize=3,siteKey='siteID'): - df = site_stats(ds,errdict,siteKey=siteKey) - kwargs = dict(swarmsize=swarmsize,width=0.2,annotate_means=True,annotate_medians=True,swarmalpha=0.4) - boxswarmplot(df,ax=ax,**kwargs) - ax.set_ylim(0,sdmax) - ax.set_ylabel('precision [nm]') - return df - -def site_stats(ds,sitedict,siteKey='siteID'): - uids, idx = np.unique(ds[siteKey],return_index=True) - sitestats = {} - for key in sitedict: - prop = sitedict[key] - sitestats[key] = ds[prop][idx] - df = pd.DataFrame.from_dict(sitestats) - return df[df > 0].dropna() # this drops rows with zeroes; these should not occur but apparently do; probably a bug somewhere - -def plotsitestats(p,origamiErrorLimit=10,figsize=None,swarmsize=3,siteKey='siteID',fignum=None): - uids = np.unique(p[siteKey]) - fig, axs = plt.subplots(2, 2, figsize=figsize,num=fignum) - plot_stats(p,axs[0, 0],dict(xd_sd_corr='error_x',x_sd='error_x_nc',x_sd_trace='error_x_ori'), - sdmax=origamiErrorLimit,swarmsize=swarmsize,siteKey=siteKey) - plot_stats(p,axs[0, 1],dict(yd_sd_corr='error_y',y_sd='error_y_nc',y_sd_trace='error_y_ori'), - sdmax=origamiErrorLimit,swarmsize=swarmsize,siteKey=siteKey) - if p.mdh.get('MINFLUX.Is3D'): - plot_stats(p,axs[1, 0],dict(zd_sd_corr='error_z',z_sd='error_z_nc',z_sd_trace='error_z_ori'), - sdmax=origamiErrorLimit,swarmsize=swarmsize,siteKey=siteKey) - - all_axes = dict(xd_sd_corr='error_x',yd_sd_corr='error_y') - if p.mdh.get('MINFLUX.Is3D'): - all_axes['zd_sd_corr'] = 'error_z' - df_allaxes = plot_stats(p,axs[1, 1],all_axes,sdmax=origamiErrorLimit,swarmsize=swarmsize,siteKey=siteKey) - fig.suptitle('Site stats for %d sites' % df_allaxes.shape[0]) - plt.tight_layout() - return df_allaxes def propcheck_density_stats(ds,warning=True): for prop in ['clst_area','clst_vol','clst_density','clst_stdz']: @@ -92,6 +55,9 @@ def plot_density_stats(ds,objectID='dbscanClumpID',scatter=False): ax1[1].scattered_boxplot(sz,labels=['Stddev Z'],showmeans=True) plt.tight_layout() +from PYMEcs.misc.matplotlib import boxswarmplot + + def plot_density_stats_sns(ds,objectID='dbscanClumpID'): if not propcheck_density_stats(ds): return @@ -115,11 +81,40 @@ def plot_density_stats_sns(ds,objectID='dbscanClumpID'): fig.suptitle('Density stats for %d clusters' % dens.size) plt.tight_layout() + # --- Save the values to csv --- (Alex B addition) + df = pd.DataFrame({ + "Metric": ["cluster density"], + "Mean": [np.mean(dens)], + "Median": [np.median(dens)], + "Unit": ["#/um^2"]}) + # --- Show the head of df in the console --- (Alex B addition) + print(df.head()) + + # --- Save as a csv file --- (Alex B addition) + # Save the cluster size file + #fn = os.path.splitext(ds.mdh.get('MINFLUX.Filename'))[0] + dirpath, filename = os.path.split(ds.filename) + fn = ds.mdh.get('MINFLUX.TimeStamp') + df.to_csv(os.path.join(dirpath, fn + "_clusterDensity.csv"), + index=False, header=True) + + # save data CSV file with dbscanClustered + df_dbscanClustered = ds.dataSources['dbscanClustered'].to_pandas() + df_dbscanClustered.to_csv( + os.path.join(dirpath, fn + "_dbscanClustered_data.csv"), index=False, header=True) + # Used as a reminder for LocRate csv saving + # print( + # f'\ncsv name is: {fn + "_clusterDensity.csv"}\nIf you did not load a session, csv file and figures will be saved on the desktop') + # By default the file is saved on the Desktop, if a session file is used, it is saved in the same directory as the session file. + return dens +# copied from experimental>NPCcalcLM.py to try getting the filename +# this should be a backwards compatible way to access the main filename associated with the pipeline/datasource + def plot_stats_minflux(deltas, durations, tdiff, tdmedian, efo_or_dtovertime, times, - showTimeAverages=False, dsKey=None, areaString=None, timestamp=None): + showTimeAverages=False, dsKey=None, areaString=None, timestamp=None, dirPath=None): # --- Create the figure (plot with 2x2 subplots) --- fig, (ax1, ax2) = plt.subplots(2, 2) @@ -179,20 +174,15 @@ def plot_stats_minflux(deltas, durations, tdiff, tdmedian, efo_or_dtovertime, ti # --- Show the head of df in the console --- (Alex B addition) print(df.head()) - - # --- Save as a csv file in the user specified path --- (Alex B addition) - # Get the timeastamp for the filename - if timestamp is None: - csv_name = input("Use the timestamp as a filename for saving (will be save in the current location):") - else: - csv_name = timestamp + # --- Save as a csv file --- (Alex B addition) - - print(f'\ncsv name is: {csv_name}\nIf you did not load a session, csv file and figures will be saved on the desktop') # Used as a reminder for LocRate csv saving - df.to_csv(csv_name + '_LocRate.csv', index=False, header=True) - - # --- Save the figure --- (Alex B addition) - plt.savefig(csv_name + '_loc_rate.png', dpi=300, bbox_inches='tight') + # Save the LocRate file + df.to_csv(os.path.join(dirPath, timestamp + "_locRate.csv"), index=False, header=True) + + # --- Save the figure --- (Alex B addition) + plt.savefig(os.path.join(dirPath, timestamp + '_locRate.png'), + dpi=300, bbox_inches='tight') + print(f'Figure and *.csv file saved to {os.path.join(dirPath, timestamp + "_locRate.png")}') # this function assumes a pandas dataframe # the pandas frame should generally be generated via the function minflux_npy2pyme from PYMEcs.IO.MINFLUX @@ -232,7 +222,10 @@ def analyse_locrate_pdframe(datain,use_invalid=False,showTimeAverages=True): tdiff = data['tim'].values[1:]-data['tim'].values[:-1] tdsmall = tdiff[tdiff <= 0.1] tdmedian = np.median(tdsmall) - + + dirpath, filename = os.path.split(datain.filename) + timeStamp = datain.mdh.get('MINFLUX.TimeStamp') + start_times = data.loc[startindex,'tim'][:-1].values # we use those for binning the deltas, we discard final time to match size of deltas if has_invalid and use_invalid: @@ -244,9 +237,12 @@ def analyse_locrate_pdframe(datain,use_invalid=False,showTimeAverages=True): if showTimeAverages: delta_averages, bin_edges, binnumber = binned_statistic(start_times,deltas,statistic='mean', bins=50) delta_av_times = 0.5*(bin_edges[:-1] + bin_edges[1:]) # bin centres - plot_stats_minflux(deltas, durations_proper, tdiff, tdmedian, delta_averages, delta_av_times, showTimeAverages=True) + plot_stats_minflux(deltas, durations_proper, tdiff, tdmedian, + delta_averages, delta_av_times, showTimeAverages=True, + timestamp=timeStamp, dirPath=dirpath) else: - plot_stats_minflux(deltas, durations_proper, tdiff, tdmedian, data['efo'], None) + plot_stats_minflux(deltas, durations_proper, tdiff, tdmedian, data['efo'], None, + timestamp=timeStamp, dirPath=dirpath) # similar version but now using a pipeline @@ -275,11 +271,17 @@ def analyse_locrate(data,datasource='Localizations',showTimeAverages=True, times area_string = 'area %.1fx%.1f um^2' % (lenx_um,leny_um) data.selectDataSource(curds) + dirpath, filename = os.path.split(data.filename) + timeStamp = data.mdh.get('MINFLUX.TimeStamp') + if showTimeAverages: delta_averages, bin_edges, binnumber = binned_statistic(starts[:-1],deltas,statistic='mean', bins=50) delta_av_times = 0.5*(bin_edges[:-1] + bin_edges[1:]) # bin centres - plot_stats_minflux(deltas, durations_proper, tdiff, tdmedian, delta_averages, delta_av_times, showTimeAverages=True, dsKey = datasource, areaString=area_string, timestamp=timestamp) + plot_stats_minflux(deltas, durations_proper, tdiff, tdmedian, + delta_averages, delta_av_times, showTimeAverages=True, + dsKey = datasource, areaString=area_string, + timestamp=timeStamp, dirPath=dirpath) else: - plot_stats_minflux(deltas, durations_proper, tdiff, tdmedian, data['efo'], None, dsKey = datasource, areaString=area_string, timestamp=timestamp) + plot_stats_minflux(deltas, durations_proper, tdiff, tdmedian, data['efo'], None, dsKey = datasource, areaString=area_string, timestamp=timeStamp, dirPath=dirpath) return (starts,deltas) diff --git a/PYMEcs/experimental/MINFLUX.py b/PYMEcs/experimental/MINFLUX.py index 015ff0b..636353c 100644 --- a/PYMEcs/experimental/MINFLUX.py +++ b/PYMEcs/experimental/MINFLUX.py @@ -43,8 +43,10 @@ def plot_errors(pipeline): plt.text(x, y, '%.0f' % y, horizontalalignment='right') # draw above, centered uids, idx = np.unique(p['clumpIndex'],return_index=True) + #print(f"bp_dict1: {bp_dict1['medians'][0].get_xydata()[0][1]}") # Get the median values for photons and background + # mean_photon = np.mean(p['nPhotons']) median_photon = np.median(p['nPhotons']) # mean_bg = np.mean(p['fbg']) @@ -72,10 +74,12 @@ def plot_errors(pipeline): # Display the plot plt.tight_layout() pipeline.selectDataSource(curds) - + dirpath, filename = os.path.split(pipeline.filename) + timestamp = pipeline.mdh.get('MINFLUX.TimeStamp') # Save the plot as a png file (Alex B addition) - plt.savefig(pipeline.mdh.get('MINFLUX.TimeStamp') + '_loc_error.png', dpi=300, bbox_inches='tight') - + plt.savefig(os.path.join(dirpath, timestamp + '_locError.png'), + dpi=300, bbox_inches='tight') + # --- Get the median of the clump size and errors x, y, z coalesced --- (Alex B addition) # mean_clump = np.mean(p['clumpSize']) @@ -103,17 +107,11 @@ def plot_errors(pipeline): # --- Show the head of df in the console --- (Alex B addition) print(df.head()) - # Get the Timestamp (from pipeline) to create csv name - csv_name = pipeline.mdh.get('MINFLUX.TimeStamp') - - # Save the df as csv - df.to_csv(csv_name + "_LocError.csv", index=False, header=True) - - print(f'\ncsv name is: {csv_name}\nIf you did not load a session, csv file and figures will be saved on the desktop') # Used as a reminder for LocRate csv saving - # By default the file is saved on the Desktop, if a session file is used, it is saved in the same directory as the session file. + # --- Save as a csv file --- (Alex B addition) + df.to_csv(os.path.join(dirpath, timestamp + "_locError.csv"), index=False, header=True) + print(f"LocError plot and csv file saved as: {timestamp + '_locError.csv'}") # By default the file is saved on the Desktop, if a session file is used, it is saved in the same directory as the session file. import pandas as pd - from PYMEcs.misc.matplotlib import boxswarmplot @@ -498,8 +496,7 @@ def plot_site_tracking(pipeline,fignum=None,plotSmoothingCurve=True): plt.tight_layout() import PYME.config -from PYME.recipes.traits import Bool, CStr, Enum, Float, HasTraits, Int, List - +from PYME.recipes.traits import Bool, CStr, Enum, Float, HasTraits, Int from PYMEcs.Analysis.MINFLUX import analyse_locrate from PYMEcs.IO.MINFLUX import findmbm from PYMEcs.misc.guiMsgBoxes import Error @@ -838,9 +835,8 @@ def compute_vld_stats(self, df_mfx, failure_map, pipeline, store_path): ### --- End of Alex B test addition function --- def OnClumpScatterPosPlot(self,event): - from scipy.stats import binned_statistic - from PYMEcs.IO.MINFLUX import get_stddev_property + from scipy.stats import binned_statistic def detect_coalesced(pipeline): # placeholder, to be implemented return False @@ -1071,25 +1067,22 @@ def OnAlphaShapes(self, event): return # now we add a layer to render our alpha shape polygons - from PYME.LMVis.layers.tracks import ( - TrackRenderLayer, # NOTE: we may rename the clumpIndex variable in this layer to polyIndex or similar - ) + from PYME.LMVis.layers.tracks import \ + TrackRenderLayer # NOTE: we may rename the clumpIndex variable in this layer to polyIndex or similar layer = TrackRenderLayer(self.visFr.pipeline, dsname='cluster_shapes', method='tracks', clump_key='polyIndex', line_width=2.0, alpha=0.5) self.visFr.add_layer(layer) def OnAddMINFLUXTracksCI(self, event): # now we add a track layer to render our traces - from PYME.LMVis.layers.tracks import ( - TrackRenderLayer, # NOTE: we may rename the clumpIndex variable in this layer to polyIndex or similar - ) + from PYME.LMVis.layers.tracks import \ + TrackRenderLayer # NOTE: we may rename the clumpIndex variable in this layer to polyIndex or similar layer = TrackRenderLayer(self.visFr.pipeline, dsname='output', method='tracks', clump_key='clumpIndex', line_width=2.0, alpha=0.5) self.visFr.add_layer(layer) def OnAddMINFLUXTracksTid(self, event): # now we add a track layer to render our traces - from PYME.LMVis.layers.tracks import ( - TrackRenderLayer, # NOTE: we may rename the clumpIndex variable in this layer to polyIndex or similar - ) + from PYME.LMVis.layers.tracks import \ + TrackRenderLayer # NOTE: we may rename the clumpIndex variable in this layer to polyIndex or similar layer = TrackRenderLayer(self.visFr.pipeline, dsname='output', method='tracks', clump_key='tid', line_width=2.0, alpha=0.5) self.visFr.add_layer(layer) @@ -1355,7 +1348,8 @@ def OnMINFLUXplotTempData(self, event): ("needs to be a **folder** location, currently set to %s" % (folder))) return - from PYMEcs.misc.utils import read_temp_csv, set_diff, timestamp_to_datetime + from PYMEcs.misc.utils import (read_temp_csv, set_diff, + timestamp_to_datetime) if len(self.visFr.pipeline.dataSources) == 0: warn("no datasources, this is probably an empty pipeline, have you loaded any data?") @@ -1527,8 +1521,8 @@ def OnOrigamiFinalFilter(self, event=None): def OnOrigamiSiteRecipe(self, event=None): from PYME.recipes.localisations import MergeClumps from PYME.recipes.tablefilters import FilterTable, Mapping - - from PYMEcs.recipes.localisations import DBSCANClustering2, OrigamiSiteTrack + from PYMEcs.recipes.localisations import (DBSCANClustering2, + OrigamiSiteTrack) pipeline = self.visFr.pipeline recipe = pipeline.recipe From cc627fc0333024ff5463d1cf93311dc514c2529a Mon Sep 17 00:00:00 2001 From: Alexfebo Date: Thu, 9 Oct 2025 09:04:46 +0200 Subject: [PATCH 26/27] Correct typo --- PYMEcs/Analysis/MINFLUX.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/PYMEcs/Analysis/MINFLUX.py b/PYMEcs/Analysis/MINFLUX.py index d273344..648f77f 100644 --- a/PYMEcs/Analysis/MINFLUX.py +++ b/PYMEcs/Analysis/MINFLUX.py @@ -3,9 +3,10 @@ import matplotlib.pyplot as plt import numpy as np import pandas as pd +from scipy.stats import binned_statistic + from PYMEcs.IO.MINFLUX import get_stddev_property from PYMEcs.pyme_warnings import warn -from scipy.stats import binned_statistic def propcheck_density_stats(ds,warning=True): @@ -112,12 +113,14 @@ def plot_density_stats_sns(ds,objectID='dbscanClumpID'): def plot_stats_minflux(deltas, durations, tdintrace, efo_or_dtovertime, times, showTimeAverages=False, dsKey=None, areaString=None, timestamp=None, dirPath=None): from scipy.stats import iqr - + # --- Create the figure (plot with 2x2 subplots) --- fig, (ax1, ax2) = plt.subplots(2, 2) + # Compute some statistics for TBT plot dtmedian = np.median(deltas) dtmean = np.mean(deltas) dtiqr = iqr(deltas,rng=(10, 90)) # we are going for the 10 to 90 % range + h = ax1[0].hist(deltas,bins=40,range=(0,dtmean + 2*dtiqr)) ax1[0].plot([dtmedian,dtmedian],[0,h[0].max()]) # this is time between one dye molecule and the next dye molecule being seen @@ -126,7 +129,7 @@ def plot_stats_minflux(deltas, durations, tdintrace, efo_or_dtovertime, times, verticalalignment='bottom', transform=ax1[0].transAxes) ax1[0].text(0.95, 0.7, ' mean %.2f s' % dtmean, horizontalalignment='right', verticalalignment='bottom', transform=ax1[0].transAxes) - if not areaString is None: + if areaString is not None: ax1[0].text(0.95, 0.6, areaString, horizontalalignment='right', verticalalignment='bottom', transform=ax1[0].transAxes) @@ -285,23 +288,13 @@ def analyse_locrate(data,datasource='Localizations',showTimeAverages=True, plot= dirpath, filename = os.path.split(data.filename) timeStamp = data.mdh.get('MINFLUX.TimeStamp') - if showTimeAverages: - delta_averages, bin_edges, binnumber = binned_statistic(starts[:-1],deltas,statistic='mean', bins=50) - delta_av_times = 0.5*(bin_edges[:-1] + bin_edges[1:]) # bin centres - plot_stats_minflux(deltas, durations_proper, tdiff, tdmedian, - delta_averages, delta_av_times, showTimeAverages=True, - dsKey = datasource, areaString=area_string, - timestamp=timeStamp, dirPath=dirpath) - else: - plot_stats_minflux(deltas, durations_proper, tdiff, tdmedian, data['efo'], None, dsKey = datasource, areaString=area_string, timestamp=timeStamp, dirPath=dirpath) - if plot: if showTimeAverages: delta_averages, bin_edges, binnumber = binned_statistic(starts[:-1],deltas,statistic='mean', bins=50) delta_av_times = 0.5*(bin_edges[:-1] + bin_edges[1:]) # bin centres plot_stats_minflux(deltas, durations_proper, tdintrace, delta_averages, delta_av_times, - showTimeAverages=True, dsKey = datasource, areaString=area_string) + showTimeAverages=True, dsKey = datasource, areaString=area_string, timestamp=timeStamp, dirPath=dirpath) else: - plot_stats_minflux(deltas, durations_proper, tdintrace, data['efo'], None, dsKey = datasource, areaString=area_string) + plot_stats_minflux(deltas, durations_proper, tdintrace, data['efo'], None, dsKey = datasource, areaString=area_string, timestamp=timeStamp, dirPath=dirpath) return (starts,ends,deltas,durations_proper,tdintrace) From 9496cf49f104951ea29e50ae92d312a0e9d26327 Mon Sep 17 00:00:00 2001 From: Alexfebo Date: Thu, 9 Oct 2025 09:17:02 +0200 Subject: [PATCH 27/27] Save the mean in csv for LocRate --- PYMEcs/Analysis/MINFLUX.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/PYMEcs/Analysis/MINFLUX.py b/PYMEcs/Analysis/MINFLUX.py index 648f77f..868c55c 100644 --- a/PYMEcs/Analysis/MINFLUX.py +++ b/PYMEcs/Analysis/MINFLUX.py @@ -116,7 +116,8 @@ def plot_stats_minflux(deltas, durations, tdintrace, efo_or_dtovertime, times, # --- Create the figure (plot with 2x2 subplots) --- fig, (ax1, ax2) = plt.subplots(2, 2) - # Compute some statistics for TBT plot + + # --- Compute some statistics for TBT plot (top-right = ax1[0]) --- dtmedian = np.median(deltas) dtmean = np.mean(deltas) dtiqr = iqr(deltas,rng=(10, 90)) # we are going for the 10 to 90 % range @@ -133,10 +134,12 @@ def plot_stats_minflux(deltas, durations, tdintrace, efo_or_dtovertime, times, ax1[0].text(0.95, 0.6, areaString, horizontalalignment='right', verticalalignment='bottom', transform=ax1[0].transAxes) + # --- Compute some statistics for trace duration plot (top-left = ax1[1]) --- durmedian = np.median(durations) durmean = np.mean(durations) duriqr = iqr(durations,rng=(10, 90)) h = ax1[1].hist(durations,bins=40,range=(0,durmean + 2*duriqr)) + ax1[1].plot([durmedian,durmedian],[0,h[0].max()]) ax1[1].set_xlabel('duration of "traces" [s]') ax1[1].text(0.95, 0.8, 'median %.0f ms' % (1e3*durmedian), horizontalalignment='right', @@ -145,11 +148,13 @@ def plot_stats_minflux(deltas, durations, tdintrace, efo_or_dtovertime, times, verticalalignment='bottom', transform=ax1[1].transAxes) # ax1[1].set_xlim(0,durmean + 2*duriqr) # superfluous since we are using the range keyword in hist + # --- Compute some statistics for time between localisations in same trace (bottom-left = ax2[0]) --- tdintrace_ms = 1e3*tdintrace tdmedian = np.median(tdintrace_ms) tdmean = np.mean(tdintrace_ms) tdiqr = iqr(tdintrace_ms,rng=(10, 90)) h = ax2[0].hist(tdintrace_ms,bins=50,range=(0,tdmean + 2*tdiqr)) + ax2[0].plot([tdmedian,tdmedian],[0,h[0].max()]) # these are times between repeated localisations of the same dye molecule ax2[0].set_xlabel('time between localisations in same trace [ms]') @@ -159,7 +164,7 @@ def plot_stats_minflux(deltas, durations, tdintrace, efo_or_dtovertime, times, verticalalignment='bottom', transform=ax2[0].transAxes) # ax2[0].set_xlim(0,tdmean + 2*tdiqr) # superfluous since we are using the range keyword in hist - # --- Compute the TBT running time average --- + # --- Compute the TBT running time average (bottom-right = ax2[1]) --- if showTimeAverages: ax2[1].plot(times,efo_or_dtovertime) ax2[1].set_xlabel('TBT running time average [s]') @@ -180,8 +185,10 @@ def plot_stats_minflux(deltas, durations, tdintrace, efo_or_dtovertime, times, # --- Save medians values to csv --- (Alex B addition) df = pd.DataFrame({ - "Metric": ["TBT (Time Between Traces)", "dimension", "area", "Trace Duration", "Time Between Localizations"], - "Median": [dtmedian, dimension, area, durmedian * 1e3, tdmedian * 1e3], # Convert seconds -> milliseconds where needed + "Metric": ["TBT (Time Between Traces)", "ROI dimension", "ROI area", "Trace Duration", "Time Between Localizations"], + "Median": [dtmedian.round(2), dimension, area, (durmedian * 1e3).round(0), tdmedian.round(0)], # Convert seconds -> milliseconds where needed + "Mean": [dtmean.round(2), dimension, area, (durmean * 1e3).round(0), tdmean.round(0)], # Convert seconds -> milliseconds where needed + "IQR": [dtiqr.round(2), dimension, area, (duriqr * 1e3).round(0), tdiqr.round(0)], # Convert seconds -> milliseconds where needed "Unit": ["s","um^2","um^2", "ms", "ms"] }) # --- Show the head of df in the console --- (Alex B addition)