Add logs and metadata directories for centralized file organization (Fixes #308)

blech_exp_info.py

@@ -868,8 +868,11 @@ def process_electrode_layout(dir_path, dir_name, electrode_files, ports, electro
     Returns:
         Tuple containing layout_dict, fin_emg_port, orig_emg_electrodes, emg_muscle_str
     """
+    # Use metadata directory for layout file
+    metadata_dir = os.path.join(dir_path, 'metadata')
+    os.makedirs(metadata_dir, exist_ok=True)
     layout_file_path = os.path.join(
-        dir_path, dir_name + "_electrode_layout.csv")
+        metadata_dir, dir_name + "_electrode_layout.csv")
 
     def yn_check(x):
         return x in ['y', 'yes', 'n', 'no', '']
@@ -930,6 +933,12 @@ def confirm_check(x):
                 exit()
 
     # Read and process the layout file
+    # Check both metadata directory and old location for backward compatibility
+    if not os.path.exists(layout_file_path):
+        old_layout_path = os.path.join(
+            dir_path, dir_name + "_electrode_layout.csv")
+        if os.path.exists(old_layout_path):
+            layout_file_path = old_layout_path
     layout_frame_filled = pd.read_csv(layout_file_path)
 
     if not args.programmatic:

blech_init.py

@@ -170,7 +170,7 @@ def generate_processing_scripts(dir_name, blech_clust_dir, electrode_layout_fram
         f.write(f'DIR={dir_name} \n')
         print(f"parallel -k -j {job_count} --noswap --load 100% --progress " +
               "--memfree 4G --ungroup --retry-failed " +
-              f"--joblog $DIR/results.log " +
+              f"--joblog $DIR/logs/results.log " +
               "bash $DIR/temp/blech_process_single.sh " +
               f"::: {' '.join([str(x) for x in bash_electrode_list])}",
               file=f)
@@ -204,7 +204,11 @@ def generate_processing_scripts(dir_name, blech_clust_dir, electrode_layout_fram
     this_pipeline_check.write_to_log(script_path, 'attempted')
     # If info_dict present but execution log is not
     # just create the execution log with blech_exp_info marked
-    if 'info_dict' in dir(metadata_handler) and not os.path.exists(metadata_handler.dir_name + '/execution_log.json'):
+    execution_log_path_new = os.path.join(
+        metadata_handler.dir_name, 'logs', 'execution_log.json')
+    execution_log_path_old = os.path.join(
+        metadata_handler.dir_name, 'execution_log.json')
+    if 'info_dict' in dir(metadata_handler) and not os.path.exists(execution_log_path_new) and not os.path.exists(execution_log_path_old):
         blech_exp_info_path = os.path.join(
             blech_clust_dir, 'blech_exp_info.py')
         this_pipeline_check.write_to_log(blech_exp_info_path, 'attempted')

blech_make_arrays.py

@@ -21,7 +21,7 @@
 from tqdm import tqdm
 from blech_clust.utils.clustering import get_filtered_electrode
 from blech_clust.utils.blech_process_utils import return_cutoff_values
-from blech_clust.utils.blech_utils import imp_metadata, pipeline_graph_check
+from blech_clust.utils.blech_utils import imp_metadata, pipeline_graph_check, get_metadata_dir
 from blech_clust.utils.read_file import DigInHandler
 from ast import literal_eval
 import matplotlib.pyplot as plt
@@ -376,7 +376,8 @@ def create_emg_trials_for_digin(
 
     trial_info_frame.to_hdf(metadata_handler.hdf5_name,
                             'trial_info_frame', mode='a')
-    csv_path = os.path.join(metadata_handler.dir_name, 'trial_info_frame.csv')
+    metadata_dir = get_metadata_dir(metadata_handler.dir_name)
+    csv_path = os.path.join(metadata_dir, 'trial_info_frame.csv')
     trial_info_frame.to_csv(csv_path, index=False)
 
     # Get list of units under the sorted_units group.

blech_post_process.py

@@ -87,7 +87,7 @@
 os.environ['OPENBLAS_NUM_THREADS'] = '1'  # noqa
 
 import blech_clust.utils.blech_post_process_utils as post_utils  # noqa
-from blech_clust.utils.blech_utils import entry_checker, imp_metadata, pipeline_graph_check  # noqa
+from blech_clust.utils.blech_utils import entry_checker, imp_metadata, pipeline_graph_check, get_metadata_dir  # noqa
 from utils import blech_waveforms_datashader  # noqa
 from multiprocessing import Pool, cpu_count  # noqa
 from functools import partial  # noqa
@@ -588,8 +588,9 @@
 print('==== Unit Table ====\n')
 print(current_unit_table)
 # Also write to disk
+metadata_dir = get_metadata_dir(metadata_handler.dir_name)
 current_unit_table.to_csv(
-    os.path.join(metadata_handler.dir_name, 'unit_descriptor.csv'),
+    os.path.join(metadata_dir, 'unit_descriptor.csv'),
     index=False,
 )
 

blech_process.py

@@ -98,7 +98,9 @@
 print(f'Processing electrode {electrode_num}')
 
 # Initialize or load processing log
-log_path = pathlib.Path(metadata_handler.dir_name) / 'blech_process.log'
+log_dir = os.path.join(metadata_handler.dir_name, 'logs')
+os.makedirs(log_dir, exist_ok=True)
+log_path = pathlib.Path(log_dir) / 'blech_process.log'
 if log_path.exists():
     with open(log_path) as f:
         process_log = json.load(f)

blech_units_characteristics.py

@@ -26,7 +26,7 @@
 import json
 import glob
 import itertools
-from blech_clust.utils.blech_utils import entry_checker, imp_metadata, pipeline_graph_check
+from blech_clust.utils.blech_utils import entry_checker, imp_metadata, pipeline_graph_check, get_metadata_dir
 from blech_clust.utils.ephys_data import ephys_data
 from blech_clust.utils.ephys_data import visualize as vz
 import pandas as pd
@@ -765,7 +765,8 @@ def palatability_corr(x):
     [this_resp_neurons, taste_bin_pval_frame, this_pal_sig_frame])
 out_frame.drop(columns=['p-unc'], inplace=True)
 
-out_frame.to_csv(os.path.join(dir_name, 'aggregated_characteristics.csv'),
+metadata_dir = get_metadata_dir(dir_name)
+out_frame.to_csv(os.path.join(metadata_dir, 'aggregated_characteristics.csv'),
                  index=False)
 out_frame.to_hdf(
     metadata_handler.hdf5_name,

emg/emg_freq_setup.py

@@ -240,6 +240,12 @@
 
 # Also delete log
 print('Deleting results.log')
+logs_dir = os.path.join(data_dir, 'logs')
+os.makedirs(logs_dir, exist_ok=True)
+results_log_path = os.path.join(logs_dir, 'results.log')
+if os.path.exists(results_log_path):
+    os.remove(results_log_path)
+# Also check old location for backward compatibility
 if os.path.exists('results.log'):
     os.remove('results.log')
 
@@ -255,7 +261,7 @@
     data_dir,
     len(emg_env_df)-1)
 print(
-    "parallel -k -j {:d} --noswap --load 100% --progress --ungroup --joblog {:s}/results.log bash blech_emg_jetstream_parallel1.sh ::: {{0..{:d}}}".format(
+    "parallel -k -j {:d} --noswap --load 100% --progress --ungroup --joblog {:s}/logs/results.log bash blech_emg_jetstream_parallel1.sh ::: {{0..{:d}}}".format(
         *format_args),
     file=f)
 f.close()
@@ -281,8 +287,18 @@
 ############################################################
 # Merge the emg_env_df with trial_info_df
 ############################################################
-# Also get trial_info_frame
-trial_info_frame = pd.read_csv(os.path.join(data_dir, 'trial_info_frame.csv'))
+# Also get trial_info_frame (check both metadata dir and old location)
+metadata_dir = os.path.join(data_dir, 'metadata')
+trial_info_path_new = os.path.join(metadata_dir, 'trial_info_frame.csv')
+trial_info_path_old = os.path.join(data_dir, 'trial_info_frame.csv')
+
+if os.path.exists(trial_info_path_new):
+    trial_info_frame = pd.read_csv(trial_info_path_new)
+elif os.path.exists(trial_info_path_old):
+    trial_info_frame = pd.read_csv(trial_info_path_old)
+else:
+    raise FileNotFoundError(
+        "trial_info_frame.csv not found in metadata/ or data directory")
 
 merge_frame = pd.merge(emg_env_df, trial_info_frame,
                        left_on=['dig_in', 'trial_inds'],

tests/test_logs_metadata_dirs.py

@@ -0,0 +1,155 @@
+"""
+Tests for verifying that logs and metadata are placed in the correct directories.
+These tests verify the source code changes directly without importing.
+"""
+
+
+def test_ensure_dir_function():
+    """Test the ensure_dir function logic directly"""
+    # Read the function source
+    with open('/workspace/project/blech_clust/utils/blech_utils.py', 'r') as f:
+        source = f.read()
+
+    # Check that ensure_dir is defined
+    assert 'def ensure_dir(data_dir, subdir):' in source
+
+    # Check that get_metadata_dir is defined
+    assert 'def get_metadata_dir(data_dir):' in source
+
+    # Check that get_logs_dir is defined
+    assert 'def get_logs_dir(data_dir):' in source
+
+    print("✓ All required functions are defined in blech_utils.py")
+
+
+def test_logs_directory_creation_in_tee():
+    """Test that Tee class uses logs directory"""
+    with open('/workspace/project/blech_clust/utils/blech_utils.py', 'r') as f:
+        source = f.read()
+
+    # Check that Tee uses logs directory
+    assert "logs_dir = ensure_dir(data_dir, 'logs')" in source
+    assert "self.log_path = os.path.join(logs_dir, name)" in source
+    print("✓ Tee class correctly uses logs directory")
+
+
+def test_logs_directory_in_blech_process():
+    """Test that blech_process.py uses logs directory"""
+    with open('/workspace/project/blech_clust/blech_process.py', 'r') as f:
+        source = f.read()
+
+    # Check that it creates logs directory
+    assert "log_dir = os.path.join(metadata_handler.dir_name, 'logs')" in source
+    assert "os.makedirs(log_dir, exist_ok=True)" in source
+    print("✓ blech_process.py correctly uses logs directory")
+
+
+def test_logs_directory_in_blech_init():
+    """Test that blech_init.py uses logs directory"""
+    with open('/workspace/project/blech_clust/blech_init.py', 'r') as f:
+        source = f.read()
+
+    # Check that it uses logs directory for results.log
+    assert "logs/results.log" in source
+    print("✓ blech_init.py correctly uses logs directory for results.log")
+
+
+def test_logs_directory_in_ram_monitor():
+    """Test that ram_monitor.py uses logs directory"""
+    with open('/workspace/project/blech_clust/utils/ram_monitor.py', 'r') as f:
+        source = f.read()
+
+    # Check that it uses logs directory
+    assert "logs_dir = os.path.join(output_dir, 'logs')" in source
+    assert "ram_usage.log" in source
+    print("✓ ram_monitor.py correctly uses logs directory")
+
+
+def test_metadata_directory_in_read_file():
+    """Test that read_file.py uses metadata directory"""
+    with open('/workspace/project/blech_clust/utils/read_file.py', 'r') as f:
+        source = f.read()
+
+    # Check that it uses get_metadata_dir
+    assert "from blech_clust.utils.blech_utils import get_metadata_dir" in source
+    assert "metadata_dir = get_metadata_dir(self.data_dir)" in source
+    print("✓ read_file.py correctly uses metadata directory")
+
+
+def test_metadata_directory_in_ephys_data():
+    """Test that ephys_data.py uses metadata directory"""
+    with open('/workspace/project/blech_clust/utils/ephys_data/ephys_data.py', 'r') as f:
+        source = f.read()
+
+    # Check that it uses get_metadata_dir
+    assert "from blech_clust.utils.blech_utils import get_metadata_dir" in source
+    print("✓ ephys_data.py correctly imports get_metadata_dir")
+
+
+def test_metadata_directory_in_blech_units_characteristics():
+    """Test that blech_units_characteristics.py uses metadata directory"""
+    with open('/workspace/project/blech_clust/blech_units_characteristics.py', 'r') as f:
+        source = f.read()
+
+    # Check that it uses get_metadata_dir for aggregated_characteristics.csv
+    assert "get_metadata_dir" in source
+    assert "aggregated_characteristics.csv" in source
+    print("✓ blech_units_characteristics.py correctly uses metadata directory")
+
+
+def test_metadata_directory_in_blech_post_process():
+    """Test that blech_post_process.py uses metadata directory"""
+    with open('/workspace/project/blech_clust/blech_post_process.py', 'r') as f:
+        source = f.read()
+
+    # Check that it uses get_metadata_dir for unit_descriptor.csv
+    assert "get_metadata_dir" in source
+    assert "unit_descriptor.csv" in source
+    print("✓ blech_post_process.py correctly uses metadata directory")
+
+
+def test_metadata_directory_in_blech_make_arrays():
+    """Test that blech_make_arrays.py uses metadata directory"""
+    with open('/workspace/project/blech_clust/blech_make_arrays.py', 'r') as f:
+        source = f.read()
+
+    # Check that it uses get_metadata_dir for trial_info_frame.csv
+    assert "get_metadata_dir" in source
+    assert "trial_info_frame.csv" in source
+    print("✓ blech_make_arrays.py correctly uses metadata directory")
+
+
+def test_backward_compatibility_check():
+    """Test that backward compatibility is maintained"""
+    # Check execution_log.json backward compatibility
+    with open('/workspace/project/blech_clust/utils/blech_utils.py', 'r') as f:
+        source = f.read()
+
+    assert "old_log_path = os.path.join(self.data_dir, 'execution_log.json')" in source
+    assert "if not os.path.exists(self.log_path) and os.path.exists(old_log_path):" in source
+    print("✓ Backward compatibility for execution_log.json is maintained")
+
+
+def test_find_file_in_dir_function():
+    """Test that find_file_in_dir function exists for backward compatibility"""
+    with open('/workspace/project/blech_clust/utils/blech_utils.py', 'r') as f:
+        source = f.read()
+
+    assert "def find_file_in_dir(data_dir, filename_pattern):" in source
+    print("✓ find_file_in_dir function exists for backward compatibility")
+
+
+if __name__ == '__main__':
+    test_ensure_dir_function()
+    test_logs_directory_creation_in_tee()
+    test_logs_directory_in_blech_process()
+    test_logs_directory_in_blech_init()
+    test_logs_directory_in_ram_monitor()
+    test_metadata_directory_in_read_file()
+    test_metadata_directory_in_ephys_data()
+    test_metadata_directory_in_blech_units_characteristics()
+    test_metadata_directory_in_blech_post_process()
+    test_metadata_directory_in_blech_make_arrays()
+    test_backward_compatibility_check()
+    test_find_file_in_dir_function()
+    print("\nAll tests passed!")

utils/blech_data_summary.py

@@ -59,7 +59,7 @@
 blech_clust_dir = os.path.dirname(os.path.dirname(script_path))
 sys.path.append(blech_clust_dir)
 from blech_clust.utils.qa_utils.channel_corr import get_all_channels, intra_corr  # noqa
-from blech_clust.utils.blech_utils import imp_metadata, pipeline_graph_check  # noqa
+from blech_clust.utils.blech_utils import imp_metadata, pipeline_graph_check, get_metadata_dir  # noqa
 
 
 def extract_unit_characteristics(dir_name):
@@ -152,8 +152,13 @@ def extract_unit_characteristics(dir_name):
         print("Falling back to aggregated_characteristics.csv...")
 
     # Fallback: read from aggregated_characteristics.csv
+    metadata_dir = get_metadata_dir(dir_name)
     characteristics_path = os.path.join(
-        dir_name, 'aggregated_characteristics.csv')
+        metadata_dir, 'aggregated_characteristics.csv')
+    # Check old location for backward compatibility
+    old_path = os.path.join(dir_name, 'aggregated_characteristics.csv')
+    if not os.path.exists(characteristics_path) and os.path.exists(old_path):
+        characteristics_path = old_path
 
     if not os.path.exists(characteristics_path):
         print(
@@ -385,7 +390,12 @@ def extract_laser_conditions(dir_name):
     Returns:
         Dictionary with laser condition information
     """
-    trial_info_path = os.path.join(dir_name, 'trial_info_frame.csv')
+    metadata_dir = get_metadata_dir(dir_name)
+    trial_info_path = os.path.join(metadata_dir, 'trial_info_frame.csv')
+    # Check old location for backward compatibility
+    old_path = os.path.join(dir_name, 'trial_info_frame.csv')
+    if not os.path.exists(trial_info_path) and os.path.exists(old_path):
+        trial_info_path = old_path
 
     if not os.path.exists(trial_info_path):
         print(f"Warning: {trial_info_path} not found.")

utils/blech_process_utils.py

@@ -52,7 +52,7 @@
 
 # Set environment variables to limit the number of threads used by various libraries
 # Do it at the start of the script to ensure it applies to all imported libraries
-from blech_clust.utils.blech_utils import ifisdir_rmdir
+from blech_clust.utils.blech_utils import ifisdir_rmdir, get_metadata_dir
 import os  # noqa
 os.environ['OMP_NUM_THREADS'] = '1'  # noqa
 os.environ['MKL_NUM_THREADS'] = '1'  # noqa
@@ -761,7 +761,9 @@ def write_out_recommendations(self):
         Write out electrode number, count, mean prediction probability, and 5,95th percentiles
         """
         waveform_thresh = self.classifier_params['min_suggestion_count']
-        out_file_path = self.data_dir + '/waveform_classifier_recommendations.csv'
+        metadata_dir = get_metadata_dir(self.data_dir)
+        out_file_path = os.path.join(
+            metadata_dir, 'waveform_classifier_recommendations.csv')
         count = len(self.pos_spike_dict['waveforms'])
         if count > waveform_thresh:
             percentile_5 = np.percentile(self.pos_spike_dict['prob'], 5)