Merge pull request #4 from ttngu207/master

shift the configuration of packages paths to `createInputJson`

Author: drewyangdev

ecephys_spike_sorting/scripts/create_input_json.py

@@ -1,4 +1,5 @@
 import os, io, json, sys
+from dotenv import load_dotenv
 
 if sys.platform == 'linux':
     import pwd
@@ -26,8 +27,8 @@ def createInputJson(
                     catGTPath=None,
                     tPrime_path=None,
                     cWaves_path=None,
-                    kilosort_output_tmp=None, 
-                    npx_directory=None, 
+                    kilosort_output_tmp=None,
+                    npx_directory=None,
                     continuous_file = None,
                     spikeGLX_data=True,
                     input_meta_path=None,
@@ -54,12 +55,12 @@ def createInputJson(
                     tPrime_3A = False,
                     toStream_path_3A = None,
                     fromStream_list_3A = None,
-                    ks_remDup = 0,                   
+                    ks_remDup = 0,
                     ks_finalSplits = 1,
                     ks_labelGood = 1,
                     ks_saveRez = 1,
                     ks_copy_fproc = 0,
-                    ks_minfr_goodchannels = 0.1,                  
+                    ks_minfr_goodchannels = 0.1,
                     ks_whiteningRadius_um = 163,
                     ks_Th = '[10,4]',
                     ks_CSBseed = 1,
@@ -81,22 +82,35 @@ def createInputJson(
 
     # KS 3.0 does not yet output pcs.
     include_pcs = KS2ver != '3.0'
-    
+
     #npy_matlab_repository = r'C:\Users\labadmin\Documents\jic\npy-matlab-master'
     #catGTPath = r'C:\Users\labadmin\Documents\jic\CatGT-win'
     #tPrime_path=r'C:\Users\labadmin\Documents\jic\TPrime-win'
     #cWaves_path=r'C:\Users\labadmin\Documents\jic\C_Waves-win'
     # these are passed through arguements now
 
-     
+
     # for config files and kilosort working space
-    # kilosort_output_tmp = r'D:\kilosort_datatemp' 
+    # kilosort_output_tmp = r'D:\kilosort_datatemp'
     # this is passed through arguements now
-    
+
+    dot_env_path = "config/sglx_process_probe.json"
+    if os.path.exists(dot_env_path):
+        load_dotenv(dot_env_path)
+
+    ecephys_directory = ecephys_directory or os.getenv('ecephys_directory')
+    kilosort_repository = kilosort_repository or os.getenv('kilosort_repository')
+    KS2ver = KS2ver or os.getenv('KS2ver')
+    npy_matlab_repository = npy_matlab_repository or os.getenv('npy_matlab_repository')
+    catGTPath = catGTPath or os.getenv('catGTPath')
+    tPrime_path = tPrime_path or os.getenv('tPrime_path')
+    cWaves_path = cWaves_path or os.getenv('cWaves_path')
+    kilosort_output_tmp = kilosort_output_tmp or os.getenv('kilosort_output_tmp')
+
     # derived directory names
-    
+
     modules_directory = os.path.join(ecephys_directory,'modules')
-    
+
     if kilosort_output_directory is None \
          and extracted_data_directory is None \
          and npx_directory is None:
@@ -105,49 +119,49 @@ def createInputJson(
 
     #default ephys params. For spikeGLX, these get replaced by values read from metadata
     sample_rate = 30000
-    num_channels = 385    
+    num_channels = 385
     reference_channels = [191]
     uVPerBit = 2.34375
     acq_system = 'PXI'
-     
-    
+
+
     if spikeGLX_data:
         # location of the raw data is the continuous file passed from script
         # metadata file should be located in same directory
-        # 
+        #
         # kilosort output will be put in the same directory as the input raw data,
         # set in kilosort_output_directory passed from script
         # kilososrt postprocessing (duplicate removal) and identification of noise
         # clusters will act on phy output in the kilosort output directory
         #
-        # 
+        #
         if input_meta_path is not None:
-            probe_type, sample_rate, num_channels, uVPerBit = SpikeGLX_utils.EphysParams(input_meta_path)  
+            probe_type, sample_rate, num_channels, uVPerBit = SpikeGLX_utils.EphysParams(input_meta_path)
             print('SpikeGLX params read from meta')
             print('probe type: {:s}, sample_rate: {:.5f}, num_channels: {:d}, uVPerBit: {:.4f}'.format\
                   (probe_type, sample_rate, num_channels, uVPerBit))
         #print('kilosort output directory: ' + kilosort_output_directory )
 
-        
+
     else:
        print('currently only supporting spikeGLX data')
-        
 
-            
+
+
 
     # geometry params by probe type. expand the dictoionaries to add types
     # vertical probe pitch vs probe type
-    vpitch = {'3A': 20, 'NP1': 20, 'NP21': 15, 'NP24': 15, 'NP1100': 6}  
-    hpitch = {'3A': 32, 'NP1': 32, 'NP21': 32, 'NP24': 32, 'NP1100': 6} 
-    nColumn = {'3A': 2, 'NP1': 2, 'NP21': 2, 'NP24': 2, 'NP1100': 8} 
-    
-    
+    vpitch = {'3A': 20, 'NP1': 20, 'NP21': 15, 'NP24': 15, 'NP1100': 6}
+    hpitch = {'3A': 32, 'NP1': 32, 'NP21': 32, 'NP24': 32, 'NP1100': 6}
+    nColumn = {'3A': 2, 'NP1': 2, 'NP21': 2, 'NP24': 2, 'NP1100': 8}
+
+
     # CatGT needs the inner and outer redii for local common average referencing
     # specified in sites
     catGT_loccar_min_sites = int(round(catGT_loccar_min_um/vpitch.get(probe_type)))
     catGT_loccar_max_sites = int(round(catGT_loccar_max_um/vpitch.get(probe_type)))
     # print('loccar min: ' + repr(catGT_loccar_min_sites))
-    
+
     # whiteningRange is the number of sites used for whitening in KIlosort
     # preprocessing. Calculate the number of sites within the user-specified
     # whitening radius for this probe geometery
@@ -156,24 +170,24 @@ def createInputJson(
     ks_whiteningRange = int(round(2*nrows*nColumn.get(probe_type)))
     if ks_whiteningRange > 384:
         ks_whiteningRange = 384
-    
+
     # nNeighbors is the number of sites kilosort includes in a template.
     # Calculate the number of sites within that radisu.
     nrows = np.sqrt((np.square(ks_templateRadius_um) - np.square(hpitch.get(probe_type))))/vpitch.get(probe_type)
     ks_nNeighbors = int(round(2*nrows*nColumn.get(probe_type)))
     if ks_nNeighbors > 64:
         ks_nNeighbors = 64          #max allowed in CUDA
     # print('ks_nNeighbors: ' + repr(ks_nNeighbors))
-    
+
     c_waves_radius_sites = int(round(c_Waves_snr_um/vpitch.get(probe_type)))
 
     # Create string designating temporary output file for KS2 (gets inserted into KS2 config.m file)
     fproc = os.path.join(kilosort_output_tmp,'temp_wh.dat') # full path for temp whitened data file
     fproc_forward_slash = fproc.replace('\\','/')
     fproc_str = "'" + fproc_forward_slash + "'"
-    
 
-    
+
+
     dictionary = \
     {
 
@@ -193,7 +207,7 @@ def createInputJson(
         "waveform_metrics" : {
             "waveform_metrics_file" : os.path.join(kilosort_output_directory, 'waveform_metrics.csv')
         },
-        
+
         "cluster_metrics" : {
             "cluster_metrics_file" : os.path.join(kilosort_output_directory, 'metrics.csv')
         },
@@ -210,7 +224,7 @@ def createInputJson(
             "lfp_band_file" : os.path.join(extracted_data_directory, 'continuous', 'Neuropix-' + acq_system + '-100.1', 'continuous.dat'),
             "reorder_lfp_channels" : True,
             "cluster_group_file_name" : 'cluster_group.tsv'
-        }, 
+        },
 
         "extract_from_npx_params" : {
             "npx_directory": npx_directory,
@@ -235,7 +249,7 @@ def createInputJson(
             "time_interval" : 5,
             "skip_s_per_pass" : 10,
             "start_time" : 10
-        }, 
+        },
 
         "median_subtraction_params" : {
             "median_subtraction_executable": "C:\\Users\\svc_neuropix\\Documents\\GitHub\\spikebandmediansubtraction\\Builds\\VisualStudio2013\\Release\\SpikeBandMedianSubtraction.exe",
@@ -289,19 +303,19 @@ def createInputJson(
         },
 
         "mean_waveform_params" : {
-        
+
             "mean_waveforms_file" : os.path.join(kilosort_output_directory, 'mean_waveforms.npy'),
             "samples_per_spike" : 82,
             "pre_samples" : 20,
             "num_epochs" : 1,           #epochs not implemented for c_waves
             "spikes_per_epoch" : 1000,
             "spread_threshold" : 0.12,
-            "site_range" : 16,    
+            "site_range" : 16,
             "cWaves_path" : cWaves_path,
             "use_C_Waves" : True,
-            "snr_radius" : c_waves_radius_sites       
+            "snr_radius" : c_waves_radius_sites
         },
-            
+
 
         "noise_waveform_params" : {
             "classifier_path" : os.path.join(modules_directory, 'noise_templates', 'rf_classifier.pkl'),
@@ -321,7 +335,7 @@ def createInputJson(
             "drift_metrics_min_spikes_per_interval" : 10,
             "include_pcs" : include_pcs
         },
-        
+
         "catGT_helper_params" : {
             "run_name" : catGT_run_name,
             "gate_string" : gate_string,
@@ -345,12 +359,12 @@ def createInputJson(
                 "tPrime_3A" : tPrime_3A,
                 "toStream_path_3A" : toStream_path_3A,
                 "fromStream_list_3A" : fromStream_list_3A
-        },  
-                
+        },
+
         "psth_events": {
                 "event_ex_param_str": event_ex_param_str
          }
-        
+
     }
 
     with io.open(output_file, 'w', encoding='utf-8') as f:

ecephys_spike_sorting/scripts/sglx_process_probe.py

@@ -2,7 +2,6 @@
 import sys
 import subprocess
 import json
-from dotenv import load_dotenv
 
 sys.path.append(os.path.dirname(__file__))
 from helpers import SpikeGLX_utils
@@ -24,14 +23,7 @@ def run_probe(prb, json_directory, npx_directory,
               catGT_loccar_min_um, catGT_loccar_max_um,
               catGT_cmd_string,
               ks_Th, refPerMS,
-              ecephys_directory=None,
-              kilosort_repository=None,
               KS2ver=None,
-              npy_matlab_repository=None,
-              catGTPath=None,
-              tPrime_path=None,
-              cWaves_path=None,
-              kilosort_output_tmp=None,
               ks_remDup=0,
               ks_saveRez=1,
               ks_copy_fproc=0,
@@ -41,18 +33,6 @@ def run_probe(prb, json_directory, npx_directory,
               c_Waves_snr_um=160,
               ni_present=True,
               ni_extract_string=None):
-    # load external tool path from .env if not given from .json
-    dot_env_path = "config/sglx_process_probe.json"
-    if os.path.exists(dot_env_path):
-        load_dotenv(dot_env_path)
-        ecephys_directory=ecephys_directory or os.getenv('ecephys_directory')
-        kilosort_repository=kilosort_repository or os.getenv('kilosort_repository')
-        KS2ver=KS2ver or os.getenv('KS2ver')
-        npy_matlab_repository=npy_matlab_repository or os.getenv('npy_matlab_repository')
-        catGTPath=catGTPath or os.getenv('catGTPath')
-        tPrime_path=tPrime_path or os.getenv('tPrime_path')
-        cWaves_path=cWaves_path or os.getenv('cWaves_path')
-        kilosort_output_tmp=kilosort_output_tmp or os.getenv('kilosort_output_tmp') 
 
     # build path to the first probe folder; look into that folder
     # to determine the range of trials if the user specified t limits as
@@ -92,14 +72,7 @@ def run_probe(prb, json_directory, npx_directory,
     input_meta_fullpath = os.path.join(input_data_directory, metaName)
     print(input_meta_fullpath)
     createInputJson(catGT_input_json,
-                    ecephys_directory=ecephys_directory,
-                    kilosort_repository=kilosort_repository,
                     KS2ver=KS2ver,
-                    npy_matlab_repository=npy_matlab_repository,
-                    catGTPath=catGTPath,
-                    tPrime_path=tPrime_path,
-                    cWaves_path=cWaves_path,
-                    kilosort_output_tmp=kilosort_output_tmp,
                     npx_directory=npx_directory,
                     continuous_file=continuous_file,
                     kilosort_output_directory=catGT_dest,
@@ -136,14 +109,7 @@ def run_probe(prb, json_directory, npx_directory,
     print(continuous_file)
     print('ks_Th: ' + repr(ks_Th) + ' ,refPerMS: ' + repr(refPerMS))
     createInputJson(module_input_json,
-                    ecephys_directory=ecephys_directory,
-                    kilosort_repository=kilosort_repository,
                     KS2ver=KS2ver,
-                    npy_matlab_repository=npy_matlab_repository,
-                    catGTPath=catGTPath,
-                    tPrime_path=tPrime_path,
-                    cWaves_path=cWaves_path,
-                    kilosort_output_tmp=kilosort_output_tmp,
                     npx_directory=npx_directory,
                     continuous_file=continuous_file,
                     spikeGLX_data=True,
@@ -188,11 +154,14 @@ def run_probe(prb, json_directory, npx_directory,
                          logFullPath)
 
 
-#if __name__ == '__main__':
 def main():
     json_fp = sys.argv[1]
 
     with open(json_fp) as f:
         kwargs = json.load(f)
 
     run_probe(**kwargs)
+
+
+if __name__ == '__main__':
+    main()