Merge pull request #2 from Yambottle/master

sglx_process_probe .json & .env is ready | tested running on EC2 as well

Author: drewyangdev

.gitignore

@@ -82,3 +82,6 @@ target/
 
 # macOS
 .DS_Store
+
+#temp config file
+ecephys_spike_sorting/scripts/config

ecephys_spike_sorting/modules/catGT_helper/__main__.py

@@ -18,7 +18,7 @@ def run_CatGT(args):
     if sys.platform.startswith('win'):
         # build windows command line
         catGTexe_fullpath = catGTPath.replace('\\', '/') + "/runit.bat"
-    elif sys.platform.starstwith('linux'):
+    elif sys.platform.startswith('linux'):
         catGTexe_fullpath = catGTPath.replace('\\', '/') + "/runit.sh"
     else:
         print('unknown system, cannot run CatGt')
@@ -59,7 +59,7 @@ def run_CatGT(args):
     print('CatGT command line:' + catGT_cmd)
 
     start = time.time()
-    subprocess.call(catGT_cmd)
+    subprocess.call(cmd_parts)
 
     execution_time = time.time() - start
 

ecephys_spike_sorting/modules/mean_waveforms/__main__.py

@@ -85,8 +85,21 @@ def calculate_mean_waveforms(args):
 
         print(cwaves_cmd)
 
+        cmd_parts = list()
+        cmd_parts.extend([exe_path, ' -spikeglx_bin=',  spikeglx_bin])
+        cmd_parts.extend([' -clus_table_npy=', clus_table_npy])
+        cmd_parts.extend([' -clus_time_npy=', clus_time_npy])
+        cmd_parts.extend([' -clus_lbl_npy=', clus_lbl_npy])
+        cmd_parts.extend([' -dest=', dest])
+        cmd_parts.extend([' -samples_per_spike=', repr(args['mean_waveform_params']['samples_per_spike'])])
+        cmd_parts.extend([' -pre_samples=', repr(args['mean_waveform_params']['pre_samples'])])
+        cmd_parts.extend([' -num_spikes=', repr(args['mean_waveform_params']['spikes_per_epoch'])])
+        cmd_parts.extend([' -snr_radius=', repr(args['mean_waveform_params']['snr_radius'])])
+        print(cmd_parts)
+
         # make the C_Waves call
-        subprocess.call(cwaves_cmd)
+        #subprocess.call(cwaves_cmd)
+        subprocess.call(cmd_parts)
 
         # for first version, retain original names
         if clu_version == 0:

ecephys_spike_sorting/scripts/create_input_json.py

@@ -17,7 +17,16 @@ def create_samba_directory(samba_server, samba_share):
 
     return data_dir
 
-def createInputJson(output_file, 
+def createInputJson(
+                    output_file,
+                    ecephys_directory=None,
+                    kilosort_repository=None,
+                    KS2ver=None,
+                    npy_matlab_repository=None,
+                    catGTPath=None,
+                    tPrime_path=None,
+                    cWaves_path=None,
+                    kilosort_output_tmp=None, 
                     npx_directory=None, 
                     continuous_file = None,
                     spikeGLX_data=True,
@@ -57,32 +66,32 @@ def createInputJson(output_file,
                     ks_LTseed = 1,
                     ks_templateRadius_um = 163,
                     c_Waves_snr_um = 160,
-                    qm_isi_thresh = 1.5/1000                    
+                    qm_isi_thresh = 1.5/1000
                     ):
 
     # hard coded paths to code on your computer and system
-    ecephys_directory = r'D:\ecephys_fork\ecephys_spike_sorting\ecephys_spike_sorting'
-    
+    # ecephys_directory = r'D:\ecephys_fork\ecephys_spike_sorting\ecephys_spike_sorting'
+    # this is passed through arguements now
+
     # location of kilosor respository and kilosort version
 
-    kilosort_repository = r'C:\Users\labadmin\Documents\jic\KS3_fork\Kilosort2'
-    KS2ver = '2.0'      # must equal '3.0', '2.5' or '2.0', and match the kiilosort_repository
-    
+    # kilosort_repository = r'C:\Users\labadmin\Documents\jic\KS3_fork\Kilosort2'
+    # KS2ver = '2.0'      # must equal '3.0', '2.5' or '2.0', and match the kiilosort_repository
+    # these are passed through arguements now
+
     # KS 3.0 does not yet output pcs.
-    if KS2ver == '3.0':
-        include_pcs = False  # set to false for KS2ver = '3.0'
-    else:
-        include_pcs = True
-    
-    npy_matlab_repository = r'C:\Users\labadmin\Documents\jic\npy-matlab-master'
-    catGTPath = r'C:\Users\labadmin\Documents\jic\CatGT-win'
-    tPrime_path=r'C:\Users\labadmin\Documents\jic\TPrime-win'
-    cWaves_path=r'C:\Users\labadmin\Documents\jic\C_Waves-win'
+    include_pcs = KS2ver != '3.0'
 
+    #npy_matlab_repository = r'C:\Users\labadmin\Documents\jic\npy-matlab-master'
+    #catGTPath = r'C:\Users\labadmin\Documents\jic\CatGT-win'
+    #tPrime_path=r'C:\Users\labadmin\Documents\jic\TPrime-win'
+    #cWaves_path=r'C:\Users\labadmin\Documents\jic\C_Waves-win'
+    # these are passed through arguements now
+
 
     # for config files and kilosort working space
-    kilosort_output_tmp = r'D:\kilosort_datatemp' 
-    
+    # kilosort_output_tmp = r'D:\kilosort_datatemp' 
+    # this is passed through arguements now
 
     # derived directory names
 
@@ -347,4 +356,4 @@ def createInputJson(output_file,
     with io.open(output_file, 'w', encoding='utf-8') as f:
         f.write(json.dumps(dictionary, ensure_ascii=False, sort_keys=True, indent=4))
 
-    return dictionary
+    return dictionary

ecephys_spike_sorting/scripts/sglx_process_probe.py

@@ -2,7 +2,9 @@
 import sys
 import subprocess
 import json
+from dotenv import load_dotenv
 
+sys.path.append(os.path.dirname(__file__))
 from helpers import SpikeGLX_utils
 from helpers import log_from_json
 from helpers import run_one_probe
@@ -22,6 +24,14 @@ def run_probe(prb, json_directory, npx_directory,
               catGT_loccar_min_um, catGT_loccar_max_um,
               catGT_cmd_string,
               ks_Th, refPerMS,
+              ecephys_directory=None,
+              kilosort_repository=None,
+              KS2ver=None,
+              npy_matlab_repository=None,
+              catGTPath=None,
+              tPrime_path=None,
+              cWaves_path=None,
+              kilosort_output_tmp=None,
               ks_remDup=0,
               ks_saveRez=1,
               ks_copy_fproc=0,
@@ -31,6 +41,19 @@ def run_probe(prb, json_directory, npx_directory,
               c_Waves_snr_um=160,
               ni_present=True,
               ni_extract_string=None):
+    # load external tool path from .env if not given from .json
+    dot_env_path = "config/sglx_process_probe.json"
+    if os.path.exists(dot_env_path):
+        load_dotenv(dot_env_path)
+        ecephys_directory=ecephys_directory or os.getenv('ecephys_directory')
+        kilosort_repository=kilosort_repository or os.getenv('kilosort_repository')
+        KS2ver=KS2ver or os.getenv('KS2ver')
+        npy_matlab_repository=npy_matlab_repository or os.getenv('npy_matlab_repository')
+        catGTPath=catGTPath or os.getenv('catGTPath')
+        tPrime_path=tPrime_path or os.getenv('tPrime_path')
+        cWaves_path=cWaves_path or os.getenv('cWaves_path')
+        kilosort_output_tmp=kilosort_output_tmp or os.getenv('kilosort_output_tmp') 
+
     # build path to the first probe folder; look into that folder
     # to determine the range of trials if the user specified t limits as
     # start and end
@@ -69,6 +92,14 @@ def run_probe(prb, json_directory, npx_directory,
     input_meta_fullpath = os.path.join(input_data_directory, metaName)
     print(input_meta_fullpath)
     createInputJson(catGT_input_json,
+                    ecephys_directory=ecephys_directory,
+                    kilosort_repository=kilosort_repository,
+                    KS2ver=KS2ver,
+                    npy_matlab_repository=npy_matlab_repository,
+                    catGTPath=catGTPath,
+                    tPrime_path=tPrime_path,
+                    cWaves_path=cWaves_path,
+                    kilosort_output_tmp=kilosort_output_tmp,
                     npx_directory=npx_directory,
                     continuous_file=continuous_file,
                     kilosort_output_directory=catGT_dest,
@@ -105,6 +136,14 @@ def run_probe(prb, json_directory, npx_directory,
     print(continuous_file)
     print('ks_Th: ' + repr(ks_Th) + ' ,refPerMS: ' + repr(refPerMS))
     createInputJson(module_input_json,
+                    ecephys_directory=ecephys_directory,
+                    kilosort_repository=kilosort_repository,
+                    KS2ver=KS2ver,
+                    npy_matlab_repository=npy_matlab_repository,
+                    catGTPath=catGTPath,
+                    tPrime_path=tPrime_path,
+                    cWaves_path=cWaves_path,
+                    kilosort_output_tmp=kilosort_output_tmp,
                     npx_directory=npx_directory,
                     continuous_file=continuous_file,
                     spikeGLX_data=True,
@@ -149,10 +188,11 @@ def run_probe(prb, json_directory, npx_directory,
                          logFullPath)
 
 
-if __name__ == '__main__':
+#if __name__ == '__main__':
+def main():
     json_fp = sys.argv[1]
 
     with open(json_fp) as f:
         kwargs = json.load(f)
 
-    run_one_probe(**kwargs)
+    run_probe(**kwargs)