Linux Serial Command Line Runs Fixed

Author: ryanurbs

streamline/ApplyModelMain.py

@@ -19,6 +19,7 @@
 import sys
 import pandas as pd
 import FeatureSelectionJob
+import ApplyModelJob
 import time
 import csv
 import glob
@@ -62,7 +63,6 @@ def main(argv):
     categorical_cutoff = metadata['Categorical Cutoff']
     sig_cutoff = metadata['Statistical Significance Cutoff']
     cv_partitions = metadata['CV Partitions']
-    random_state = metadata['Random Seed']
     scale_data = metadata['Use Data Scaling']
     impute_data = metadata['Use Data Imputation']
     multi_impute = metadata['Use Multivariate Imputation']
@@ -110,11 +110,11 @@ def main(argv):
             if file_extension == 'txt' or file_extension == 'csv':
                 if apply_name not in unique_datanames:
                     unique_datanames.append(apply_name)
+                    job_counter += 1
                     if eval(options.run_parallel):
-                        job_counter += 1
-                        submitClusterJob(options.reserved_memory,options.maximum_memory,options.queue,experiment_path,datasetFilename,full_path,class_label,instance_label,categorical_cutoff,sig_cutoff,cv_partitions,scale_data,impute_data,primary_metric,options.dataset_for_rep,options.match_label,options.plot_ROC,options.plot_PRC,options.plot_metric_boxplots,options.export_feature_correlations,jupyterRun,multi_impute,random_state)
+                        submitClusterJob(options.reserved_memory,options.maximum_memory,options.queue,experiment_path,datasetFilename,full_path,class_label,instance_label,categorical_cutoff,sig_cutoff,cv_partitions,scale_data,impute_data,primary_metric,options.dataset_for_rep,options.match_label,options.plot_ROC,options.plot_PRC,options.plot_metric_boxplots,options.export_feature_correlations,jupyterRun,multi_impute)
                     else:
-                        submitLocalJob(datasetFilename,full_path,class_label,instance_label,categorical_cutoff,sig_cutoff,cv_partitions,scale_data,impute_data,primary_metric,options.dataset_for_rep,options.match_label,options.plot_ROC,options.plot_PRC,options.plot_metric_boxplots,options.export_feature_correlations,jupyterRun,multi_impute,random_state)
+                        submitLocalJob(datasetFilename,full_path,class_label,instance_label,categorical_cutoff,sig_cutoff,cv_partitions,scale_data,impute_data,primary_metric,options.dataset_for_rep,options.match_label,options.plot_ROC,options.plot_PRC,options.plot_metric_boxplots,options.export_feature_correlations,jupyterRun,multi_impute)
                     file_count += 1
 
         if file_count == 0: #Check that there was at least 1 dataset
@@ -138,11 +138,11 @@ def main(argv):
     if not options.do_check:
         print(str(job_counter)+ " jobs submitted in Phase 9")
 
-def submitLocalJob(datasetFilename,full_path,class_label,instance_label,categorical_cutoff,sig_cutoff,cv_partitions,scale_data,impute_data,primary_metric,dataset_for_rep,match_label,plot_ROC,plot_PRC,plot_metric_boxplots,export_feature_correlations,jupyterRun,multi_impute,random_state):
+def submitLocalJob(datasetFilename,full_path,class_label,instance_label,categorical_cutoff,sig_cutoff,cv_partitions,scale_data,impute_data,primary_metric,dataset_for_rep,match_label,plot_ROC,plot_PRC,plot_metric_boxplots,export_feature_correlations,jupyterRun,multi_impute):
     """ Runs ApplyModelJob.py on each dataset in dataset_path locally. These runs will be completed serially rather than in parallel. """
-    ApplyModelJob.job(datasetFilename,full_path,class_label,instance_label,categorical_cutoff,sig_cutoff,cv_partitions,scale_data,impute_data,primary_metric,dataset_for_rep,match_label,plot_ROC,plot_PRC,plot_metric_boxplots,export_feature_correlations,jupyterRun,multi_impute,random_state)
+    ApplyModelJob.job(datasetFilename,full_path,class_label,instance_label,categorical_cutoff,sig_cutoff,cv_partitions,scale_data,impute_data,primary_metric,dataset_for_rep,match_label,plot_ROC,plot_PRC,plot_metric_boxplots,export_feature_correlations,jupyterRun,multi_impute)
 
-def submitClusterJob(reserved_memory,maximum_memory,queue,experiment_path,datasetFilename,full_path,class_label,instance_label,categorical_cutoff,sig_cutoff,cv_partitions,scale_data,impute_data,primary_metric,dataset_for_rep,match_label,plot_ROC,plot_PRC,plot_metric_boxplots,export_feature_correlations,jupyterRun,multi_impute,random_state):
+def submitClusterJob(reserved_memory,maximum_memory,queue,experiment_path,datasetFilename,full_path,class_label,instance_label,categorical_cutoff,sig_cutoff,cv_partitions,scale_data,impute_data,primary_metric,dataset_for_rep,match_label,plot_ROC,plot_PRC,plot_metric_boxplots,export_feature_correlations,jupyterRun,multi_impute):
     """ Runs ApplyModelJob.py on each dataset in rep_data_path. Runs in parallel on a linux-based computing cluster that uses an IBM Spectrum LSF for job scheduling."""
     train_name = full_path.split('/')[-1] #original training data name
     apply_name = datasetFilename.split('/')[-1].split('.')[0]

streamline/DataCompareMain.py

@@ -49,9 +49,8 @@ def main(argv):
         #Load variables specified earlier in the pipeline from metadata
         sig_cutoff = metadata['Statistical Significance Cutoff']
         jupyterRun = metadata['Run From Jupyter Notebook']
-
+        job_counter += 1
         if eval(options.run_parallel):
-            job_counter += 1
             submitClusterJob(options.output_path+'/'+options.experiment_name,options.reserved_memory,options.maximum_memory,options.queue,sig_cutoff,jupyterRun)
         else:
             submitLocalJob(options.output_path+'/'+options.experiment_name,sig_cutoff,jupyterRun)

streamline/FeatureImportanceMain.py

@@ -87,8 +87,8 @@ def main(argv):
                     os.mkdir(full_path+"/feature_selection/mutualinformation")
                 for cv_train_path in glob.glob(full_path+"/CVDatasets/*_CV_*Train.csv"):
                     command_text = '/FeatureImportanceJob.py ' + cv_train_path+" "+experiment_path+" "+str(random_state)+" "+class_label+" "+instance_label+" " +str(options.instance_subset)+" mi "+str(options.n_jobs)+' '+str(options.use_TURF)+' '+str(options.TURF_pct)
+                    job_counter += 1
                     if eval(options.run_parallel):
-                        job_counter += 1
                         submitClusterJob(command_text, experiment_path,options.reserved_memory,options.maximum_memory,options.queue,jupyterRun)
                     else:
                         submitLocalJob(cv_train_path,experiment_path,random_state,class_label,instance_label,options.instance_subset,'mi',options.n_jobs,options.use_TURF,options.TURF_pct,jupyterRun)

streamline/FeatureSelectionMain.py

@@ -76,8 +76,8 @@ def main(argv):
 
         for dataset_directory_path in dataset_paths:
             full_path = options.output_path + "/" + options.experiment_name + "/" + dataset_directory_path
+            job_counter += 1
             if eval(options.run_parallel):
-                job_counter += 1
                 submitClusterJob(full_path,options.output_path+'/'+options.experiment_name,do_mutual_info,do_multisurf,options.max_features_to_keep,options.filter_poor_features,options.top_features,options.export_scores,class_label,instance_label,cv_partitions,options.overwrite_cv,options.reserved_memory,options.maximum_memory,options.queue,jupyterRun)
             else:
                 submitLocalJob(full_path,do_mutual_info,do_multisurf,options.max_features_to_keep,options.filter_poor_features,options.top_features,options.export_scores,class_label,instance_label,cv_partitions,options.overwrite_cv,jupyterRun)

streamline/ModelMain.py

@@ -197,8 +197,8 @@ def main(argv):
                 for algorithm in algorithms:
                     algAbrev = algInfo[algorithm][1]
                     algNoSpace = algorithm.replace(" ", "_")
+                    job_counter += 1
                     if eval(options.run_parallel):
-                        job_counter += 1
                         submitClusterJob(algNoSpace,train_file_path,test_file_path,full_path,options.n_trials,options.timeout,options.lcs_timeout,options.export_hyper_sweep_plots,instance_label,class_label,random_state,options.output_path+'/'+options.experiment_name,cvCount,filter_poor_features,options.reserved_memory,options.maximum_memory,options.do_lcs_sweep,options.nu,options.iterations,options.N,options.training_subsample,options.queue,options.use_uniform_FI,options.primary_metric,algAbrev,jupyterRun)
                     else:
                         submitLocalJob(algNoSpace,train_file_path,test_file_path,full_path,options.n_trials,options.timeout,options.lcs_timeout,options.export_hyper_sweep_plots,instance_label,class_label,random_state,cvCount,filter_poor_features,options.do_lcs_sweep,options.nu,options.iterations,options.N,options.training_subsample,options.use_uniform_FI,options.primary_metric,algAbrev,jupyterRun)

streamline/PDF_ReportMain.py

@@ -20,6 +20,7 @@
 import argparse
 import time
 import glob
+import PDF_ReportJob
 
 def main(argv):
     #Parse arguments
@@ -50,8 +51,8 @@ def main(argv):
             raise Exception('Replication and Dataset paths must be specified as arguments to generate PDF summary on new data analysis!')
 
     if not options.do_check: #Run job submission
+        job_counter += 1
         if eval(options.run_parallel):
-            job_counter += 1
             submitClusterJob(experiment_path,options.training,options.rep_data_path,options.dataset_for_rep,options.reserved_memory,options.maximum_memory,options.queue)
         else:
             submitLocalJob(experiment_path,options.training,options.rep_data_path,options.dataset_for_rep)

streamline/StatsMain.py

@@ -86,9 +86,8 @@ def main(argv):
                 os.mkdir(full_path+'/model_evaluation/DT_Viz')
             if eval(do_GP) and not os.path.exists(full_path+'/model_evaluation/GP_Viz'):
                 os.mkdir(full_path+'/model_evaluation/GP_Viz')
-
+            job_counter += 1
             if eval(options.run_parallel):
-                job_counter += 1
                 submitClusterJob(full_path,options.plot_ROC,options.plot_PRC,options.plot_FI_box,class_label,instance_label,options.output_path+'/'+options.experiment_name,cv_partitions,scale_data,options.reserved_memory,options.maximum_memory,options.queue,options.plot_metric_boxplots,primary_metric,options.top_model_features,sig_cutoff,options.metric_weight,jupyterRun)
             else:
                 submitLocalJob(full_path,options.plot_ROC,options.plot_PRC,options.plot_FI_box,class_label,instance_label,cv_partitions,scale_data,options.plot_metric_boxplots,primary_metric,options.top_model_features,sig_cutoff,options.metric_weight,jupyterRun)