Merge branch 'feature/munjung-dnnsp_workflow_fcls' of https://github.com/SBNSoftware/sbndcode into feature/munjung-dnnsp_workflow_fcls

pull version with nate's gpu knob

Author: Mun Jung Jung

sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_bothrois.fcl

@@ -6,4 +6,4 @@
 
 #include "standard_detsim_sbnd.fcl"
 
-physics.producers.simtpc2d: @local::sbnd_wcls_simsp_bothrois
+physics.producers.simtpc2d: @local::sbnd_wcls_simsp_bothrois

sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_bothrois_gpu.fcl

@@ -0,0 +1,11 @@
+# File: standard_detsim_sbnd_bothrois.fcl
+# Authors: Mun Jung Jung (munjung@uchicago.edu)
+
+# runs WC 2D signal processing during detsim using both traditional and DNN-based ROI finding
+# create three Wire products: simtpc2d:gauss, simtpc2d:wiener (from traditional), simtpc2d:dnnsp (from DNN)
+
+#include "standard_detsim_sbnd.fcl"
+
+physics.producers.simtpc2d: @local::sbnd_wcls_simsp_bothrois
+physics.producers.simtpc2d.wcls_main.params.wc_device: "gpu"
+physics.producers.simtpc2d.wcls_main.plugins: [@sequence::sbnd_wcls_simsp_dnnroi.wcls_main.plugins, "WireCellCuda"]

sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_dnnroi.fcl

@@ -7,4 +7,4 @@
 
 #include "standard_detsim_sbnd.fcl"
 
-physics.producers.simtpc2d: @local::sbnd_wcls_simsp_dnnroi
+physics.producers.simtpc2d: @local::sbnd_wcls_simsp_dnnroi

sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_dnnroi_gpu.fcl

@@ -0,0 +1,12 @@
+# File: standard_detsim_sbnd_dnnroi.fcl
+# Authors: Mun Jung Jung (munjung@uchicago.edu)
+
+# runs WC 2D signal processing during detsim using only DNN-based ROI finding
+# create one Wire product: simtpc2d:dnnsp
+
+
+#include "standard_detsim_sbnd.fcl"
+
+physics.producers.simtpc2d: @local::sbnd_wcls_simsp_dnnroi
+physics.producers.simtpc2d.wcls_main.params.wc_device: "gpu" 
+physics.producers.simtpc2d.wcls_main.plugins: [@sequence::sbnd_wcls_simsp_dnnroi.wcls_main.plugins, "WireCellCuda"]

sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp-data.jsonnet

@@ -22,6 +22,7 @@
 
 //local epoch = std.extVar('epoch');  // eg "dynamic", "after", "before", "perfect"
 local sigoutform = std.extVar('signal_output_form');  // eg "sparse" or "dense"
+local wc_device = std.extVar('wc_device');
 local raw_input_label = std.extVar('raw_input_label');  // eg "daq"
 local use_paramresp = std.extVar('use_paramresp');  // eg "true" or "false"
 local roi = std.extVar('roi');
@@ -49,9 +50,9 @@ local params = data_params {
     },
 };
 
-
-local tools = tools_maker(params);
-
+local default_tools = tools_maker(params);
+local tools = if wc_device == 'gpu' then std.mergePatch(default_tools,
+    {dft: {type: "TorchDFT", data: {device: wc_device}}}) else default_tools;
 
 local mega_anode = {
   type: 'MegaAnodePlane',
@@ -217,7 +218,7 @@ local ts_p0 = {
     tick_per_slice: tick_per_slice, 
     data: {
         model: dnnroi_model_p0,
-        device: "cpu",
+        device: wc_device,
         concurrency: 1,
     },
 };
@@ -228,7 +229,7 @@ local ts_p1 = {
     tick_per_slice: tick_per_slice, 
     data: {
         model: dnnroi_model_p1,
-        device: "cpu",
+        device: wc_device,
         concurrency: 1,
     },
 };

sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp.jsonnet

@@ -21,6 +21,7 @@
 
 
 local epoch = std.extVar('epoch');  // eg "dynamic", "after", "before", "perfect"
+local wc_device = std.extVar('wc_device');
 local sigoutform = std.extVar('signal_output_form');  // eg "sparse" or "dense"
 local raw_input_label = std.extVar('raw_input_label');  // eg "daq"
 local use_dnnroi = std.extVar('use_dnnroi');
@@ -32,12 +33,15 @@ local dnnroi_model_p1 = std.extVar('dnnroi_model_p1');
 local g = import 'pgraph.jsonnet';
 local f = import 'pgrapher/experiment/sbnd/funcs.jsonnet';
 local wc = import 'wirecell.jsonnet';
-local tools_maker = import 'pgrapher/common/tools.jsonnet';
+local tools_maker = import 'pgrapher/common/tools.jsonnet';'
 
 local simu_params = import 'simparams.jsonnet';
 local params = simu_params;
 
-local tools = tools_maker(params);
+local default_tools = tools_maker(params);
+local tools = if wc_device == 'gpu' then std.mergePatch(default_tools,
+    {dft: {type: "TorchDFT", data: {device: wc_device}}}) else default_tools;
+
 
 local mega_anode = {
   type: 'MegaAnodePlane',
@@ -180,7 +184,7 @@ local ts_p0 = {
     tick_per_slice: tick_per_slice, 
     data: {
         model: dnnroi_model_p0,
-        device: "cpu",
+        device: wc_device,
         concurrency: 1,
     },
 };
@@ -191,7 +195,7 @@ local ts_p1 = {
     tick_per_slice: tick_per_slice, 
     data: {
         model: dnnroi_model_p1,
-        device: "cpu",
+        device: wc_device,
         concurrency: 1,
     },
 };

sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet

@@ -14,6 +14,7 @@ local dnnroi_model_p1 = std.extVar('dnnroi_model_p1');
 local g = import 'pgraph.jsonnet';
 local f = import 'pgrapher/common/funcs.jsonnet';
 local wc = import 'wirecell.jsonnet';
+local wc_device = std.extVar('wc_device');
 local io = import 'pgrapher/common/fileio.jsonnet';
 local tools_maker = import 'pgrapher/common/tools.jsonnet';
 
@@ -39,7 +40,9 @@ local params = base {
   }
 };
 
-local tools = tools_maker(params);
+local default_tools = tools_maker(params);
+local tools = if wc_device == 'gpu' then std.mergePatch(default_tools,
+    {dft: {type: "TorchDFT", data: {device: wc_device}}}) else default_tools;
 local sim_maker = import 'pgrapher/experiment/sbnd/sim.jsonnet';
 local sim = sim_maker(params, tools);
 local nanodes = std.length(tools.anodes);
@@ -317,7 +320,7 @@ local ts_p0 = {
     tick_per_slice: tick_per_slice, 
     data: {
         model: dnnroi_model_p0,
-        device: "cpu",
+        device: wc_device,
         concurrency: 1,
     },
 };
@@ -328,7 +331,7 @@ local ts_p1 = {
     tick_per_slice: tick_per_slice, 
     data: {
         model: dnnroi_model_p1,
-        device: "cpu",
+        device: wc_device,
         concurrency: 1,
     },
 };

sbndcode/WireCell/wcsimsp_sbnd.fcl

@@ -43,7 +43,10 @@ sbnd_wcls_simsp.wcls_main.params:   {
 
                                      ## Set "data" vs. "sim".  The epoch below probably should follow suit.
                                      reality: "sim"
-             
+
+                                     ## if using dnnroi, use this device 
+                                     wc_device: "cpu" 
+
                                      ## if epoch is "dynamic" you MUST add
                                      ## "wclsMultiChannelNoiseDB" to "inputers" and must NOT
                                      ## add it if not "dynamic"
@@ -116,6 +119,9 @@ sbnd_wcls_sp.wcls_main.params :   {
                                    # This locates the input raw::RawDigit collection in the art::Event 
                                    raw_input_label: "daq"   
 
+                                   # if dnnroi is used, this device will be used
+                                   wc_device: "cpu"   
+
                                    # Set "data" vs. "sim".  The epoch below probably should follow suit.
                                    reality: "sim"   
 
@@ -154,7 +160,6 @@ sbnd_wcls_samples_rec.wcls_main.params:   {
 
                                      ## Set "data" vs. "sim".  The epoch below probably should follow suit.
                                      reality: "sim"
-             
                                      ## if epoch is "dynamic" you MUST add
                                      ## "wclsMultiChannelNoiseDB" to "inputers" and must NOT
                                      ## add it if not "dynamic"
@@ -197,7 +202,7 @@ sbnd_wcls_samples_tru.wcls_main.params:   {
 
                                      ## Set "data" vs. "sim".  The epoch below probably should follow suit.
                                      reality: "sim"
-             
+           
                                      ## if epoch is "dynamic" you MUST add
                                      ## "wclsMultiChannelNoiseDB" to "inputers" and must NOT
                                      ## add it if not "dynamic"

sbndcode/WireCell/wcsp_data_sbnd.fcl

@@ -41,6 +41,8 @@ sbnd_wcls_sp_data.wcls_main.outputers: ["wclsFrameSaver:spsaver"]
 sbnd_wcls_sp_data.wcls_main.params :   {
                                         # This locates the input raw::RawDigit collection in the art::Event 
                                         raw_input_label: "daq"   
+                                        # if dnnroi is used, this device will be used
+                                        wc_device: "cpu"   
 
                                         # if epoch is "dynamic" you MUST add
                                         # "wclsMultiChannelNoiseDB" to "inputers" and must NOT