From 3e349bff58773ca0e43359282f08953c162ad313 Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Fri, 29 May 2026 09:43:06 -0500 Subject: [PATCH 1/3] Modified various integtests to clean up the setting of trigger and readout window parameters and start making use of common functions to set those params. --- integtest/disabled_output_test.py | 8 ++---- integtest/hdf5_compression_test.py | 31 +++------------------ integtest/insufficient_disk_space_test.py | 17 ++++-------- integtest/large_trigger_record_test.py | 33 +++++++---------------- integtest/max_file_size_test.py | 30 +++------------------ integtest/multiple_data_writers_test.py | 31 +++------------------ integtest/offline_prod_run_test.py | 10 ++----- integtest/trmonrequestor_test.py | 14 +++------- 8 files changed, 31 insertions(+), 143 deletions(-) diff --git a/integtest/disabled_output_test.py b/integtest/disabled_output_test.py index aa359131..b4b6471b 100644 --- a/integtest/disabled_output_test.py +++ b/integtest/disabled_output_test.py @@ -10,6 +10,7 @@ import integrationtest.basic_checks as basic_checks import integrationtest.data_classes as data_classes import integrationtest.resource_validation as resource_validation +import integrationtest.utility_functions2 as utility_functions from integrationtest.get_pytest_tmpdir import get_pytest_tmpdir from integrationtest.verbosity_helper import IntegtestVerbosityLevels @@ -100,17 +101,12 @@ conf_dict.op_env = "integtest" conf_dict.config_session_name= "disabled" conf_dict.tpg_enabled = False +utility_functions.set_RTCM_trigger_params(conf_dict, trigger_rate=trigger_rate) # We accept the default values for all of the other integtest config parameters # (defined in integrationtest/src/integrationtest/data_classes.py), including the "frame_file", # which is the data file that is used to emulated the data. The current default for that field # specifies a set of WIBEth frames from a relatively recent run at EHN1.) -conf_dict.config_substitutions.append( - data_classes.attribute_substitution( - obj_class="RandomTCMakerConf", - updates={"trigger_rate_hz": trigger_rate}, - ) -) conf_dict.config_substitutions.append( data_classes.attribute_substitution( obj_class="TCDataProcessor", diff --git a/integtest/hdf5_compression_test.py b/integtest/hdf5_compression_test.py index 02d3f137..10433bfa 100644 --- a/integtest/hdf5_compression_test.py +++ b/integtest/hdf5_compression_test.py @@ -8,6 +8,7 @@ import integrationtest.basic_checks as basic_checks import integrationtest.data_classes as data_classes import integrationtest.resource_validation as resource_validation +import integrationtest.utility_functions2 as utility_functions from integrationtest.get_pytest_tmpdir import get_pytest_tmpdir from integrationtest.verbosity_helper import IntegtestVerbosityLevels @@ -124,7 +125,9 @@ conf_dict.op_env = "integtest" conf_dict.config_session_name= "hdf5compression" conf_dict.tpg_enabled = True -conf_dict.fake_hsi_enabled = True +utility_functions.enable_fake_hsi_trigger(conf_dict, trigger_rate=10.0, + readout_window_before_ticks=120000, + readout_window_after_ticks=1000) conf_dict.dro_map_config.det_id = 2 # det_id = 2 for kHD_PDS conf_dict.frame_file = "asset://?checksum=a8990a9eb3a505d4ded62dfdfa9e2681" # run 36012 DAPHNE data #conf_dict.frame_file = "file:///home/nfs/biery/dunedaq/12MayFDv5.3.2DevInstrUpdate/sourcecode/dfmodules/integtest/np02vdcoldbox_run035227_sample_hd_pds.bin" @@ -137,32 +140,6 @@ ) ) -conf_dict.config_substitutions.append( - data_classes.attribute_substitution( - obj_class="FakeHSIEventGeneratorConf", - updates={"trigger_rate": 10.0}, - ) -) - -conf_dict.config_substitutions.append( - data_classes.attribute_substitution( - obj_class="HSISignalWindow", - updates={ - "time_before": 1000, - "time_after": 500, - }, - ) -) -conf_dict.config_substitutions.append( - data_classes.attribute_substitution( - obj_class="TCReadoutMap", - obj_id = "def-hsi-tc-map", - updates={ - "time_before": 120000, - "time_after": 1000, - }, - ) -) conf_dict.config_substitutions.append( data_classes.attribute_substitution( obj_class="DataStoreConf", diff --git a/integtest/insufficient_disk_space_test.py b/integtest/insufficient_disk_space_test.py index c504e661..90cce663 100644 --- a/integtest/insufficient_disk_space_test.py +++ b/integtest/insufficient_disk_space_test.py @@ -8,6 +8,7 @@ import integrationtest.basic_checks as basic_checks import integrationtest.data_classes as data_classes import integrationtest.resource_validation as resource_validation +import integrationtest.utility_functions2 as utility_functions from integrationtest.get_pytest_tmpdir import get_pytest_tmpdir from integrationtest.verbosity_helper import IntegtestVerbosityLevels @@ -97,18 +98,10 @@ conf_dict.tpg_enabled = False conf_dict.n_df_apps = number_of_dataflow_apps conf_dict.fake_hsi_enabled = False - -conf_dict.config_substitutions.append( - data_classes.attribute_substitution( - obj_class="RandomTCMakerConf", - updates={ - "trigger_rate_hz": trigger_rate, - "candidate_backshift_ts": 0, - "candidate_window_before_ts": readout_window_time_before, - "candidate_window_after_ts": readout_window_time_after - }, - ) -) +utility_functions.set_RTCM_trigger_params(conf_dict, trigger_rate=trigger_rate, + readout_window_backshift_ticks=0, + readout_window_before_ticks=readout_window_time_before, + readout_window_after_ticks=readout_window_time_after) conf_dict.config_substitutions.append( data_classes.attribute_substitution( diff --git a/integtest/large_trigger_record_test.py b/integtest/large_trigger_record_test.py index be3ff21e..0d551f01 100644 --- a/integtest/large_trigger_record_test.py +++ b/integtest/large_trigger_record_test.py @@ -15,6 +15,7 @@ import integrationtest.basic_checks as basic_checks import integrationtest.data_classes as data_classes import integrationtest.resource_validation as resource_validation +import integrationtest.utility_functions2 as utility_functions from integrationtest.get_pytest_tmpdir import get_pytest_tmpdir from integrationtest.verbosity_helper import IntegtestVerbosityLevels @@ -90,18 +91,11 @@ conf_dict.config_session_name= "largerecord" conf_dict.tpg_enabled = False conf_dict.n_df_apps = number_of_dataflow_apps +utility_functions.set_RTCM_trigger_params(conf_dict, trigger_rate=trigger_rate, + readout_window_backshift_ticks=0, + readout_window_before_ticks=readout_window_time_before, + readout_window_after_ticks=readout_window_time_after) -conf_dict.config_substitutions.append( - data_classes.attribute_substitution( - obj_class="RandomTCMakerConf", - updates={ - "trigger_rate_hz": trigger_rate, - "candidate_backshift_ts": 0, - "candidate_window_before_ts": readout_window_time_before, - "candidate_window_after_ts": readout_window_time_after - }, - ) -) conf_dict.config_substitutions.append( data_classes.attribute_substitution( obj_class="DataStoreConf", @@ -132,19 +126,10 @@ ) ) oversize_conf = copy.deepcopy(conf_dict) # Copy before setting the readout window - -# Now set the readout window for the over-size case -oversize_conf.config_substitutions.append( - data_classes.attribute_substitution( - obj_class="RandomTCMakerConf", - updates={ - "trigger_rate_hz": trigger_rate, - "candidate_backshift_ts": 0, - "candidate_window_before_ts": 2.5 * readout_window_time_before, - "candidate_window_after_ts": readout_window_time_after - }, - ) -) +utility_functions.set_RTCM_trigger_params(oversize_conf, trigger_rate=trigger_rate, + readout_window_backshift_ticks=0, + readout_window_before_ticks=2.5*readout_window_time_before, + readout_window_after_ticks=readout_window_time_after) confgen_arguments = { "TRSize_55PercentOfMaxFileSize": conf_dict, diff --git a/integtest/max_file_size_test.py b/integtest/max_file_size_test.py index 12a072b4..3e622664 100644 --- a/integtest/max_file_size_test.py +++ b/integtest/max_file_size_test.py @@ -8,6 +8,7 @@ import integrationtest.basic_checks as basic_checks import integrationtest.data_classes as data_classes import integrationtest.resource_validation as resource_validation +import integrationtest.utility_functions2 as utility_functions from integrationtest.get_pytest_tmpdir import get_pytest_tmpdir from integrationtest.verbosity_helper import IntegtestVerbosityLevels @@ -106,10 +107,11 @@ conf_dict.op_env = "integtest" conf_dict.config_session_name= "maxfilesize" conf_dict.tpg_enabled = True -conf_dict.fake_hsi_enabled = True conf_dict.frame_file = ( "asset://?checksum=dd156b4895f1b06a06b6ff38e37bd798" # WIBEth All Zeros ) +utility_functions.enable_fake_hsi_trigger(conf_dict, trigger_rate=10.0, + readout_window_before_ticks=52000, readout_window_after_ticks=1000) conf_dict.config_substitutions.append( data_classes.attribute_substitution( @@ -124,32 +126,6 @@ ) ) -conf_dict.config_substitutions.append( - data_classes.attribute_substitution( - obj_class="FakeHSIEventGeneratorConf", - updates={"trigger_rate": 10.0}, - ) -) - -conf_dict.config_substitutions.append( - data_classes.attribute_substitution( - obj_class="HSISignalWindow", - updates={ - "time_before": 1000, - "time_after": 500, - }, - ) -) -conf_dict.config_substitutions.append( - data_classes.attribute_substitution( - obj_class="TCReadoutMap", - obj_id = "def-hsi-tc-map", - updates={ - "time_before": 52000, - "time_after": 1000, - }, - ) -) conf_dict.config_substitutions.append( data_classes.attribute_substitution( obj_class="DataStoreConf", diff --git a/integtest/multiple_data_writers_test.py b/integtest/multiple_data_writers_test.py index 24ad902c..2deb22bf 100644 --- a/integtest/multiple_data_writers_test.py +++ b/integtest/multiple_data_writers_test.py @@ -8,6 +8,7 @@ import integrationtest.basic_checks as basic_checks import integrationtest.data_classes as data_classes import integrationtest.resource_validation as resource_validation +import integrationtest.utility_functions2 as utility_functions from integrationtest.get_pytest_tmpdir import get_pytest_tmpdir from integrationtest.verbosity_helper import IntegtestVerbosityLevels @@ -85,35 +86,9 @@ conf_dict.dro_map_config.n_apps = number_of_readout_apps conf_dict.op_env = "integtest" conf_dict.config_session_name= "multidatawriter" -conf_dict.fake_hsi_enabled = True conf_dict.n_data_writers = 3 - -conf_dict.config_substitutions.append( - data_classes.attribute_substitution( - obj_class="FakeHSIEventGeneratorConf", - updates={"trigger_rate": 10.0}, - ) -) - -conf_dict.config_substitutions.append( - data_classes.attribute_substitution( - obj_class="HSISignalWindow", - updates={ - "time_before": 1000, - "time_after": 500, - }, - ) -) -conf_dict.config_substitutions.append( - data_classes.attribute_substitution( - obj_class="TCReadoutMap", - obj_id = "def-hsi-tc-map", - updates={ - "time_before": 52000, - "time_after": 1000, - }, - ) -) +utility_functions.enable_fake_hsi_trigger(conf_dict, trigger_rate=10.0, + readout_window_before_ticks=52000, readout_window_after_ticks=1000) conf_dict.config_substitutions.append( data_classes.attribute_substitution( diff --git a/integtest/offline_prod_run_test.py b/integtest/offline_prod_run_test.py index 51d3b95e..dbd124eb 100644 --- a/integtest/offline_prod_run_test.py +++ b/integtest/offline_prod_run_test.py @@ -7,6 +7,7 @@ import integrationtest.log_file_checks as log_file_checks import integrationtest.basic_checks as basic_checks import integrationtest.data_classes as data_classes +import integrationtest.utility_functions2 as utility_functions from integrationtest.verbosity_helper import IntegtestVerbosityLevels import functools @@ -66,19 +67,12 @@ conf_dict.op_env = "integtest" conf_dict.config_session_name= "prodruntype" conf_dict.tpg_enabled = False -conf_dict.fake_hsi_enabled = True +utility_functions.enable_fake_hsi_trigger(conf_dict, trigger_rate=1.0) conf_dict.config_substitutions.append( data_classes.attribute_substitution(obj_class="LatencyBuffer", updates={"size": 50000}) ) -conf_dict.config_substitutions.append( - data_classes.attribute_substitution( - obj_class="FakeHSIEventGeneratorConf", - updates={"trigger_rate": 1.0}, - ) -) - confgen_arguments = {"OfflineProdRun": conf_dict} # The commands to run in dunerc, as a list dunerc_command_list = ( diff --git a/integtest/trmonrequestor_test.py b/integtest/trmonrequestor_test.py index 9550074b..f61e4ad8 100755 --- a/integtest/trmonrequestor_test.py +++ b/integtest/trmonrequestor_test.py @@ -7,6 +7,7 @@ import integrationtest.log_file_checks as log_file_checks import integrationtest.basic_checks as basic_checks import integrationtest.data_classes as data_classes +import integrationtest.utility_functions2 as utility_functions from integrationtest.verbosity_helper import IntegtestVerbosityLevels import functools @@ -67,17 +68,8 @@ conf_dict.tpg_enabled = False conf_dict.trmon_app_enabled = True conf_dict.n_df_apps = number_of_dataflow_apps - - -substitution = data_classes.attribute_substitution( - obj_id="random-tc-generator", - obj_class="RandomTCMakerConf", - updates={ - "trigger_rate_hz": trigger_rate, - "candidate_backshift_ts": 0, - }, -) -conf_dict.config_substitutions.append(substitution) +utility_functions.enable_fake_hsi_trigger(conf_dict, trigger_rate=trigger_rate, + readout_window_backshift_ticks=0) substitution = data_classes.attribute_substitution( obj_id="tr_mon_dw-01", From 086f35ab01220be6a6ea939eaeecc1f87dd3a0af Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Fri, 29 May 2026 12:48:23 -0500 Subject: [PATCH 2/3] Fixed trigger_params function call - should be RTCM, not fake_hsi. --- integtest/trmonrequestor_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integtest/trmonrequestor_test.py b/integtest/trmonrequestor_test.py index f61e4ad8..929229c9 100755 --- a/integtest/trmonrequestor_test.py +++ b/integtest/trmonrequestor_test.py @@ -68,7 +68,7 @@ conf_dict.tpg_enabled = False conf_dict.trmon_app_enabled = True conf_dict.n_df_apps = number_of_dataflow_apps -utility_functions.enable_fake_hsi_trigger(conf_dict, trigger_rate=trigger_rate, +utility_functions.set_RTCM_trigger_params(conf_dict, trigger_rate=trigger_rate, readout_window_backshift_ticks=0) substitution = data_classes.attribute_substitution( From d97a8ab8c8bef256e40a75ada55d08cbc34221da Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Tue, 30 Jun 2026 13:08:23 -0500 Subject: [PATCH 3/3] Modified the integtests to take into account Michal's suggestions in PR #491 and DUNE-DAQ/interationtest#161 --- integtest/disabled_output_test.py | 2 +- integtest/insufficient_disk_space_test.py | 2 +- integtest/large_trigger_record_test.py | 6 +++--- integtest/trmonrequestor_test.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/integtest/disabled_output_test.py b/integtest/disabled_output_test.py index 168d423e..e9ffb18e 100644 --- a/integtest/disabled_output_test.py +++ b/integtest/disabled_output_test.py @@ -100,7 +100,7 @@ conf_dict.op_env = "integtest" conf_dict.config_session_name= "disabled" conf_dict.tpg_enabled = False -utility_functions.set_RTCM_trigger_params(conf_dict, trigger_rate=trigger_rate) +utility_functions.set_rtcm_trigger_params(conf_dict, trigger_rate=trigger_rate) # We accept the default values for all of the other integtest config parameters # (defined in integrationtest/src/integrationtest/data_classes.py), including the "frame_file", # which is the data file that is used to emulated the data. The current default for that field diff --git a/integtest/insufficient_disk_space_test.py b/integtest/insufficient_disk_space_test.py index cea29654..603700b2 100644 --- a/integtest/insufficient_disk_space_test.py +++ b/integtest/insufficient_disk_space_test.py @@ -98,7 +98,7 @@ conf_dict.n_df_apps = number_of_dataflow_apps conf_dict.fake_hsi_enabled = False conf_dict.remove_hdf5_files = True -utility_functions.set_RTCM_trigger_params(conf_dict, trigger_rate=trigger_rate, +utility_functions.set_rtcm_trigger_params(conf_dict, trigger_rate=trigger_rate, readout_window_backshift_ticks=0, readout_window_before_ticks=readout_window_time_before, readout_window_after_ticks=readout_window_time_after) diff --git a/integtest/large_trigger_record_test.py b/integtest/large_trigger_record_test.py index 6b25a9f8..63d30a25 100644 --- a/integtest/large_trigger_record_test.py +++ b/integtest/large_trigger_record_test.py @@ -91,7 +91,7 @@ conf_dict.tpg_enabled = False conf_dict.n_df_apps = number_of_dataflow_apps conf_dict.remove_hdf5_files = True -utility_functions.set_RTCM_trigger_params(conf_dict, trigger_rate=trigger_rate, +utility_functions.set_rtcm_trigger_params(conf_dict, trigger_rate=trigger_rate, readout_window_backshift_ticks=0, readout_window_before_ticks=readout_window_time_before, readout_window_after_ticks=readout_window_time_after) @@ -126,9 +126,9 @@ ) ) oversize_conf = copy.deepcopy(conf_dict) # Copy before setting the readout window -utility_functions.set_RTCM_trigger_params(oversize_conf, trigger_rate=trigger_rate, +utility_functions.set_rtcm_trigger_params(oversize_conf, trigger_rate=trigger_rate, readout_window_backshift_ticks=0, - readout_window_before_ticks=2.5*readout_window_time_before, + readout_window_before_ticks=int(2.5*readout_window_time_before), readout_window_after_ticks=readout_window_time_after) confgen_arguments = { diff --git a/integtest/trmonrequestor_test.py b/integtest/trmonrequestor_test.py index 7784ae51..ff77f2a4 100755 --- a/integtest/trmonrequestor_test.py +++ b/integtest/trmonrequestor_test.py @@ -67,7 +67,7 @@ conf_dict.tpg_enabled = False conf_dict.trmon_app_enabled = True conf_dict.n_df_apps = number_of_dataflow_apps -utility_functions.set_RTCM_trigger_params(conf_dict, trigger_rate=trigger_rate, +utility_functions.set_rtcm_trigger_params(conf_dict, trigger_rate=trigger_rate, readout_window_backshift_ticks=0) substitution = data_classes.attribute_substitution(