Skip to content

Commit e88b6b9

Browse files
jcfreeman2jcfreeman2
authored
Merge pull request #201 from DUNE-DAQ/johnfreeman/merge_fddaq-v5.3.2-rc1_to_develop
Johnfreeman/merge fddaq v5.3.2 rc1 to develop
2 parents 6f1a495 + b6f8a7c commit e88b6b9

8 files changed

Lines changed: 126 additions & 119 deletions

‎CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
cmake_minimum_required(VERSION 3.12)
2-
project(daqsystemtest VERSION 4.2.0)
2+
project(daqsystemtest VERSION 4.2.1)
33

44
find_package(daq-cmake REQUIRED)
55

‎integtest/3ru_1df_multirun_test.py‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -254,17 +254,19 @@ def test_data_files(run_nanorc):
254254
# Run some tests on the output data file
255255
assert len(run_nanorc.data_files) == expected_number_of_data_files
256256

257+
all_ok = True
257258
for idx in range(len(run_nanorc.data_files)):
258259
data_file = data_file_checks.DataFile(run_nanorc.data_files[idx])
259-
assert data_file_checks.sanity_check(data_file)
260-
assert data_file_checks.check_file_attributes(data_file)
261-
assert data_file_checks.check_event_count(
260+
all_ok &= data_file_checks.sanity_check(data_file)
261+
all_ok &= data_file_checks.check_file_attributes(data_file)
262+
all_ok &= data_file_checks.check_event_count(
262263
data_file, local_expected_event_count, local_event_count_tolerance
263264
)
264265
for jdx in range(len(fragment_check_list)):
265-
assert data_file_checks.check_fragment_count(
266+
all_ok &= data_file_checks.check_fragment_count(
266267
data_file, fragment_check_list[jdx]
267268
)
268-
assert data_file_checks.check_fragment_sizes(
269+
all_ok &= data_file_checks.check_fragment_sizes(
269270
data_file, fragment_check_list[jdx]
270271
)
272+
assert all_ok

‎integtest/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ pytest -s -k TPG 3ru_3df_multirun_test.py
1717

1818
For reference, here are the ideas behind the tests that currently exist in this repository:
1919
* `minimal_system_quick_test.py` - verify that a small emulator system works and successfully writes data in a short run
20-
* `readout_type_scan.py` - verify that we can write different types of data (WIBEth, PDS, TPG, etc.)
20+
* `readout_type_scan.py` - verify that we can write different types of data (WIBEth, DAPHNE, TDE, etc.)
2121
* `3ru_3df_multirun_test.py` - verify that a system with multiple RUs and multiple DF Apps works as expected, including TPG
2222
* `small_footprint_quick_test.py` - like minimal_system_quick_test, but using even fewer computer resources
2323
* `fake_data_producer_test.py` - verify that the FakeDataProdModule DAQModule works as expected

‎integtest/fake_data_producer_test.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,8 +143,6 @@ def test_log_files(run_nanorc):
143143
def test_data_files(run_nanorc):
144144
local_expected_event_count = expected_event_count
145145
local_event_count_tolerance = expected_event_count_tolerance
146-
# frag_params=wib1_frag_hsi_trig_params # ProtoWIB
147-
# frag_params=wib2_frag_params # DuneWIB
148146
frag_params = wibeth_frag_params
149147
current_test = os.environ.get("PYTEST_CURRENT_TEST")
150148
if "Double" in current_test:

‎integtest/readout_type_scan.py‎

Lines changed: 85 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
# Values that help determine the running conditions
1717
number_of_data_producers = 2
1818
run_duration = 20 # seconds
19-
data_rate_slowdown_factor = 10
19+
data_rate_slowdown_factor = 10 # is this still used anywhere? (KAB, 28-Apr-2050)
2020
ta_prescale = 100
2121

2222
# Default values for validation parameters
@@ -32,13 +32,6 @@
3232
"min_size_bytes": 7272,
3333
"max_size_bytes": 14472,
3434
}
35-
wibeth_frag_multi_trig_params = {
36-
"fragment_type_description": "WIBEth",
37-
"fragment_type": "WIBEth",
38-
"expected_fragment_count": number_of_data_producers,
39-
"min_size_bytes": 7272,
40-
"max_size_bytes": 14472,
41-
}
4235
tde_frag_params = {
4336
"fragment_type_description": "TDEEth",
4437
"fragment_type": "TDEEth",
@@ -51,20 +44,23 @@
5144
# num frames = ro_win / tick diff = 977
5245
# fragment size = num frames * frame size = 461026
5346

54-
pds_stream_frag_params = {
55-
"fragment_type_description": "PDSStream",
47+
daphne_stream_frag_params = {
48+
"fragment_type_description": "DAPHNEStream",
5649
"fragment_type": "DAPHNEStream",
5750
"expected_fragment_count": number_of_data_producers,
5851
"min_size_bytes": 72+461026-20*472,
5952
"max_size_bytes": 72+461026+20*472,
6053
}
61-
pds_frag_params = {
62-
"fragment_type_description": "PDS",
54+
daphne_frag_params = {
55+
"fragment_type_description": "DAPHNE",
6356
"fragment_type": "DAPHNE",
6457
"expected_fragment_count": number_of_data_producers,
65-
"min_size_bytes": 435912,
66-
"max_size_bytes": 1133256,
67-
} # 20 x 21792; 52 x 21792 (+72)
58+
"min_size_bytes": 1936,
59+
"max_size_bytes": 120000,
60+
"frag_sizes_by_TC_type": {"kPrescale": {"min_size_bytes": 1936, "max_size_bytes": 32000},
61+
"kRandom": {"min_size_bytes": 112000, "max_size_bytes": 118000},
62+
"default": {"min_size_bytes": 1936, "max_size_bytes": 118000} }
63+
}
6864

6965
# sizes: 128 is for one TC with zero TAs inside it (72+56)
7066
# 208 is for one TC with one TA inside it (72+56+80)
@@ -97,13 +93,20 @@
9793
}
9894
# sizes: 72 is for an empty TP fragment
9995
# 144 is for a fragment with three TPs in it (72+24+24+24)
100-
triggerprimitive_frag_params = {
96+
wibeth_triggerprimitive_frag_params = {
10197
"fragment_type_description": "Trigger Primitive",
10298
"fragment_type": "Trigger_Primitive",
103-
"expected_fragment_count": 3,
99+
"expected_fragment_count": (1 * 3), # number of readout apps * 3
104100
"min_size_bytes": 72,
105101
"max_size_bytes": 144,
106102
}
103+
daphne_triggerprimitive_frag_params = {
104+
"fragment_type_description": "Trigger Primitive",
105+
"fragment_type": "Trigger_Primitive",
106+
"expected_fragment_count": 1, # number of readout apps
107+
"min_size_bytes": 96,
108+
"max_size_bytes": 4392,
109+
}
107110
ignored_logfile_problems = {
108111
"-controller": [
109112
"Worker with pid \\d+ was terminated due to signal 1",
@@ -141,12 +144,12 @@
141144
)
142145
)
143146

144-
tpg_conf = copy.deepcopy(conf_dict)
145-
tpg_conf.tpg_enabled = True
146-
tpg_conf.frame_file = (
147+
wib_tpg_conf = copy.deepcopy(conf_dict)
148+
wib_tpg_conf.tpg_enabled = True
149+
wib_tpg_conf.frame_file = (
147150
"asset://?checksum=dd156b4895f1b06a06b6ff38e37bd798" # WIBEth All Zeros
148151
)
149-
tpg_conf.config_substitutions.append(
152+
wib_tpg_conf.config_substitutions.append(
150153
data_classes.config_substitution(
151154
obj_class="TAMakerPrescaleAlgorithm",
152155
obj_id="dummy-ta-maker",
@@ -163,11 +166,11 @@
163166
tde_conf.frame_file = "asset://?checksum=dd156b4895f1b06a06b6ff38e37bd798" # WIBEth All Zeros
164167
#tde_conf.frame_file = "asset://?checksum=759e5351436bead208cf4963932d6327"
165168

166-
pds_stream_conf = copy.deepcopy(conf_dict)
167-
pds_stream_conf.dro_map_config.det_id = 2 # det_id = 2 for HD_PDS
168-
pds_stream_conf.frame_file = "asset://?label=DAPHNEStream&subsystem=readout"
169+
daphne_stream_conf = copy.deepcopy(conf_dict)
170+
daphne_stream_conf.dro_map_config.det_id = 2 # det_id = 2 for HD_PDS
171+
daphne_stream_conf.frame_file = "asset://?label=DAPHNEStream&subsystem=readout"
169172

170-
pds_stream_conf.config_substitutions.append(
173+
daphne_stream_conf.config_substitutions.append(
171174
data_classes.config_substitution(
172175
obj_class="TCReadoutMap",
173176
obj_id = "def-random-readout",
@@ -178,10 +181,10 @@
178181
)
179182
)
180183

181-
pds_conf = copy.deepcopy(conf_dict)
182-
pds_conf.dro_map_config.det_id = 2 # det_id = 2 for HD_PDS
183-
pds_conf.frame_file = "asset://?label=DAPHNE&subsystem=readout"
184-
pds_conf.config_substitutions.append(
184+
daphne_conf = copy.deepcopy(conf_dict)
185+
daphne_conf.dro_map_config.det_id = 2 # det_id = 2 for HD_PDS
186+
daphne_conf.frame_file = "asset://?checksum=a8990a9eb3a505d4ded62dfdfa9e2681"
187+
daphne_conf.config_substitutions.append(
185188
data_classes.config_substitution(
186189
obj_class="TCReadoutMap",
187190
obj_id = "def-random-readout",
@@ -191,24 +194,25 @@
191194
},
192195
)
193196
)
194-
# pacman_conf = copy.deepcopy(conf_dict)
195-
# pacman_conf.dro_map_config.det_id = 32 # det_id = 32 for NDLAr_TPC
196-
# pacman_conf.frame_file == "asset://?label=PACMAN&subsystem=readout"
197197

198-
# mpd_conf = copy.deepcopy(conf_dict)
199-
# mpd_conf.dro_map_config.det_id = 33 # det_id = 33 for NDLAr_PDS
200-
# mpd_conf.frame_file = "asset://?label=MPD&subsystem=readout"
198+
daphne_tpg_conf = copy.deepcopy(daphne_conf)
199+
daphne_tpg_conf.tpg_enabled = True
200+
daphne_tpg_conf.config_substitutions.append(
201+
data_classes.config_substitution(
202+
obj_class="TAMakerPrescaleAlgorithm",
203+
obj_id="dummy-ta-maker",
204+
updates={"prescale": 750},
205+
)
206+
)
201207

202208

203209
confgen_arguments = {
204-
# "WIB1_System": wib1_conf,
205210
"WIBEth_System": wibeth_conf,
206-
"TPG_System": tpg_conf,
207-
"PDS_Stream_System": pds_stream_conf,
208-
# "PDS_System": pds_conf,
209-
"TDEEth_System": tde_conf,
210-
# "PACMAN_System": pacman_conf,
211-
# "MPD_System": mpd_conf
211+
"WIBEth_TPG_System": wib_tpg_conf,
212+
"DAPHNE_Stream_System": daphne_stream_conf,
213+
"DAPHNE_System": daphne_conf,
214+
"DAPHNE_TPG_System": daphne_tpg_conf,
215+
"TDEEth_System": tde_conf
212216
}
213217

214218
# The commands to run in nanorc, as a list
@@ -217,6 +221,7 @@
217221
+ [str(run_duration)]
218222
+ "disable-triggers wait 2 drain-dataflow stop-trigger-sources wait 2 stop scrap terminate".split()
219223
)
224+
# + "disable-triggers wait 5 drain-dataflow wait 2 stop-trigger-sources wait 2 stop scrap terminate".split()
220225

221226
# The tests themselves
222227

@@ -247,34 +252,46 @@ def test_log_files(run_nanorc):
247252
def test_data_files(run_nanorc):
248253
local_expected_event_count = expected_event_count
249254
local_event_count_tolerance = expected_event_count_tolerance
250-
fragment_check_list = []
255+
fragment_check_list = [triggercandidate_frag_params]
256+
current_test = os.environ.get("PYTEST_CURRENT_TEST")
257+
if "DAPHNE_Stream" in current_test:
258+
fragment_check_list.append(daphne_stream_frag_params)
259+
elif "DAPHNE" in current_test:
260+
fragment_check_list.append(daphne_frag_params)
261+
elif "WIBEth" in current_test:
262+
fragment_check_list.append(wibeth_frag_params)
263+
elif "TDEEth" in current_test:
264+
fragment_check_list.append(tde_frag_params)
251265
if run_nanorc.confgen_config.tpg_enabled:
252-
local_expected_event_count += (
253-
(6250 / ta_prescale)
254-
* number_of_data_producers
255-
* run_duration
256-
/ 100
257-
)
258-
local_event_count_tolerance += (
259-
(250 / ta_prescale)
260-
* number_of_data_producers
261-
* run_duration
262-
/ 100
263-
)
264-
fragment_check_list.append(wibeth_frag_multi_trig_params)
265266
fragment_check_list.append(triggeractivity_frag_params)
266-
fragment_check_list.append(triggerprimitive_frag_params)
267-
else:
268-
fragment_check_list.append(triggercandidate_frag_params)
269-
current_test = os.environ.get("PYTEST_CURRENT_TEST")
270-
if "PDS_Stream" in current_test:
271-
fragment_check_list.append(pds_stream_frag_params)
272-
elif "PDS" in current_test:
273-
fragment_check_list.append(pds_frag_params)
274-
elif "WIBEth" in current_test:
275-
fragment_check_list.append(wibeth_frag_params)
276-
elif "TDEEth" in current_test:
277-
fragment_check_list.append(tde_frag_params)
267+
if "WIBEth" in current_test:
268+
fragment_check_list.append(wibeth_triggerprimitive_frag_params)
269+
local_expected_event_count += (
270+
(6250 / ta_prescale)
271+
* number_of_data_producers
272+
* run_duration
273+
/ 100
274+
)
275+
local_event_count_tolerance += (
276+
(250 / ta_prescale)
277+
* number_of_data_producers
278+
* run_duration
279+
/ 100
280+
)
281+
if "DAPHNE" in current_test:
282+
fragment_check_list.append(daphne_triggerprimitive_frag_params)
283+
local_expected_event_count += (
284+
(6250 / ta_prescale)
285+
* number_of_data_producers
286+
* run_duration * 3
287+
/ 200
288+
)
289+
local_event_count_tolerance += (
290+
(250 / ta_prescale)
291+
* number_of_data_producers
292+
* run_duration * 6
293+
/ 250
294+
)
278295

279296
# Run some tests on the output data file
280297
all_ok = True

‎integtest/small_footprint_quick_test.py‎

Lines changed: 7 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -17,20 +17,6 @@
1717
check_for_logfile_errors = True
1818
expected_event_count = run_duration
1919
expected_event_count_tolerance = 2
20-
wib1_frag_hsi_trig_params = {
21-
"fragment_type_description": "WIB",
22-
"fragment_type": "ProtoWIB",
23-
"expected_fragment_count": number_of_data_producers,
24-
"min_size_bytes": 37656,
25-
"max_size_bytes": 37656,
26-
}
27-
wib2_frag_params = {
28-
"fragment_type_description": "WIB2",
29-
"fragment_type": "WIB",
30-
"expected_fragment_count": number_of_data_producers,
31-
"min_size_bytes": 29808,
32-
"max_size_bytes": 30280,
33-
}
3420
wibeth_frag_params = {
3521
"fragment_type_description": "WIBEth",
3622
"fragment_type": "WIBEth",
@@ -128,21 +114,21 @@ def test_data_files(run_nanorc):
128114
assert len(run_nanorc.data_files) == expected_number_of_data_files
129115

130116
fragment_check_list = [triggercandidate_frag_params, hsi_frag_params]
131-
# fragment_check_list.append(wib1_frag_hsi_trig_params) # ProtoWIB
132-
# fragment_check_list.append(wib2_frag_params) # DuneWIB
133117
fragment_check_list.append(wibeth_frag_params) # WIBEth
134118

119+
all_ok = True
135120
for idx in range(len(run_nanorc.data_files)):
136121
data_file = data_file_checks.DataFile(run_nanorc.data_files[idx])
137-
assert data_file_checks.sanity_check(data_file)
138-
assert data_file_checks.check_file_attributes(data_file)
139-
assert data_file_checks.check_event_count(
122+
all_ok &= data_file_checks.sanity_check(data_file)
123+
all_ok &= data_file_checks.check_file_attributes(data_file)
124+
all_ok &= data_file_checks.check_event_count(
140125
data_file, expected_event_count, expected_event_count_tolerance
141126
)
142127
for jdx in range(len(fragment_check_list)):
143-
assert data_file_checks.check_fragment_count(
128+
all_ok &= data_file_checks.check_fragment_count(
144129
data_file, fragment_check_list[jdx]
145130
)
146-
assert data_file_checks.check_fragment_sizes(
131+
all_ok &= data_file_checks.check_fragment_sizes(
147132
data_file, fragment_check_list[jdx]
148133
)
134+
assert all_ok

‎integtest/tpstream_writing_test.py‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -230,20 +230,22 @@ def test_data_files(run_nanorc):
230230
or len(run_nanorc.data_files) == low_number_of_files
231231
)
232232

233+
all_ok = True
233234
for idx in range(len(run_nanorc.data_files)):
234235
data_file = data_file_checks.DataFile(run_nanorc.data_files[idx])
235-
assert data_file_checks.sanity_check(data_file)
236-
assert data_file_checks.check_file_attributes(data_file)
237-
assert data_file_checks.check_event_count(
236+
all_ok &= data_file_checks.sanity_check(data_file)
237+
all_ok &= data_file_checks.check_file_attributes(data_file)
238+
all_ok &= data_file_checks.check_event_count(
238239
data_file, local_expected_event_count, local_event_count_tolerance
239240
)
240241
for jdx in range(len(fragment_check_list)):
241-
assert data_file_checks.check_fragment_count(
242+
all_ok &= data_file_checks.check_fragment_count(
242243
data_file, fragment_check_list[jdx]
243244
)
244-
assert data_file_checks.check_fragment_sizes(
245+
all_ok &= data_file_checks.check_fragment_sizes(
245246
data_file, fragment_check_list[jdx]
246247
)
248+
assert all_ok
247249

248250

249251
def test_tpstream_files(run_nanorc):
@@ -256,17 +258,19 @@ def test_tpstream_files(run_nanorc):
256258

257259
assert len(tpstream_files) == 2 # one for each run
258260

261+
all_ok = True
259262
for idx in range(len(tpstream_files)):
260263
data_file = data_file_checks.DataFile(tpstream_files[idx])
261-
# assert data_file_checks.sanity_check(data_file) # Sanity check doesn't work for stream files
262-
assert data_file_checks.check_file_attributes(data_file)
263-
assert data_file_checks.check_event_count(
264+
# all_ok &= data_file_checks.sanity_check(data_file) # Sanity check doesn't work for stream files
265+
all_ok &= data_file_checks.check_file_attributes(data_file)
266+
all_ok &= data_file_checks.check_event_count(
264267
data_file, local_expected_event_count, local_event_count_tolerance
265268
)
266269
for jdx in range(len(fragment_check_list)):
267-
assert data_file_checks.check_fragment_count(
270+
all_ok &= data_file_checks.check_fragment_count(
268271
data_file, fragment_check_list[jdx]
269272
)
270-
assert data_file_checks.check_fragment_sizes(
273+
all_ok &= data_file_checks.check_fragment_sizes(
271274
data_file, fragment_check_list[jdx]
272275
)
276+
assert all_ok

0 commit comments

Comments
 (0)