Skip to content

Commit ddcf397

Browse files
Updates to validation
1 parent 48844fc commit ddcf397

7 files changed

Lines changed: 250 additions & 218 deletions

File tree

validation/README.md

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,13 @@ We want to make sure that any changes we make to the code don't:
88
From ```EventNtuple/``` directory:
99

1010
```
11-
. ./validation/test_fcls.sh [MDC2020 or MDC2025 or Run1B]
11+
python validation/test_fcls.py --tests validation/commands.txt --datasets validation/datasets_XXXXX.txt
1212
```
13+
where ```XXXXXX``` is any of ```MDC2020```, ```MDC2025```, or ```Run1B```.
1314

14-
This script will run all the supported fcl files over relevant datasets. All tests should report ```OK```. If one reports ```FAIL```, you can find the exact command that failed in the ```test_fcls.log``` file in order to debug.
15+
This script will run all the supported fcl files over relevant datasets. The output should be self-explanatory.
16+
17+
The reason for separating the commands from the datasets is so that we can easily change to a different set of datasets with new simulation campaigns, as well as update datasets in currently-supported simulation campaigns.
1518

1619
### Creating a Validation File
1720
Before making any changes, create an EventNtuple file and run the validation script. This creates a new ROOT file with histograms created from the EventNtuple ntuple:

validation/commands.txt

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# Test mock datasets with and without MC
2+
mu2e -c fcl/from_mcs-mockdata.fcl -S {mock} --TFileName nts.ntuple.mock.root -n 100
3+
mu2e -c fcl/from_mcs-mockdata_noMC.fcl -S {mock} --TFileName nts.ntuple.mockNoMC.root -n 100
4+
5+
# Test some individual reconstructed datasets
6+
mu2e -c fcl/from_mcs-primary.fcl -S {primary} --TFileName nts.ntuple.primary.root -n 100
7+
mu2e -c fcl/from_mcs-mixed.fcl -S {mixed} --TFileName nts.ntuple.mixed.root -n 100
8+
mu2e -c fcl/from_mcs-extracted.fcl -S {extracted} --TFileName nts.ntuple.extracted.root -n 100
9+
10+
# Test ceSimReco works
11+
mu2e -c Production/Validation/ceSimReco.fcl -n 10
12+
mu2e -c fcl/from_mcs-ceSimReco.fcl -s mcs.owner.val-ceSimReco.dsconf.seq.art --TFileName nts.ntuple.ceSimReco.root
13+
14+
# Create a dataset with trkhitcalib branch
15+
mu2e -c validation/ceSimReco.fcl -n 100
16+
mu2e -c validation/ceTrig.fcl -s mcs.owner.val-ceSimRecoVal.dsconf.seq.art --TFileName nts.ntuple.ceTrig.roo
17+
mu2e -c fcl/from_mcs-ceSimRecoVal.fcl -s mcs.owner.val-ceTrig.dsconf.seq.art --TFileName nts.ntuple.ceSimRecoVal.root
18+
19+
# Test some options
20+
mu2e -c fcl/from_mcs-mockdata_separateTrkBranches.fcl -S {mock} --TFileName nts.ntuple.mockSepTrkBranches.root -n 100
21+
mu2e -c fcl/from_mcs-mockdata_selectorExample.fcl -S {mock} --TFileName nts.ntuple.mockSelector.root -n 100
22+
mu2e -c fcl/from_mcs-primary_addVDSteps.fcl -S {primary} --TFileName nts.ntuple.primaryVDSteps.root -n 100
23+
24+
# Test digi datasets
25+
mu2e -c fcl/from_dig-mockdata.fcl -S {digi} --TFileName nts.ntuple.dig.root -n 100
26+
27+
# Create validation histogras
28+
mu2e -c fcl/from_mcs-mockdata.fcl -S {mock} --TFileName nts.ntuple.after.root -n 100
29+
root -l -b -q validation/create_val_file_rooutil.C\(\"nts.ntuple.after.root\",\"val.ntuple.after.root\"\)
30+
31+
# CRV KPP (real data)
32+
mu2e -c fcl/from_rec-crv-kpp.fcl -S {crv_kpp} --TFileName nts.ntuple.crv-kpp.root -n 100
33+
mu2e -c fcl/from_rec-crv-kpp_withCrvDigis.fcl -S {crv_kpp} --TFileName nts.ntuple.crv-kpp_withCrvDigis.root -n 100
34+
35+
# DeCalib tests
36+
mu2e -c fcl/from_mcs-DeCalib.fcl -S {mixed} --TFileName nts.ntuple.deCalib.root -n 100

validation/datasets_MDC2020.txt

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Test Datasets for Validating EventNtuple with MDC2020
2+
{mock} mcs.mu2e.ensembleMDS2cMix1BBTriggered.MDC2020ba_best_v1_3.art
3+
{primary} mcs.mu2e.CeEndpointOnSpillTriggered.MDC2020aw_best_v1_3.art
4+
{mixed} mcs.mu2e.CeEndpointMix1BBTriggered.MDC2020aw_best_v1_3.art
5+
{extracted} mcs.mu2e.CosmicCRYExtractedTriggered.MDC2020aw.art
6+
{digi} dig.mu2e.DIOtail95OnSpillTriggered.MDC2020au_perfect_v1_3.art
7+
{crv_kpp} N/A

validation/datasets_MDC2025.txt

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Test Datasets for Validating EventNtuple with MDC2025
2+
{mock} mcs.mu2e.ensembleMDS3aOnSpillTriggered.MDC2025af_best_v1_3.art
3+
{primary} mcs.mu2e.CeEndpointOnSpillTriggered.MDC2025ae_best_v1_3.art
4+
{mixed} mcs.mu2e.CeMLeadingLogMix1BBTriggered.MDC2025af_best_v1_1.art
5+
{extracted} mcs.mu2e.CosmicCRYExtractedTriggered.MDC2025ae_best_v1_3.art
6+
{digi} dig.mu2e.FlatGammaMix1BBTriggered.MDC2025af_best_v1_1.art
7+
{crv_kpp} rec.mu2e.cosmics_crv.crvreco-004-000.art

validation/datasets_Run1B.txt

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Test Datasets for Validating EventNtuple with MDC2025
2+
{mock} N/A
3+
{primary} mcs.mu2e.CeEndpointOnSpillTriggerable-KL.Run1Baf_best_v1_4-000.art
4+
{mixed} mcs.mu2e.CeEndpointMix1BB-KL.Run1Bah_best_v1_4-001.art
5+
{extracted} N/A
6+
{digi} N/A
7+
{crv_kpp} N/A

validation/test_fcls.py

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
#!/usr/bin/env python3
2+
"""
3+
Test script to run EventNtuple fcl files on datasets.
4+
Similar to test_fcls.sh but in Python for better control and error handling.
5+
"""
6+
7+
import os
8+
import subprocess
9+
import sys
10+
import argparse
11+
from pathlib import Path
12+
13+
from enum import Enum
14+
15+
# Define the enum
16+
class Result(Enum):
17+
FAILED = 0
18+
PASSED = 1
19+
SKIPPED = 2
20+
21+
def run(cmd, datasets_dict):
22+
"""
23+
Run a test
24+
25+
Args:
26+
test (str): Command to run
27+
28+
Returns:
29+
bool: True if successful, False otherwise
30+
"""
31+
""" if not os.path.exists(fcl_file):
32+
print(f"Error: FCL file not found: {fcl_file}")
33+
return False
34+
35+
filelist_path = Path("../filelists/") / f"{input_dataset}.list"
36+
if not os.path.exists(filelist_path):
37+
# Get the filelist using metacat
38+
cmd = [f"muse setup ops && metacat query files from mu2e:{input_dataset} | mdh print-url -l tape - > {filelist_path}"]
39+
print(f"Creating filelist for dataset: {input_dataset}")
40+
try:
41+
subprocess.run(cmd, shell=True, check=True)
42+
except subprocess.CalledProcessError as e:
43+
print(f"Error: Failed to create filelist for dataset: {input_dataset}")
44+
return False
45+
46+
# Build mu2e command
47+
cmd = ["mu2e", "-c", fcl_file]
48+
49+
# Add input dataset
50+
cmd.extend(["-S", filelist_path.as_posix()])
51+
52+
# Add output file
53+
cmd.extend(["-o", output_file])
54+
55+
# Add max events if specified
56+
if max_events:
57+
cmd.extend(["-n", str(max_events)]) """
58+
59+
# Check for a filelists/ directory
60+
filelist_dir = Path("../filelists/")
61+
if not os.path.exists(filelist_dir):
62+
print(f"Target directory does not exist: {filelist_dir}")
63+
reply = input("Would you like to create this new directory? (y/n): ").strip().lower()
64+
65+
if reply in ('y', 'yes'):
66+
try:
67+
# parents=True creates intermediate folders if missing (like mkdir -p)
68+
filelist_dir.mkdir(parents=True, exist_ok=True)
69+
print(f"Successfully created directory: {filelist_dir}")
70+
except PermissionError:
71+
print(f"Permission denied: Cannot create directory at {filelist_dir}")
72+
except Exception as e:
73+
print(f"Failed to create directory. Error: {e}")
74+
else:
75+
print("Directory creation canceled by user.")
76+
return Result.FAILED
77+
78+
# Search for dataset name and turn into filelist
79+
input_dataset = cmd.split('-S ')
80+
if len(input_dataset)>1:
81+
input_dataset = input_dataset[1].split()[0]
82+
if input_dataset in datasets_dict:
83+
old_input_dataset = input_dataset
84+
input_dataset = datasets_dict[input_dataset]
85+
if (input_dataset == "N/A"):
86+
print(f"Warning: Dataset name for '{old_input_dataset}' is marked as N/A in datasets file. Skipping test...")
87+
return Result.SKIPPED # returning SKIPPED to avoid counting as a failure, but skipping the test since dataset is not available
88+
cmd = cmd.replace(old_input_dataset, input_dataset)
89+
else:
90+
print(f"Warning: Dataset name '{input_dataset}' not found in datasets file. Using as-is.")
91+
92+
# Check for filelist
93+
filelist_path = Path("../filelists/") / f"{input_dataset}.list"
94+
if not os.path.exists(filelist_path):
95+
# Get the filelist using metacat
96+
subcmd = [f"muse setup ops && metacat query files from mu2e:{input_dataset} | mdh print-url -l tape - > {filelist_path}"]
97+
print(f"Creating filelist for dataset: {input_dataset}")
98+
try:
99+
subprocess.run(subcmd, shell=True, check=True)
100+
except subprocess.CalledProcessError as e:
101+
print(f"Error: Failed to create filelist for dataset: {input_dataset}")
102+
return FAILED
103+
cmd = cmd.replace(input_dataset, str(filelist_path))
104+
105+
print(f"Running: {cmd}")
106+
107+
try:
108+
result = subprocess.run(cmd.split(), check=True, capture_output=True, text=True)
109+
print(f"✓ Successfully processed: {cmd}")
110+
#print(f" Output: {output_file}")
111+
return Result.PASSED
112+
except subprocess.CalledProcessError as e:
113+
print(f"✗ Error processing {cmd}")
114+
print(f" Stdout: {e.stdout}")
115+
print(f" Stderr: {e.stderr}")
116+
return Result.FAILED
117+
118+
except FileNotFoundError as e:
119+
print(f"{e}")
120+
return Result.FAILED
121+
122+
123+
def main():
124+
parser = argparse.ArgumentParser(
125+
description="Run EventNtuple fcl files on datasets"
126+
)
127+
parser.add_argument(
128+
"--tests",
129+
help="Path to file containing list of tests to run (one command per line)",
130+
required=True
131+
)
132+
parser.add_argument(
133+
"--datasets",
134+
help="Path to file containing list of dataset names",
135+
required=True
136+
)
137+
138+
args = parser.parse_args()
139+
140+
# Check if batch processing requested
141+
datasets = { }
142+
if not os.path.exists(args.datasets):
143+
print(f"Error: Datasets file not found: {args.datasets}")
144+
sys.exit(1)
145+
with open(args.datasets, 'r') as f:
146+
for line in f:
147+
line = line.strip()
148+
if line and not line.startswith('#'):
149+
parts = line.split()
150+
if len(parts) == 2:
151+
datasets[parts[0]] = parts[1]
152+
else:
153+
print(f"Warning: Invalid line in datasets file (skipping): {line}")
154+
155+
if not os.path.exists(args.tests):
156+
print(f"Error: Tests file not found: {args.tests}")
157+
sys.exit(1)
158+
159+
results = []
160+
with open(args.tests, 'r') as f:
161+
tests = [line.strip() for line in f if line.strip() and not line.strip().startswith('#')]
162+
163+
print(f"Processing {len(tests)} tests...")
164+
165+
for i, (test) in enumerate(tests, 1):
166+
print(f"\n[{i}/{len(tests)}] Processing: {test}")
167+
result = run(test, datasets)
168+
results.append((test, result))
169+
170+
# Print summary
171+
print("\n" + "="*60)
172+
print("TEST PROCESSING SUMMARY")
173+
print("="*60)
174+
passed = sum(1 for _, result in results if result == Result.PASSED)
175+
failed = sum(1 for _, result in results if result == Result.FAILED)
176+
skipped = sum(1 for _, result in results if result == Result.SKIPPED)
177+
print(f"Total: {len(results)}, Passed: {passed}, Failed: {failed}, Skipped: {skipped}")
178+
179+
if passed < len(results):
180+
print("\nFailed datasets:")
181+
for command, result in results:
182+
if result == Result.FAILED:
183+
print(f" - {command}")
184+
sys.exit(1)
185+
186+
187+
if __name__ == "__main__":
188+
main()

0 commit comments

Comments
 (0)