Skip to content

Commit 1e6ca16

Browse files
joshua-benchlingjoshua-benchling
committed
add unread data
1 parent de8bac0 commit 1e6ca16

4 files changed

Lines changed: 260 additions & 13 deletions

File tree

‎src/allotropy/parsers/bd_biosciences_facsdiva/bd_biosciences_facsdiva_structure.py‎

Lines changed: 96 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,19 @@
7878
]
7979

8080

81+
def _filter_unread_data(unread_data: dict[str, str | None]) -> dict[str, str | None]:
82+
"""Filter out empty, null, or whitespace-only values from unread data."""
83+
return {
84+
key: value
85+
for key, value in unread_data.items()
86+
if not (
87+
(value == "")
88+
or (value is None)
89+
or (isinstance(value, str) and value.strip() == "")
90+
)
91+
}
92+
93+
8194
class RegionType(Enum):
8295
POLYGON = "POLYGON"
8396
RECTANGLE = "RECTANGLE"
@@ -87,14 +100,27 @@ class RegionType(Enum):
87100
def create_metadata(root_element: StrictXmlElement, file_path: str) -> Metadata:
88101
equipment_serial_number = None
89102
model_number = None
103+
all_custom_info = {}
90104

91105
tube = root_element.recursive_find_or_none(["experiment", "specimen", "tube"])
92106
if tube:
93107
equipment_serial_number = tube.find_or_none("data_instrument_serial_number")
94108
model_number = tube.find_or_none("data_instrument_name")
109+
tube.mark_read("name")
95110

96111
software_version = root_element.get_attr_or_none("version")
97112

113+
# Collect unread data from root element
114+
root_unread = root_element.get_unread(
115+
skip={
116+
"version",
117+
"release_version",
118+
}
119+
)
120+
if root_unread:
121+
filtered_root_unread = _filter_unread_data(root_unread)
122+
all_custom_info.update(filtered_root_unread)
123+
98124
return Metadata(
99125
file_name=Path(file_path).name,
100126
unc_path=file_path,
@@ -107,6 +133,7 @@ def create_metadata(root_element: StrictXmlElement, file_path: str) -> Metadata:
107133
software_name=constants.SOFTWARE_NAME,
108134
software_version=software_version,
109135
asm_file_identifier=Path(file_path).with_suffix(".json").name,
136+
custom_info=all_custom_info if all_custom_info else None,
110137
)
111138

112139

@@ -134,6 +161,7 @@ def _extract_statistics_from_calculations(
134161
for calc_schedule in statistics_element.findall("calculation_schedule"):
135162
gate = calc_schedule.get_attr_or_none("gate")
136163
if gate != gate_name:
164+
calc_schedule.get_unread()
137165
continue
138166

139167
parameter = calc_schedule.get_attr_or_none("parameter")
@@ -208,31 +236,43 @@ def _create_data_regions(tube: StrictXmlElement) -> list[DataRegion]:
208236
# First pass: collect gate information
209237
for gate in gates_element.findall("gate"):
210238
# Skip All Events full name (no region)
211-
if gate.get_attr_or_none("type") == "EventSource_Classifier":
239+
gate_type = gate.get_attr_or_none("type")
240+
fullname = gate.get_attr_or_none("fullname")
241+
242+
if gate_type == "EventSource_Classifier":
212243
continue
213244

214245
region = gate.find_or_none("region")
215246
if not region:
216247
continue
217248

218-
fullname = gate.get_attr_or_none("fullname")
219249
if not fullname:
220250
continue
221251

222252
region_name = region.get_attr_or_none("name")
223253
if region_name:
224254
fullname_to_region[fullname] = region_name
225255

256+
region.mark_read(
257+
{
258+
"type",
259+
"xparm",
260+
"yparm",
261+
}
262+
)
263+
226264
# Second pass: create data regions
227265
for gate in gates_element.findall("gate"):
228-
if gate.get_attr_or_none("type") == "EventSource_Classifier":
266+
gate_type = gate.get_attr_or_none("type")
267+
fullname = gate.get_attr_or_none("fullname")
268+
269+
if gate_type == "EventSource_Classifier":
229270
continue
230271

231272
region = gate.find_or_none("region")
232273
if not region:
233274
continue
234275

235-
fullname = gate.get_attr_or_none("fullname")
236276
if not fullname:
237277
continue
238278

@@ -348,6 +388,20 @@ def _build_population_tree(full_name: str) -> Population | None:
348388
region = _gate.find_or_none("region")
349389
region_id = region.get_attr_or_none("name") if region else None
350390

391+
if region:
392+
region.mark_read(
393+
{
394+
"type",
395+
"xparm",
396+
"yparm",
397+
}
398+
)
399+
_gate.mark_read(
400+
{
401+
"type",
402+
}
403+
)
404+
351405
sub_populations = []
352406
if full_name in gate_to_children:
353407
for child_fullname in gate_to_children[full_name]:
@@ -378,11 +432,16 @@ def _build_population_tree(full_name: str) -> Population | None:
378432
)
379433

380434
# Find the root population (All Events) and build the tree
381-
root_gates = [
382-
gate
383-
for gate in gates_element.findall("gate")
384-
if gate.get_attr_or_none("type") == "EventSource_Classifier"
385-
]
435+
root_gates = []
436+
for gate in gates_element.findall("gate"):
437+
gate_type = gate.get_attr_or_none("type")
438+
if gate_type == "EventSource_Classifier":
439+
root_gates.append(gate)
440+
gate.mark_read(
441+
{
442+
"fullname",
443+
}
444+
)
386445

387446
if not root_gates:
388447
return []
@@ -416,6 +475,8 @@ def _create_compensation_matrix_groups(
416475
name = param.get_attr_or_none("name")
417476
if name and name not in EXCLUDED_CHANNELS:
418477
parameters.append(param)
478+
else:
479+
param.get_unread()
419480

420481
if not parameters:
421482
return None
@@ -450,13 +511,22 @@ def _create_compensation_matrix_groups(
450511
compensation_value=coef_value,
451512
)
452513
)
514+
param.mark_read(
515+
"type",
516+
)
453517

454518
result.append(
455519
CompensationMatrixGroup(
456520
dimension_identifier=dimension_id,
457521
compensation_matrices=matrices if matrices else None,
458522
)
459523
)
524+
instrument_settings.mark_read(
525+
{
526+
"name",
527+
"template",
528+
}
529+
)
460530

461531
return result if result else None
462532

@@ -477,10 +547,24 @@ def _process_tube(
477547

478548
processed_data_id = random_uuid_str()
479549

550+
# Extract custom info from keywords
551+
custom_info = {}
552+
keywords_element = tube.find_or_none("keywords")
553+
if keywords_element:
554+
for keyword in keywords_element.findall("keyword"):
555+
keyword_name = keyword.get_attr_or_none("name")
556+
keyword.mark_read("type")
557+
558+
if keyword_name:
559+
value_element = keyword.find_or_none("value")
560+
if value_element:
561+
keyword_value = value_element.get_text_or_none()
562+
if keyword_value:
563+
custom_info[keyword_name] = keyword_value
564+
480565
# Create populations and data regions
481566
populations = _create_populations(tube)
482567
data_regions = _create_data_regions(tube)
483-
484568
measurement = Measurement(
485569
measurement_identifier=random_uuid_str(),
486570
sample_identifier=tube_name,
@@ -490,6 +574,7 @@ def _process_tube(
490574
processed_data_identifier=processed_data_id,
491575
populations=populations,
492576
data_regions=data_regions,
577+
custom_info=custom_info if custom_info else None,
493578
)
494579

495580
return [measurement]
@@ -511,6 +596,7 @@ def create_measurement_groups(root_element: StrictXmlElement) -> list[Measuremen
511596
if not experiment:
512597
msg = "No experiment found in the XML file."
513598
raise AllotropeParsingError(msg)
599+
514600
measurement_time = experiment.recursive_find_or_none(["specimen", "tube", "date"])
515601
experiment_identifier = experiment.get_attr_or_none("name")
516602
analyst = experiment.recursive_find_or_none(["owner_name"])

‎src/allotropy/parsers/utils/strict_xml_element.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -373,14 +373,21 @@ def recursive_find_or_none(self, names: list[str]) -> StrictXmlElement | None:
373373
return self
374374
name, *sub_names = names
375375
if element := self.find_or_none(name):
376+
# Mark all attributes as read for intermediate elements
377+
# (not for the final element since that's the one we want to return)
378+
if sub_names:
379+
element.get_unread()
376380
return element.recursive_find_or_none(sub_names)
377381
return None
378382

379383
def recursive_find(self, names: list[str]) -> StrictXmlElement:
380384
if len(names) == 0:
381385
return self
382386
name, *sub_names = names
383-
return self.find(name).recursive_find(sub_names)
387+
element = self.find(name)
388+
if sub_names:
389+
element.get_unread()
390+
return element.recursive_find(sub_names)
384391

385392
def findall(self, name: str) -> list[StrictXmlElement]:
386393
self.read_keys.add(f"element:{name}")

‎tests/parsers/bd_biosciences_facsdiva/testdata/facsdiva_example_1.json‎

Lines changed: 71 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -569,6 +569,16 @@
569569
]
570570
}
571571
]
572+
},
573+
"custom information document": {
574+
"$OP": "CRCR",
575+
"GUID": "e2bfae9f-1b2e-435c-ab3a-86f9a5b98665",
576+
"CST SETUP STATUS": "SUCCESS",
577+
"CST BEADS LOT ID": "49706",
578+
"CYTOMETER CONFIG NAME": "2-Blue 3-Violet 3-Red 5-YG (PRBB)",
579+
"CYTOMETER CONFIG CREATE DATE": "04/25/2012 05:48:01 AM",
580+
"CST SETUP DATE": "11/02/2016 02:15:19 PM",
581+
"CST BASELINE DATE": "03/30/2016 03:58:09 PM"
572582
}
573583
}
574584
],
@@ -2423,6 +2433,16 @@
24232433
]
24242434
}
24252435
]
2436+
},
2437+
"custom information document": {
2438+
"$OP": "CRCR",
2439+
"GUID": "81bb5331-69e1-46dd-8fe9-015bdcf17d36",
2440+
"CST SETUP STATUS": "SUCCESS",
2441+
"CST BEADS LOT ID": "49706",
2442+
"CYTOMETER CONFIG NAME": "2-Blue 3-Violet 3-Red 5-YG (PRBB)",
2443+
"CYTOMETER CONFIG CREATE DATE": "04/25/2012 05:48:01 AM",
2444+
"CST SETUP DATE": "11/02/2016 02:15:19 PM",
2445+
"CST BASELINE DATE": "03/30/2016 03:58:09 PM"
24262446
}
24272447
}
24282448
],
@@ -4260,6 +4280,16 @@
42604280
]
42614281
}
42624282
]
4283+
},
4284+
"custom information document": {
4285+
"$OP": "CRCR",
4286+
"GUID": "e4000a05-5758-483b-903e-31cda86e9340",
4287+
"CST SETUP STATUS": "SUCCESS",
4288+
"CST BEADS LOT ID": "49706",
4289+
"CYTOMETER CONFIG NAME": "2-Blue 3-Violet 3-Red 5-YG (PRBB)",
4290+
"CYTOMETER CONFIG CREATE DATE": "04/25/2012 05:48:01 AM",
4291+
"CST SETUP DATE": "11/02/2016 02:15:19 PM",
4292+
"CST BASELINE DATE": "03/30/2016 03:58:09 PM"
42634293
}
42644294
}
42654295
],
@@ -6081,6 +6111,16 @@
60816111
]
60826112
}
60836113
]
6114+
},
6115+
"custom information document": {
6116+
"$OP": "CRCR",
6117+
"GUID": "f2d693fc-6805-45fb-9b13-9770ef3bc00e",
6118+
"CST SETUP STATUS": "SUCCESS",
6119+
"CST BEADS LOT ID": "49706",
6120+
"CYTOMETER CONFIG NAME": "2-Blue 3-Violet 3-Red 5-YG (PRBB)",
6121+
"CYTOMETER CONFIG CREATE DATE": "04/25/2012 05:48:01 AM",
6122+
"CST SETUP DATE": "11/02/2016 02:15:19 PM",
6123+
"CST BASELINE DATE": "03/30/2016 03:58:09 PM"
60846124
}
60856125
}
60866126
],
@@ -7910,6 +7950,16 @@
79107950
]
79117951
}
79127952
]
7953+
},
7954+
"custom information document": {
7955+
"$OP": "CRCR",
7956+
"GUID": "85a760c3-b155-4fef-8679-a908509d2fc4",
7957+
"CST SETUP STATUS": "SUCCESS",
7958+
"CST BEADS LOT ID": "49706",
7959+
"CYTOMETER CONFIG NAME": "2-Blue 3-Violet 3-Red 5-YG (PRBB)",
7960+
"CYTOMETER CONFIG CREATE DATE": "04/25/2012 05:48:01 AM",
7961+
"CST SETUP DATE": "11/02/2016 02:15:19 PM",
7962+
"CST BASELINE DATE": "03/30/2016 03:58:09 PM"
79137963
}
79147964
}
79157965
],
@@ -9520,6 +9570,16 @@
95209570
]
95219571
}
95229572
]
9573+
},
9574+
"custom information document": {
9575+
"$OP": "CRCR",
9576+
"GUID": "c0960d94-94f9-4244-8712-999b80d5c87b",
9577+
"CST SETUP STATUS": "SUCCESS",
9578+
"CST BEADS LOT ID": "49706",
9579+
"CYTOMETER CONFIG NAME": "2-Blue 3-Violet 3-Red 5-YG (PRBB)",
9580+
"CYTOMETER CONFIG CREATE DATE": "04/25/2012 05:48:01 AM",
9581+
"CST SETUP DATE": "11/02/2016 02:15:19 PM",
9582+
"CST BASELINE DATE": "03/30/2016 03:58:09 PM"
95239583
}
95249584
}
95259585
],
@@ -11579,6 +11639,16 @@
1157911639
]
1158011640
}
1158111641
]
11642+
},
11643+
"custom information document": {
11644+
"$OP": "CRCR",
11645+
"GUID": "c1fa81e6-3204-4681-9bf5-4c499723f67b",
11646+
"CST SETUP STATUS": "SUCCESS",
11647+
"CST BEADS LOT ID": "49706",
11648+
"CYTOMETER CONFIG NAME": "2-Blue 3-Violet 3-Red 5-YG (PRBB)",
11649+
"CYTOMETER CONFIG CREATE DATE": "04/25/2012 05:48:01 AM",
11650+
"CST SETUP DATE": "11/02/2016 02:15:19 PM",
11651+
"CST BASELINE DATE": "03/30/2016 03:58:09 PM"
1158211652
}
1158311653
}
1158411654
],
@@ -12873,7 +12943,7 @@
1287312943
"UNC path": "tests/parsers/bd_biosciences_facsdiva/testdata/facsdiva_example_1.xml",
1287412944
"file name": "facsdiva_example_1.xml",
1287512945
"ASM converter name": "allotropy_bd_biosciences_facsdiva",
12876-
"ASM converter version": "0.1.82",
12946+
"ASM converter version": "0.1.93",
1287712947
"software name": "BD FACSDiva",
1287812948
"software version": "Version 6.2"
1287912949
},

0 commit comments

Comments
 (0)