Skip to content

Commit 3c5d33d

Browse files
fix: Resolve duplicate columns in Stunner DLS output (#1213)
## Summary - Stunner parser now recognizes alternate DLS column names used by newer instrument software (e.g., `Z-Avg Dia. (nm)` vs `Z Ave. Dia (nm)`, `Rayleigh ratio R (1/km)` vs `(cm^-1)`, `Viscosity at 25°C` vs `20°C`) - Previously unmapped DLS fields (`Number of Peaks`, `Number of Angles`, `Angles Measured`, `Intercept`) are now structured as calculated data documents instead of leaking to custom info - `number_of_averages` and `Number of acquisitions used` now try both `(total)` and non-`(total)` column name variants - `B/S/F/R` (sample role type) added to sample custom info - Removed 52 N/A placeholder columns from `data system document.custom information document` that were causing duplicate columns in connector CSV output ## Context Follow-up to #1205. Customer (ProFound) reported 17 remaining duplicate columns after the PkN fix. Root cause was twofold: 1. `CALCULATED_DATA_LOOKUP` did not recognize newer column naming conventions, so DLS fields fell through `get_unread()` into `measurement_custom_info` 2. When the parser reads metadata from the first data row (no-header mode), unread DLS/peak columns leaked into `data_system_document.custom_information_document` as N/A placeholders — the CSV conversion lib then produced one column from the placeholder and one from the actual structured data ## Test plan - [x] All 34 Stunner/Lunatic tests pass (33 existing + 1 new) - [x] All 12 Lunatic discovery tests pass - [x] New anonymized test file (example03) exercises the new PkN column format with blank, no-peaks, single-peak, and multi-peak measurements - [x] Verified customer file: 0/85 measurements have `measurement_custom_info` content - [x] Verified `data_system_document.custom_information_document` is empty (was 52 N/A keys) - [x] Lint clean (black, ruff, mypy) 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 10b2e3d commit 3c5d33d

7 files changed

Lines changed: 1572 additions & 86 deletions

File tree

src/allotropy/parsers/unchained_labs_lunatic_stunner/constants.py

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
"The plate data does not contain absorbance measurement for {}."
2424
)
2525

26-
CALCULATED_DATA_LOOKUP = {
26+
CALCULATED_DATA_LOOKUP: dict[str, list[dict[str, str | list[str]]]] = {
2727
"a260": [
2828
{
2929
"column": "a260 concentration (ng/ul)",
@@ -102,6 +102,7 @@
102102
},
103103
{
104104
"column": "z ave. dia (nm)",
105+
"alt_columns": ["z-avg dia. (nm)"],
105106
"name": "Z Average Diameter",
106107
"feature": "dynamic light scattering",
107108
"unit": "nm",
@@ -114,6 +115,7 @@
114115
},
115116
{
116117
"column": "sd dia (nm)",
118+
"alt_columns": ["sd dia. (nm)"],
117119
"name": "Diameter Standard Deviation",
118120
"feature": "dynamic light scattering",
119121
"unit": "nm",
@@ -168,6 +170,7 @@
168170
},
169171
{
170172
"column": "rayleigh ratio r (cm^-1)",
173+
"alt_columns": ["rayleigh ratio r (1/km)"],
171174
"name": "Rayleigh ratio R",
172175
"feature": "dynamic light scattering",
173176
"unit": UNITLESS,
@@ -192,6 +195,7 @@
192195
},
193196
{
194197
"column": "viscosity at 20°c (cp)",
198+
"alt_columns": ["viscosity at 25°c (cp)"],
195199
"name": "Viscosity at 20°C (cP)",
196200
"feature": "dynamic light scattering",
197201
"unit": UNITLESS,
@@ -204,6 +208,7 @@
204208
},
205209
{
206210
"column": "ri at 20°c",
211+
"alt_columns": ["ri at 25°c"],
207212
"name": "RI at 20°C",
208213
"feature": "dynamic light scattering",
209214
"unit": UNITLESS,
@@ -214,5 +219,29 @@
214219
"feature": "dynamic light scattering",
215220
"unit": "nm",
216221
},
222+
{
223+
"column": "number of peaks",
224+
"name": "Number of Peaks",
225+
"feature": "dynamic light scattering",
226+
"unit": UNITLESS,
227+
},
228+
{
229+
"column": "number of angles",
230+
"name": "Number of Angles",
231+
"feature": "dynamic light scattering",
232+
"unit": UNITLESS,
233+
},
234+
{
235+
"column": "angles measured (°)",
236+
"name": "Angles Measured",
237+
"feature": "dynamic light scattering",
238+
"unit": UNITLESS,
239+
},
240+
{
241+
"column": "intercept",
242+
"name": "Intercept",
243+
"feature": "dynamic light scattering",
244+
"unit": UNITLESS,
245+
},
217246
],
218247
}

src/allotropy/parsers/unchained_labs_lunatic_stunner/unchained_labs_lunatic_stunner_calcdocs.py

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,34 @@ def create_calculated_data(
262262
view_data=detection_type_view,
263263
source_configs=(dynamic_light_scattering_conf,),
264264
),
265+
CalculatedDataConfig(
266+
name="Number of Peaks",
267+
value="number of peaks",
268+
unit=UNITLESS,
269+
view_data=detection_type_view,
270+
source_configs=(dynamic_light_scattering_conf,),
271+
),
272+
CalculatedDataConfig(
273+
name="Number of Angles",
274+
value="number of angles",
275+
unit=UNITLESS,
276+
view_data=detection_type_view,
277+
source_configs=(dynamic_light_scattering_conf,),
278+
),
279+
CalculatedDataConfig(
280+
name="Angles Measured",
281+
value="angles measured (°)",
282+
unit=UNITLESS,
283+
view_data=detection_type_view,
284+
source_configs=(dynamic_light_scattering_conf,),
285+
),
286+
CalculatedDataConfig(
287+
name="Intercept",
288+
value="intercept",
289+
unit=UNITLESS,
290+
view_data=detection_type_view,
291+
source_configs=(dynamic_light_scattering_conf,),
292+
),
265293
]
266294
)
267295

src/allotropy/parsers/unchained_labs_lunatic_stunner/unchained_labs_lunatic_stunner_structure.py

Lines changed: 57 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -244,7 +244,9 @@ def _create_measurement(
244244
well_plate_identifier=well_plate_data.get(str, "plate id"),
245245
batch_identifier=well_plate_data.get(str, "sample group"),
246246
firmware_version=header.get(str, "client version"),
247-
number_of_averages=well_plate_data.get(float, "number of acquisitions"),
247+
number_of_averages=well_plate_data.get(
248+
float, ("number of acquisitions", "number of acquisitions (total)")
249+
),
248250
integration_time=well_plate_data.get(float, "acquisition time (s)"),
249251
compartment_temperature=well_plate_data.get(float, "temperature (°c)"),
250252
sample_custom_info={
@@ -259,13 +261,18 @@ def _create_measurement(
259261
"molecular weight (kda)": well_plate_data.get(
260262
float, "molecular weight (kda)"
261263
),
264+
"sample role type": well_plate_data.get(str, "b/s/f/r"),
262265
},
263266
device_control_custom_info={
264267
"path length mode": well_plate_data.get(str, "path length mode"),
265268
"pump": well_plate_data.get(str, "pump"),
266269
"column": header.get(str, "column") or well_plate_data.get(str, "column"),
267270
"Number of acquisitions used": well_plate_data.get(
268-
str, "number of acquisitions used"
271+
str,
272+
(
273+
"number of acquisitions used",
274+
"number of acquisitions used (total)",
275+
),
269276
),
270277
"Acquisition filtering": well_plate_data.get(str, "acquisition filtering"),
271278
},
@@ -434,20 +441,27 @@ def _get_calculated_data_values(
434441
error_documents: list[ErrorDocument] = []
435442
for wavelength_column in wavelength_columns:
436443
for item in CALCULATED_DATA_LOOKUP.get(wavelength_column, []):
437-
value, is_na = _get_calculated_value_and_is_na(
438-
well_plate_data, item["column"]
439-
)
440-
# Only create error docs when the cell is the literal "N/A"
444+
canonical_column = str(item["column"])
445+
alt_columns = item.get("alt_columns", [])
446+
columns_to_try = [
447+
canonical_column,
448+
*(alt_columns if isinstance(alt_columns, list) else []),
449+
]
450+
value: float | None = None
451+
is_na = False
452+
for col in columns_to_try:
453+
value, is_na = _get_calculated_value_and_is_na(well_plate_data, col)
454+
if value is not None:
455+
break
441456
if is_na:
442457
error_documents.append(
443458
ErrorDocument(
444459
error=NOT_APPLICABLE,
445-
error_feature=item["name"],
460+
error_feature=str(item["name"]),
446461
)
447462
)
448-
# Skip missing cells entirely; include numeric values
449463
if value is not None:
450-
calculated_data_values[item["column"]] = value
464+
calculated_data_values[canonical_column] = value
451465
return calculated_data_values, error_documents
452466

453467

@@ -506,6 +520,40 @@ def create_metadata(header: SeriesData, file_path: str) -> Metadata:
506520
"background wvl. (nm)",
507521
"plate id",
508522
"plate position",
523+
"analyte",
524+
"buffer",
525+
"b/s/f/r",
526+
"acquisition filtering",
527+
"number of acquisitions",
528+
"number of acquisitions used",
529+
"number of acquisitions (total)",
530+
"number of acquisitions used (total)",
531+
"acquisition time (s)",
532+
"temperature (°c)",
533+
"molecular weight (kda)",
534+
# DLS calculated data columns
535+
"intercept",
536+
"number of peaks",
537+
"number of angles",
538+
r"^angles measured.*",
539+
r"^z.avg dia.*",
540+
"pdi",
541+
r"^sd dia.*",
542+
r"^diffusion coefficient.*",
543+
r"^rayleigh ratio r.*",
544+
r"^optical contrast constant.*",
545+
r"^viscosity at.*",
546+
r"^ri at.*",
547+
r"^derived intensity.*",
548+
r"^diameter @.*",
549+
r"^kc/r.*",
550+
r"^kd .*",
551+
r"^b22.*",
552+
# Peak columns (pk1, pk2, ..., pkoi, peak 1, peak 2, ...)
553+
r"^pk\d+.*",
554+
r"^pkoi.*",
555+
r"^peak \d+.*",
556+
r"^peak of interest.*",
509557
# Strings to skip since these are already captured as measurements/calculated data
510558
# Skip absorbance spectrum measurements with a### (10mm)
511559
r"^a\d{3} \(10mm\)$",

0 commit comments

Comments
 (0)