Skip to content

Commit 1b963ec

Browse files
nathan-stenderclaude
nathan-stender
and
claude
authored
fix: Handle dilutions, replicate measurements, and mmol/l unit in Roche Cedex Bio HT (#1244)
## Summary A V6/V7 Roche Cedex Bio HT export failed to parse. Three distinct issues surfaced, each fixed here: 1. **Dilutions treated as duplicates.** The same analyte can be measured at multiple dilutions within one sample (e.g. IgG human undiluted `3.933` and at `1/5` → `6.174`, a minute apart). The dilution factor lived in the previously-discarded `col8`. It's now named `dilution factor`, threaded through `RawMeasurement`, and included in the analyte grouping key so the two measurements no longer collide. The factor is also preserved in the analyte custom information document. 2. **Genuine replicate measurements raised an error.** The same analyte can be re-measured (a second round) inside the 1-hour grouping window (e.g. glucose `15.17` then `15.43` ~43 min later). Instead of raising `Duplicate measurement`, we now split the later measurement into a new group keyed by its own time — realizing the improvement the previous code's comment explicitly anticipated. 3. **Lowercase `mmol/l` unit.** A handful of rows report the unit as `mmol/l`; it's now normalized to `mmol/L` via `UNIT_CONVERSIONS`. ## Changes - `constants.py`: rename `col8` → `dilution factor` in `DATA_HEADER_V6_V7`; add `mmol/l` → `mmol/L` to `UNIT_CONVERSIONS`. - `roche_cedex_bioht_structure.py`: add `dilution_factor` to `RawMeasurement`, include it in the grouping key and custom info; split on duplicate instead of raising (removes now-unused `AllotropyParserError`). - Updated affected unit tests; added `test_create_measurements_same_analyte_different_dilution` and reworked the duplicate test to assert the split behavior. - Added trimmed regression file `roche_cedex_bioht_v7_dilution_and_replicates.txt` (+ expected JSON) covering all three cases. ## Testing - Full 2.4MB source export now parses (5864 groups). - All 22 `roche_cedex_bioht` tests pass; `hatch run lint` (ruff, black, mypy) clean. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
1 parent e705eb6 commit 1b963ec

6 files changed

Lines changed: 495 additions & 33 deletions

File tree

‎src/allotropy/parsers/roche_cedex_bioht/constants.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
"sample role type",
3030
"col6",
3131
"analyte name",
32-
"col8",
32+
"dilution factor",
3333
"concentration unit",
3434
"flag",
3535
"concentration value",

‎src/allotropy/parsers/roche_cedex_bioht/roche_cedex_bioht_structure.py‎

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
MeasurementGroup,
1717
Metadata,
1818
)
19-
from allotropy.exceptions import AllotropyParserError
2019
from allotropy.parsers.constants import NOT_APPLICABLE
2120
from allotropy.parsers.roche_cedex_bioht.constants import (
2221
BELOW_TEST_RANGE,
@@ -32,6 +31,8 @@
3231
UNIT_CONVERSIONS: dict[str, tuple[str, float]] = {
3332
"mg/L": ("g/L", 0.001),
3433
"mM": ("mmol/L", 1.0),
34+
# Some files report a lowercase-liter variant that means the same thing.
35+
"mmol/l": ("mmol/L", 1.0),
3536
}
3637

3738

@@ -62,6 +63,7 @@ class RawMeasurement:
6263
concentration_value: JsonFloat
6364
unit: str
6465
analyte_code: str
66+
dilution_factor: str | None = None
6567
error: str | None = None
6668
custom_info: dict[str, Any] | None = None
6769

@@ -70,6 +72,11 @@ def create(data: SeriesData) -> RawMeasurement:
7072
flag = data.get(str, "flag", "").strip()
7173
error = FLAG_TO_ERROR[flag] if flag else None
7274

75+
# V6/V7 files report a dilution factor (e.g. "1/5") for measurements that were
76+
# diluted before reading. The same analyte may be measured at multiple dilutions
77+
# in one sample, so it distinguishes otherwise-identical measurements.
78+
dilution_factor = data.get(str, "dilution factor", "").strip() or None
79+
7380
concentration_value = data.get(float, "concentration value", NaN)
7481

7582
value, unit = RawMeasurement._get_value_and_unit(
@@ -87,6 +94,7 @@ def create(data: SeriesData) -> RawMeasurement:
8794
value,
8895
unit,
8996
data[str, "analyte code"],
97+
dilution_factor,
9098
error,
9199
custom_info=dict(
92100
sorted(
@@ -96,6 +104,7 @@ def create(data: SeriesData) -> RawMeasurement:
96104
"analyte code": data.get(str, "analyte code"),
97105
"record type": data.get(str, "row type"),
98106
"flag": error or None,
107+
"dilution factor": dilution_factor,
99108
},
100109
**data.get_unread(
101110
skip={
@@ -142,7 +151,9 @@ def create_measurements(data: pd.DataFrame) -> dict[str, dict[str, RawMeasuremen
142151
current_measurement_time = measurements[0].measurement_time
143152
previous_measurement_time = current_measurement_time
144153
for analyte in measurements:
145-
analyte_id = f"{analyte.name}_{analyte.analyte_code}"
154+
# Include dilution factor in the id so the same analyte measured at different
155+
# dilutions within a group is not treated as a duplicate measurement.
156+
analyte_id = f"{analyte.name}_{analyte.analyte_code}_{analyte.dilution_factor}"
146157
time_diff = parser.parse(analyte.measurement_time) - parser.parse(
147158
previous_measurement_time
148159
)
@@ -151,16 +162,16 @@ def create_measurements(data: pd.DataFrame) -> dict[str, dict[str, RawMeasuremen
151162
if analyte_id in groups[current_measurement_time]:
152163
if analyte.concentration_value is NaN:
153164
continue
154-
# NOTE: if this fails, it's probably because MAX_MEASUREMENT_TIME_GROUP_DIFFERENCE is too big
155-
# and we're erroneously grouping two groups of measurements into one.
156-
# We could potentially make this more robust by just splitting into a new group if a duplicate
157-
# measurement is found, but cross that bridge when we come to it.
158165
if (
159166
groups[current_measurement_time][analyte_id].concentration_value
160167
is not NaN
161168
):
162-
msg = f"Duplicate measurement for {analyte.analyte_code} in the same measurement group. {analyte.concentration_value} vs {groups[current_measurement_time][analyte_id].concentration_value}"
163-
raise AllotropyParserError(msg)
169+
# A real duplicate measurement of the same analyte (same dilution) within
170+
# the time window means we've observed a second round of measurements for
171+
# the sample. Start a new group so both rounds are preserved rather than
172+
# colliding. (This can also happen if MAX_MEASUREMENT_TIME_GROUP_DIFFERENCE
173+
# is too large and is erroneously merging distinct rounds.)
174+
current_measurement_time = analyte.measurement_time
164175
groups[current_measurement_time][analyte_id] = analyte
165176
previous_measurement_time = analyte.measurement_time
166177

‎tests/parsers/roche_cedex_bioht/roche_cedex_bioht_data.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ def get_reader_samples() -> pd.DataFrame:
9898
"sample role type",
9999
"col6",
100100
"analyte name",
101-
"col8",
101+
"dilution factor",
102102
"concentration unit",
103103
"flag",
104104
"concentration value",

‎tests/parsers/roche_cedex_bioht/roche_cedex_bioht_structure_test.py‎

Lines changed: 110 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import pandas as pd
22
import pytest
33

4-
from allotropy.exceptions import AllotropeConversionError, AllotropyParserError
4+
from allotropy.exceptions import AllotropeConversionError
55
from allotropy.parsers.roche_cedex_bioht.roche_cedex_bioht_structure import (
66
create_measurements,
77
RawMeasurement,
@@ -106,32 +106,50 @@ def test_create_measurements() -> None:
106106

107107
assert measurements == {
108108
"2021-05-20T16:55:51+00:00": {
109-
"lactate_LAC2B": RawMeasurement(
109+
"lactate_LAC2B_None": RawMeasurement(
110110
"lactate",
111111
"2021-05-20T16:55:51+00:00",
112112
2.45,
113113
"g/L",
114114
"LAC2B",
115115
None,
116-
{"analyte code": "LAC2B", "record type": None, "flag": None},
116+
None,
117+
{
118+
"analyte code": "LAC2B",
119+
"dilution factor": None,
120+
"record type": None,
121+
"flag": None,
122+
},
117123
),
118-
"glutamine_GLN2B": RawMeasurement(
124+
"glutamine_GLN2B_None": RawMeasurement(
119125
"glutamine",
120126
"2021-05-20T16:56:51+00:00",
121127
4.35,
122128
"mmol/L",
123129
"GLN2B",
124130
None,
125-
{"analyte code": "GLN2B", "record type": None, "flag": None},
131+
None,
132+
{
133+
"analyte code": "GLN2B",
134+
"dilution factor": None,
135+
"record type": None,
136+
"flag": None,
137+
},
126138
),
127-
"osmolality_OSM2B": RawMeasurement(
139+
"osmolality_OSM2B_None": RawMeasurement(
128140
"osmolality",
129141
"2021-05-20T16:57:51+00:00",
130142
3.7448,
131143
"mosm/kg",
132144
"OSM2B",
133145
None,
134-
{"analyte code": "OSM2B", "record type": None, "flag": None},
146+
None,
147+
{
148+
"analyte code": "OSM2B",
149+
"dilution factor": None,
150+
"record type": None,
151+
"flag": None,
152+
},
135153
),
136154
}
137155
}
@@ -155,40 +173,61 @@ def test_create_measurements_more_than_one_measurement_docs() -> None:
155173

156174
assert measurements == {
157175
"2021-05-20T16:55:51+00:00": {
158-
"lactate_LAC2B": RawMeasurement(
176+
"lactate_LAC2B_None": RawMeasurement(
159177
"lactate",
160178
"2021-05-20T16:55:51+00:00",
161179
2.45,
162180
"g/L",
163181
"LAC2B",
164182
None,
165-
{"analyte code": "LAC2B", "record type": None, "flag": None},
183+
None,
184+
{
185+
"analyte code": "LAC2B",
186+
"dilution factor": None,
187+
"record type": None,
188+
"flag": None,
189+
},
166190
),
167-
"glutamine_GLN2B": RawMeasurement(
191+
"glutamine_GLN2B_None": RawMeasurement(
168192
"glutamine",
169193
"2021-05-20T16:56:51+00:00",
170194
4.35,
171195
"mmol/L",
172196
"GLN2B",
173197
None,
174-
{"analyte code": "GLN2B", "record type": None, "flag": None},
198+
None,
199+
{
200+
"analyte code": "GLN2B",
201+
"dilution factor": None,
202+
"record type": None,
203+
"flag": None,
204+
},
175205
),
176206
},
177207
"2021-05-21T16:57:51+00:00": {
178-
"glutamine_GLN2B": RawMeasurement(
208+
"glutamine_GLN2B_None": RawMeasurement(
179209
"glutamine",
180210
"2021-05-21T16:57:51+00:00",
181211
3.45,
182212
"mmol/L",
183213
"GLN2B",
184214
None,
185-
{"analyte code": "GLN2B", "record type": None, "flag": None},
215+
None,
216+
{
217+
"analyte code": "GLN2B",
218+
"dilution factor": None,
219+
"record type": None,
220+
"flag": None,
221+
},
186222
),
187223
},
188224
}
189225

190226

191-
def test_create_measurements_duplicate_measurements() -> None:
227+
def test_create_measurements_duplicate_measurements_split_into_new_group() -> None:
228+
# A repeated analyte at the same dilution within the time window represents a second
229+
# round of measurements, so it is split into a new group keyed by its own time rather
230+
# than colliding with the first round.
192231
data = pd.DataFrame(
193232
{
194233
"analyte name": ["lactate", "glutamine", "glutamine"],
@@ -202,12 +241,48 @@ def test_create_measurements_duplicate_measurements() -> None:
202241
"analyte code": ["LAC2B", "GLN2B", "GLN2B"],
203242
},
204243
)
244+
measurements = create_measurements(data)
245+
246+
assert set(measurements) == {
247+
"2021-05-20T16:55:51+00:00",
248+
"2021-05-20T16:57:51+00:00",
249+
}
250+
assert (
251+
measurements["2021-05-20T16:55:51+00:00"][
252+
"glutamine_GLN2B_None"
253+
].concentration_value
254+
== 4.35
255+
)
256+
assert (
257+
measurements["2021-05-20T16:57:51+00:00"][
258+
"glutamine_GLN2B_None"
259+
].concentration_value
260+
== 3.45
261+
)
205262

206-
with pytest.raises(
207-
AllotropyParserError,
208-
match="Duplicate measurement for GLN2B in the same measurement group. 3.45 vs 4.35",
209-
):
210-
create_measurements(data)
263+
264+
def test_create_measurements_same_analyte_different_dilution() -> None:
265+
# The same analyte measured at different dilutions in the same group is distinct, not
266+
# a duplicate, so both are retained under separate ids.
267+
data = pd.DataFrame(
268+
{
269+
"analyte name": ["glutamine", "glutamine"],
270+
"measurement time": [
271+
"2021-05-20T16:55:51+00:00",
272+
"2021-05-20T16:56:51+00:00",
273+
],
274+
"dilution factor": ["", "1/5"],
275+
"concentration unit": ["mmol/L", "mmol/L"],
276+
"concentration value": [4.35, 3.45],
277+
"analyte code": ["GLN2B", "GLN2B"],
278+
},
279+
)
280+
measurements = create_measurements(data)
281+
282+
group = measurements["2021-05-20T16:55:51+00:00"]
283+
assert set(group) == {"glutamine_GLN2B_None", "glutamine_GLN2B_1/5"}
284+
assert group["glutamine_GLN2B_None"].concentration_value == 4.35
285+
assert group["glutamine_GLN2B_1/5"].concentration_value == 3.45
211286

212287

213288
def test_create_sample() -> None:
@@ -235,23 +310,35 @@ def test_create_sample() -> None:
235310
assert sample.batch == "batch_id"
236311
assert sample.measurements == {
237312
"2021-05-20 16:55:51": {
238-
"lactate_LAC2B": RawMeasurement(
313+
"lactate_LAC2B_None": RawMeasurement(
239314
"lactate",
240315
"2021-05-20 16:55:51",
241316
2.45,
242317
"g/L",
243318
"LAC2B",
244319
None,
245-
{"analyte code": "LAC2B", "record type": None, "flag": None},
320+
None,
321+
{
322+
"analyte code": "LAC2B",
323+
"dilution factor": None,
324+
"record type": None,
325+
"flag": None,
326+
},
246327
),
247-
"glutamine_GLN2B": RawMeasurement(
328+
"glutamine_GLN2B_None": RawMeasurement(
248329
"glutamine",
249330
"2021-05-20 16:56:51",
250331
4.35,
251332
"mmol/L",
252333
"GLN2B",
253334
None,
254-
{"analyte code": "GLN2B", "record type": None, "flag": None},
335+
None,
336+
{
337+
"analyte code": "GLN2B",
338+
"dilution factor": None,
339+
"record type": None,
340+
"flag": None,
341+
},
255342
),
256343
}
257344
}

0 commit comments

Comments
 (0)