fix: Handle dilutions, replicate measurements, and mmol/l unit in Roche Cedex Bio HT

src/allotropy/parsers/roche_cedex_bioht/constants.py

@@ -29,7 +29,7 @@
     "sample role type",
     "col6",
     "analyte name",
-    "col8",
+    "dilution factor",
     "concentration unit",
     "flag",
     "concentration value",

src/allotropy/parsers/roche_cedex_bioht/roche_cedex_bioht_structure.py

@@ -16,7 +16,6 @@
     MeasurementGroup,
     Metadata,
 )
-from allotropy.exceptions import AllotropyParserError
 from allotropy.parsers.constants import NOT_APPLICABLE
 from allotropy.parsers.roche_cedex_bioht.constants import (
     BELOW_TEST_RANGE,
@@ -32,6 +31,8 @@
 UNIT_CONVERSIONS: dict[str, tuple[str, float]] = {
     "mg/L": ("g/L", 0.001),
     "mM": ("mmol/L", 1.0),
+    # Some files report a lowercase-liter variant that means the same thing.
+    "mmol/l": ("mmol/L", 1.0),
 }
 
 
@@ -62,6 +63,7 @@ class RawMeasurement:
     concentration_value: JsonFloat
     unit: str
     analyte_code: str
+    dilution_factor: str | None = None
     error: str | None = None
     custom_info: dict[str, Any] | None = None
 
@@ -70,6 +72,11 @@ def create(data: SeriesData) -> RawMeasurement:
         flag = data.get(str, "flag", "").strip()
         error = FLAG_TO_ERROR[flag] if flag else None
 
+        # V6/V7 files report a dilution factor (e.g. "1/5") for measurements that were
+        # diluted before reading. The same analyte may be measured at multiple dilutions
+        # in one sample, so it distinguishes otherwise-identical measurements.
+        dilution_factor = data.get(str, "dilution factor", "").strip() or None
+
         concentration_value = data.get(float, "concentration value", NaN)
 
         value, unit = RawMeasurement._get_value_and_unit(
@@ -87,6 +94,7 @@ def create(data: SeriesData) -> RawMeasurement:
             value,
             unit,
             data[str, "analyte code"],
+            dilution_factor,
             error,
             custom_info=dict(
                 sorted(
@@ -96,6 +104,7 @@ def create(data: SeriesData) -> RawMeasurement:
                             "analyte code": data.get(str, "analyte code"),
                             "record type": data.get(str, "row type"),
                             "flag": error or None,
+                            "dilution factor": dilution_factor,
                         },
                         **data.get_unread(
                             skip={
@@ -142,7 +151,9 @@ def create_measurements(data: pd.DataFrame) -> dict[str, dict[str, RawMeasuremen
     current_measurement_time = measurements[0].measurement_time
     previous_measurement_time = current_measurement_time
     for analyte in measurements:
-        analyte_id = f"{analyte.name}_{analyte.analyte_code}"
+        # Include dilution factor in the id so the same analyte measured at different
+        # dilutions within a group is not treated as a duplicate measurement.
+        analyte_id = f"{analyte.name}_{analyte.analyte_code}_{analyte.dilution_factor}"
         time_diff = parser.parse(analyte.measurement_time) - parser.parse(
             previous_measurement_time
         )
@@ -151,16 +162,16 @@ def create_measurements(data: pd.DataFrame) -> dict[str, dict[str, RawMeasuremen
         if analyte_id in groups[current_measurement_time]:
             if analyte.concentration_value is NaN:
                 continue
-            # NOTE: if this fails, it's probably because MAX_MEASUREMENT_TIME_GROUP_DIFFERENCE is too big
-            # and we're erroneously grouping two groups of measurements into one.
-            # We could potentially make this more robust by just splitting into a new group if a duplicate
-            # measurement is found, but cross that bridge when we come to it.
             if (
                 groups[current_measurement_time][analyte_id].concentration_value
                 is not NaN
             ):
-                msg = f"Duplicate measurement for {analyte.analyte_code} in the same measurement group. {analyte.concentration_value} vs {groups[current_measurement_time][analyte_id].concentration_value}"
-                raise AllotropyParserError(msg)
+                # A real duplicate measurement of the same analyte (same dilution) within
+                # the time window means we've observed a second round of measurements for
+                # the sample. Start a new group so both rounds are preserved rather than
+                # colliding. (This can also happen if MAX_MEASUREMENT_TIME_GROUP_DIFFERENCE
+                # is too large and is erroneously merging distinct rounds.)
+                current_measurement_time = analyte.measurement_time
         groups[current_measurement_time][analyte_id] = analyte
         previous_measurement_time = analyte.measurement_time
 

tests/parsers/roche_cedex_bioht/roche_cedex_bioht_data.py

@@ -98,7 +98,7 @@ def get_reader_samples() -> pd.DataFrame:
             "sample role type",
             "col6",
             "analyte name",
-            "col8",
+            "dilution factor",
             "concentration unit",
             "flag",
             "concentration value",

tests/parsers/roche_cedex_bioht/roche_cedex_bioht_structure_test.py

@@ -1,7 +1,7 @@
 import pandas as pd
 import pytest
 
-from allotropy.exceptions import AllotropeConversionError, AllotropyParserError
+from allotropy.exceptions import AllotropeConversionError
 from allotropy.parsers.roche_cedex_bioht.roche_cedex_bioht_structure import (
     create_measurements,
     RawMeasurement,
@@ -106,32 +106,50 @@ def test_create_measurements() -> None:
 
     assert measurements == {
         "2021-05-20T16:55:51+00:00": {
-            "lactate_LAC2B": RawMeasurement(
+            "lactate_LAC2B_None": RawMeasurement(
                 "lactate",
                 "2021-05-20T16:55:51+00:00",
                 2.45,
                 "g/L",
                 "LAC2B",
                 None,
-                {"analyte code": "LAC2B", "record type": None, "flag": None},
+                None,
+                {
+                    "analyte code": "LAC2B",
+                    "dilution factor": None,
+                    "record type": None,
+                    "flag": None,
+                },
             ),
-            "glutamine_GLN2B": RawMeasurement(
+            "glutamine_GLN2B_None": RawMeasurement(
                 "glutamine",
                 "2021-05-20T16:56:51+00:00",
                 4.35,
                 "mmol/L",
                 "GLN2B",
                 None,
-                {"analyte code": "GLN2B", "record type": None, "flag": None},
+                None,
+                {
+                    "analyte code": "GLN2B",
+                    "dilution factor": None,
+                    "record type": None,
+                    "flag": None,
+                },
             ),
-            "osmolality_OSM2B": RawMeasurement(
+            "osmolality_OSM2B_None": RawMeasurement(
                 "osmolality",
                 "2021-05-20T16:57:51+00:00",
                 3.7448,
                 "mosm/kg",
                 "OSM2B",
                 None,
-                {"analyte code": "OSM2B", "record type": None, "flag": None},
+                None,
+                {
+                    "analyte code": "OSM2B",
+                    "dilution factor": None,
+                    "record type": None,
+                    "flag": None,
+                },
             ),
         }
     }
@@ -155,40 +173,61 @@ def test_create_measurements_more_than_one_measurement_docs() -> None:
 
     assert measurements == {
         "2021-05-20T16:55:51+00:00": {
-            "lactate_LAC2B": RawMeasurement(
+            "lactate_LAC2B_None": RawMeasurement(
                 "lactate",
                 "2021-05-20T16:55:51+00:00",
                 2.45,
                 "g/L",
                 "LAC2B",
                 None,
-                {"analyte code": "LAC2B", "record type": None, "flag": None},
+                None,
+                {
+                    "analyte code": "LAC2B",
+                    "dilution factor": None,
+                    "record type": None,
+                    "flag": None,
+                },
             ),
-            "glutamine_GLN2B": RawMeasurement(
+            "glutamine_GLN2B_None": RawMeasurement(
                 "glutamine",
                 "2021-05-20T16:56:51+00:00",
                 4.35,
                 "mmol/L",
                 "GLN2B",
                 None,
-                {"analyte code": "GLN2B", "record type": None, "flag": None},
+                None,
+                {
+                    "analyte code": "GLN2B",
+                    "dilution factor": None,
+                    "record type": None,
+                    "flag": None,
+                },
             ),
         },
         "2021-05-21T16:57:51+00:00": {
-            "glutamine_GLN2B": RawMeasurement(
+            "glutamine_GLN2B_None": RawMeasurement(
                 "glutamine",
                 "2021-05-21T16:57:51+00:00",
                 3.45,
                 "mmol/L",
                 "GLN2B",
                 None,
-                {"analyte code": "GLN2B", "record type": None, "flag": None},
+                None,
+                {
+                    "analyte code": "GLN2B",
+                    "dilution factor": None,
+                    "record type": None,
+                    "flag": None,
+                },
             ),
         },
     }
 
 
-def test_create_measurements_duplicate_measurements() -> None:
+def test_create_measurements_duplicate_measurements_split_into_new_group() -> None:
+    # A repeated analyte at the same dilution within the time window represents a second
+    # round of measurements, so it is split into a new group keyed by its own time rather
+    # than colliding with the first round.
     data = pd.DataFrame(
         {
             "analyte name": ["lactate", "glutamine", "glutamine"],
@@ -202,12 +241,48 @@ def test_create_measurements_duplicate_measurements() -> None:
             "analyte code": ["LAC2B", "GLN2B", "GLN2B"],
         },
     )
+    measurements = create_measurements(data)
+
+    assert set(measurements) == {
+        "2021-05-20T16:55:51+00:00",
+        "2021-05-20T16:57:51+00:00",
+    }
+    assert (
+        measurements["2021-05-20T16:55:51+00:00"][
+            "glutamine_GLN2B_None"
+        ].concentration_value
+        == 4.35
+    )
+    assert (
+        measurements["2021-05-20T16:57:51+00:00"][
+            "glutamine_GLN2B_None"
+        ].concentration_value
+        == 3.45
+    )
 
-    with pytest.raises(
-        AllotropyParserError,
-        match="Duplicate measurement for GLN2B in the same measurement group. 3.45 vs 4.35",
-    ):
-        create_measurements(data)
+
+def test_create_measurements_same_analyte_different_dilution() -> None:
+    # The same analyte measured at different dilutions in the same group is distinct, not
+    # a duplicate, so both are retained under separate ids.
+    data = pd.DataFrame(
+        {
+            "analyte name": ["glutamine", "glutamine"],
+            "measurement time": [
+                "2021-05-20T16:55:51+00:00",
+                "2021-05-20T16:56:51+00:00",
+            ],
+            "dilution factor": ["", "1/5"],
+            "concentration unit": ["mmol/L", "mmol/L"],
+            "concentration value": [4.35, 3.45],
+            "analyte code": ["GLN2B", "GLN2B"],
+        },
+    )
+    measurements = create_measurements(data)
+
+    group = measurements["2021-05-20T16:55:51+00:00"]
+    assert set(group) == {"glutamine_GLN2B_None", "glutamine_GLN2B_1/5"}
+    assert group["glutamine_GLN2B_None"].concentration_value == 4.35
+    assert group["glutamine_GLN2B_1/5"].concentration_value == 3.45
 
 
 def test_create_sample() -> None:
@@ -235,23 +310,35 @@ def test_create_sample() -> None:
     assert sample.batch == "batch_id"
     assert sample.measurements == {
         "2021-05-20 16:55:51": {
-            "lactate_LAC2B": RawMeasurement(
+            "lactate_LAC2B_None": RawMeasurement(
                 "lactate",
                 "2021-05-20 16:55:51",
                 2.45,
                 "g/L",
                 "LAC2B",
                 None,
-                {"analyte code": "LAC2B", "record type": None, "flag": None},
+                None,
+                {
+                    "analyte code": "LAC2B",
+                    "dilution factor": None,
+                    "record type": None,
+                    "flag": None,
+                },
             ),
-            "glutamine_GLN2B": RawMeasurement(
+            "glutamine_GLN2B_None": RawMeasurement(
                 "glutamine",
                 "2021-05-20 16:56:51",
                 4.35,
                 "mmol/L",
                 "GLN2B",
                 None,
-                {"analyte code": "GLN2B", "record type": None, "flag": None},
+                None,
+                {
+                    "analyte code": "GLN2B",
+                    "dilution factor": None,
+                    "record type": None,
+                    "flag": None,
+                },
             ),
         }
     }