diff --git a/src/allotropy/parsers/roche_cedex_bioht/constants.py b/src/allotropy/parsers/roche_cedex_bioht/constants.py index b91950501..6fa174511 100644 --- a/src/allotropy/parsers/roche_cedex_bioht/constants.py +++ b/src/allotropy/parsers/roche_cedex_bioht/constants.py @@ -29,7 +29,7 @@ "sample role type", "col6", "analyte name", - "col8", + "dilution factor", "concentration unit", "flag", "concentration value", diff --git a/src/allotropy/parsers/roche_cedex_bioht/roche_cedex_bioht_structure.py b/src/allotropy/parsers/roche_cedex_bioht/roche_cedex_bioht_structure.py index d1a9a6cbc..d8136ff2d 100644 --- a/src/allotropy/parsers/roche_cedex_bioht/roche_cedex_bioht_structure.py +++ b/src/allotropy/parsers/roche_cedex_bioht/roche_cedex_bioht_structure.py @@ -16,7 +16,6 @@ MeasurementGroup, Metadata, ) -from allotropy.exceptions import AllotropyParserError from allotropy.parsers.constants import NOT_APPLICABLE from allotropy.parsers.roche_cedex_bioht.constants import ( BELOW_TEST_RANGE, @@ -32,6 +31,8 @@ UNIT_CONVERSIONS: dict[str, tuple[str, float]] = { "mg/L": ("g/L", 0.001), "mM": ("mmol/L", 1.0), + # Some files report a lowercase-liter variant that means the same thing. + "mmol/l": ("mmol/L", 1.0), } @@ -62,6 +63,7 @@ class RawMeasurement: concentration_value: JsonFloat unit: str analyte_code: str + dilution_factor: str | None = None error: str | None = None custom_info: dict[str, Any] | None = None @@ -70,6 +72,11 @@ def create(data: SeriesData) -> RawMeasurement: flag = data.get(str, "flag", "").strip() error = FLAG_TO_ERROR[flag] if flag else None + # V6/V7 files report a dilution factor (e.g. "1/5") for measurements that were + # diluted before reading. The same analyte may be measured at multiple dilutions + # in one sample, so it distinguishes otherwise-identical measurements. + dilution_factor = data.get(str, "dilution factor", "").strip() or None + concentration_value = data.get(float, "concentration value", NaN) value, unit = RawMeasurement._get_value_and_unit( @@ -87,6 +94,7 @@ def create(data: SeriesData) -> RawMeasurement: value, unit, data[str, "analyte code"], + dilution_factor, error, custom_info=dict( sorted( @@ -96,6 +104,7 @@ def create(data: SeriesData) -> RawMeasurement: "analyte code": data.get(str, "analyte code"), "record type": data.get(str, "row type"), "flag": error or None, + "dilution factor": dilution_factor, }, **data.get_unread( skip={ @@ -142,7 +151,9 @@ def create_measurements(data: pd.DataFrame) -> dict[str, dict[str, RawMeasuremen current_measurement_time = measurements[0].measurement_time previous_measurement_time = current_measurement_time for analyte in measurements: - analyte_id = f"{analyte.name}_{analyte.analyte_code}" + # Include dilution factor in the id so the same analyte measured at different + # dilutions within a group is not treated as a duplicate measurement. + analyte_id = f"{analyte.name}_{analyte.analyte_code}_{analyte.dilution_factor}" time_diff = parser.parse(analyte.measurement_time) - parser.parse( previous_measurement_time ) @@ -151,16 +162,16 @@ def create_measurements(data: pd.DataFrame) -> dict[str, dict[str, RawMeasuremen if analyte_id in groups[current_measurement_time]: if analyte.concentration_value is NaN: continue - # NOTE: if this fails, it's probably because MAX_MEASUREMENT_TIME_GROUP_DIFFERENCE is too big - # and we're erroneously grouping two groups of measurements into one. - # We could potentially make this more robust by just splitting into a new group if a duplicate - # measurement is found, but cross that bridge when we come to it. if ( groups[current_measurement_time][analyte_id].concentration_value is not NaN ): - msg = f"Duplicate measurement for {analyte.analyte_code} in the same measurement group. {analyte.concentration_value} vs {groups[current_measurement_time][analyte_id].concentration_value}" - raise AllotropyParserError(msg) + # A real duplicate measurement of the same analyte (same dilution) within + # the time window means we've observed a second round of measurements for + # the sample. Start a new group so both rounds are preserved rather than + # colliding. (This can also happen if MAX_MEASUREMENT_TIME_GROUP_DIFFERENCE + # is too large and is erroneously merging distinct rounds.) + current_measurement_time = analyte.measurement_time groups[current_measurement_time][analyte_id] = analyte previous_measurement_time = analyte.measurement_time diff --git a/tests/parsers/roche_cedex_bioht/roche_cedex_bioht_data.py b/tests/parsers/roche_cedex_bioht/roche_cedex_bioht_data.py index eb9239a10..5878debeb 100644 --- a/tests/parsers/roche_cedex_bioht/roche_cedex_bioht_data.py +++ b/tests/parsers/roche_cedex_bioht/roche_cedex_bioht_data.py @@ -98,7 +98,7 @@ def get_reader_samples() -> pd.DataFrame: "sample role type", "col6", "analyte name", - "col8", + "dilution factor", "concentration unit", "flag", "concentration value", diff --git a/tests/parsers/roche_cedex_bioht/roche_cedex_bioht_structure_test.py b/tests/parsers/roche_cedex_bioht/roche_cedex_bioht_structure_test.py index d763c71bc..557be789e 100644 --- a/tests/parsers/roche_cedex_bioht/roche_cedex_bioht_structure_test.py +++ b/tests/parsers/roche_cedex_bioht/roche_cedex_bioht_structure_test.py @@ -1,7 +1,7 @@ import pandas as pd import pytest -from allotropy.exceptions import AllotropeConversionError, AllotropyParserError +from allotropy.exceptions import AllotropeConversionError from allotropy.parsers.roche_cedex_bioht.roche_cedex_bioht_structure import ( create_measurements, RawMeasurement, @@ -106,32 +106,50 @@ def test_create_measurements() -> None: assert measurements == { "2021-05-20T16:55:51+00:00": { - "lactate_LAC2B": RawMeasurement( + "lactate_LAC2B_None": RawMeasurement( "lactate", "2021-05-20T16:55:51+00:00", 2.45, "g/L", "LAC2B", None, - {"analyte code": "LAC2B", "record type": None, "flag": None}, + None, + { + "analyte code": "LAC2B", + "dilution factor": None, + "record type": None, + "flag": None, + }, ), - "glutamine_GLN2B": RawMeasurement( + "glutamine_GLN2B_None": RawMeasurement( "glutamine", "2021-05-20T16:56:51+00:00", 4.35, "mmol/L", "GLN2B", None, - {"analyte code": "GLN2B", "record type": None, "flag": None}, + None, + { + "analyte code": "GLN2B", + "dilution factor": None, + "record type": None, + "flag": None, + }, ), - "osmolality_OSM2B": RawMeasurement( + "osmolality_OSM2B_None": RawMeasurement( "osmolality", "2021-05-20T16:57:51+00:00", 3.7448, "mosm/kg", "OSM2B", None, - {"analyte code": "OSM2B", "record type": None, "flag": None}, + None, + { + "analyte code": "OSM2B", + "dilution factor": None, + "record type": None, + "flag": None, + }, ), } } @@ -155,40 +173,61 @@ def test_create_measurements_more_than_one_measurement_docs() -> None: assert measurements == { "2021-05-20T16:55:51+00:00": { - "lactate_LAC2B": RawMeasurement( + "lactate_LAC2B_None": RawMeasurement( "lactate", "2021-05-20T16:55:51+00:00", 2.45, "g/L", "LAC2B", None, - {"analyte code": "LAC2B", "record type": None, "flag": None}, + None, + { + "analyte code": "LAC2B", + "dilution factor": None, + "record type": None, + "flag": None, + }, ), - "glutamine_GLN2B": RawMeasurement( + "glutamine_GLN2B_None": RawMeasurement( "glutamine", "2021-05-20T16:56:51+00:00", 4.35, "mmol/L", "GLN2B", None, - {"analyte code": "GLN2B", "record type": None, "flag": None}, + None, + { + "analyte code": "GLN2B", + "dilution factor": None, + "record type": None, + "flag": None, + }, ), }, "2021-05-21T16:57:51+00:00": { - "glutamine_GLN2B": RawMeasurement( + "glutamine_GLN2B_None": RawMeasurement( "glutamine", "2021-05-21T16:57:51+00:00", 3.45, "mmol/L", "GLN2B", None, - {"analyte code": "GLN2B", "record type": None, "flag": None}, + None, + { + "analyte code": "GLN2B", + "dilution factor": None, + "record type": None, + "flag": None, + }, ), }, } -def test_create_measurements_duplicate_measurements() -> None: +def test_create_measurements_duplicate_measurements_split_into_new_group() -> None: + # A repeated analyte at the same dilution within the time window represents a second + # round of measurements, so it is split into a new group keyed by its own time rather + # than colliding with the first round. data = pd.DataFrame( { "analyte name": ["lactate", "glutamine", "glutamine"], @@ -202,12 +241,48 @@ def test_create_measurements_duplicate_measurements() -> None: "analyte code": ["LAC2B", "GLN2B", "GLN2B"], }, ) + measurements = create_measurements(data) + + assert set(measurements) == { + "2021-05-20T16:55:51+00:00", + "2021-05-20T16:57:51+00:00", + } + assert ( + measurements["2021-05-20T16:55:51+00:00"][ + "glutamine_GLN2B_None" + ].concentration_value + == 4.35 + ) + assert ( + measurements["2021-05-20T16:57:51+00:00"][ + "glutamine_GLN2B_None" + ].concentration_value + == 3.45 + ) - with pytest.raises( - AllotropyParserError, - match="Duplicate measurement for GLN2B in the same measurement group. 3.45 vs 4.35", - ): - create_measurements(data) + +def test_create_measurements_same_analyte_different_dilution() -> None: + # The same analyte measured at different dilutions in the same group is distinct, not + # a duplicate, so both are retained under separate ids. + data = pd.DataFrame( + { + "analyte name": ["glutamine", "glutamine"], + "measurement time": [ + "2021-05-20T16:55:51+00:00", + "2021-05-20T16:56:51+00:00", + ], + "dilution factor": ["", "1/5"], + "concentration unit": ["mmol/L", "mmol/L"], + "concentration value": [4.35, 3.45], + "analyte code": ["GLN2B", "GLN2B"], + }, + ) + measurements = create_measurements(data) + + group = measurements["2021-05-20T16:55:51+00:00"] + assert set(group) == {"glutamine_GLN2B_None", "glutamine_GLN2B_1/5"} + assert group["glutamine_GLN2B_None"].concentration_value == 4.35 + assert group["glutamine_GLN2B_1/5"].concentration_value == 3.45 def test_create_sample() -> None: @@ -235,23 +310,35 @@ def test_create_sample() -> None: assert sample.batch == "batch_id" assert sample.measurements == { "2021-05-20 16:55:51": { - "lactate_LAC2B": RawMeasurement( + "lactate_LAC2B_None": RawMeasurement( "lactate", "2021-05-20 16:55:51", 2.45, "g/L", "LAC2B", None, - {"analyte code": "LAC2B", "record type": None, "flag": None}, + None, + { + "analyte code": "LAC2B", + "dilution factor": None, + "record type": None, + "flag": None, + }, ), - "glutamine_GLN2B": RawMeasurement( + "glutamine_GLN2B_None": RawMeasurement( "glutamine", "2021-05-20 16:56:51", 4.35, "mmol/L", "GLN2B", None, - {"analyte code": "GLN2B", "record type": None, "flag": None}, + None, + { + "analyte code": "GLN2B", + "dilution factor": None, + "record type": None, + "flag": None, + }, ), } } diff --git a/tests/parsers/roche_cedex_bioht/testdata/roche_cedex_bioht_v7_dilution_and_replicates.json b/tests/parsers/roche_cedex_bioht/testdata/roche_cedex_bioht_v7_dilution_and_replicates.json new file mode 100644 index 000000000..3d3a9a8fe --- /dev/null +++ b/tests/parsers/roche_cedex_bioht/testdata/roche_cedex_bioht_v7_dilution_and_replicates.json @@ -0,0 +1,347 @@ +{ + "$asm.manifest": "http://purl.allotrope.org/manifests/solution-analyzer/REC/2024/09/solution-analyzer.manifest", + "solution analyzer aggregate document": { + "data system document": { + "ASM file identifier": "roche_cedex_bioht_v7_dilution_and_replicates.json", + "data system instance identifier": "N/A", + "ASM converter name": "allotropy_roche_cedex_bioht", + "ASM converter version": "0.1.141", + "file name": "roche_cedex_bioht_v7_dilution_and_replicates.txt", + "software name": "CEDEX BIO HT", + "software version": "7.0.0.2410 (2410)", + "UNC path": "tests/parsers/roche_cedex_bioht/testdata/roche_cedex_bioht_v7_dilution_and_replicates.txt" + }, + "device system document": { + "device identifier": "N/A", + "equipment serial number": "600106", + "model number": "CEDEX BIO HT" + }, + "solution analyzer document": [ + { + "measurement aggregate document": { + "measurement document": [ + { + "device control aggregate document": { + "device control document": [ + { + "device type": "solution-analyzer" + } + ] + }, + "measurement identifier": "ROCHE_CEDEX_BIOHT_TEST_ID_0", + "measurement time": "2026-05-11T09:26:45+00:00", + "sample document": { + "sample identifier": "10AMB26002BR01" + }, + "analyte aggregate document": { + "analyte document": [ + { + "analyte name": "IgG human", + "mass concentration": { + "value": 6.174, + "unit": "g/L" + }, + "custom information document": { + "analyte code": "IGGHD", + "detection kit": "High with Pre - Dilution", + "detection kit range": "0.002665 - 0.05335 mmol/L ", + "dilution factor": "1/5", + "flag": "verification", + "record type": "40" + } + }, + { + "analyte name": "IgG human", + "mass concentration": { + "value": 3.933, + "unit": "g/L" + }, + "custom information document": { + "analyte code": "IGGHD", + "detection kit": "High with Pre - Dilution", + "detection kit range": "0.002665 - 0.05335 mmol/L ", + "record type": "40" + } + }, + { + "analyte name": "ldh", + "molar concentration": { + "value": 32.6151, + "unit": "mmol/L" + }, + "custom information document": { + "analyte code": "LDH2D", + "detection kit": "High with Pre - Dilution", + "detection kit range": "200 - 10000 U/L", + "record type": "40" + } + } + ] + } + } + ], + "data processing time": "2026-06-02T15:13:31+00:00" + }, + "analyst": "laboper" + }, + { + "measurement aggregate document": { + "measurement document": [ + { + "device control aggregate document": { + "device control document": [ + { + "device type": "solution-analyzer" + } + ] + }, + "measurement identifier": "ROCHE_CEDEX_BIOHT_TEST_ID_1", + "measurement time": "2026-02-16T09:46:38+00:00", + "sample document": { + "sample identifier": "11AMB26001NBR04" + }, + "analyte aggregate document": { + "analyte document": [ + { + "analyte name": "IgG human", + "mass concentration": { + "value": 6.67, + "unit": "g/L" + }, + "custom information document": { + "analyte code": "IGGHD", + "detection kit": "High with Pre - Dilution", + "detection kit range": "0.002665 - 0.05335 mmol/L ", + "dilution factor": "1/10", + "flag": "verification", + "record type": "40" + } + }, + { + "analyte name": "IgG human", + "mass concentration": { + "value": 7.549, + "unit": "g/L" + }, + "custom information document": { + "analyte code": "IGGHD", + "detection kit": "High with Pre - Dilution", + "detection kit range": "0.002665 - 0.05335 mmol/L ", + "flag": "verification", + "record type": "40" + } + }, + { + "analyte name": "glucose", + "molar concentration": { + "value": 15.17, + "unit": "mmol/L" + }, + "custom information document": { + "analyte code": "GLC3D", + "detection kit": "High with Pre - Dilution", + "detection kit range": "1.11 - 416.3 mmol/L", + "record type": "40" + } + }, + { + "analyte name": "ldh", + "molar concentration": { + "value": 116.4825, + "unit": "mmol/L" + }, + "custom information document": { + "analyte code": "LDH2D", + "detection kit": "High with Pre - Dilution", + "detection kit range": "200 - 10000 U/L", + "record type": "40" + } + } + ] + } + }, + { + "device control aggregate document": { + "device control document": [ + { + "device type": "solution-analyzer" + } + ] + }, + "measurement identifier": "ROCHE_CEDEX_BIOHT_TEST_ID_2", + "measurement time": "2026-02-16T10:49:48+00:00", + "sample document": { + "sample identifier": "11AMB26001NBR04" + }, + "analyte aggregate document": { + "analyte document": [ + { + "analyte name": "IgG human", + "mass concentration": { + "value": 9.051, + "unit": "g/L" + }, + "custom information document": { + "analyte code": "IGGHD", + "detection kit": "High with Pre - Dilution", + "detection kit range": "0.002665 - 0.05335 mmol/L ", + "dilution factor": "1/2", + "flag": "verification", + "record type": "40" + } + }, + { + "analyte name": "glucose", + "molar concentration": { + "value": 15.43, + "unit": "mmol/L" + }, + "custom information document": { + "analyte code": "GLC3D", + "detection kit": "High with Pre - Dilution", + "detection kit range": "1.11 - 416.3 mmol/L", + "record type": "40" + } + }, + { + "analyte name": "ldh", + "molar concentration": { + "value": 116.5159, + "unit": "mmol/L" + }, + "custom information document": { + "analyte code": "LDH2D", + "detection kit": "High with Pre - Dilution", + "detection kit range": "200 - 10000 U/L", + "record type": "40" + } + } + ] + } + } + ], + "data processing time": "2026-06-02T15:13:31+00:00" + }, + "analyst": "laboper" + }, + { + "measurement aggregate document": { + "measurement document": [ + { + "device control aggregate document": { + "device control document": [ + { + "device type": "solution-analyzer" + } + ] + }, + "measurement identifier": "ROCHE_CEDEX_BIOHT_TEST_ID_3", + "measurement time": "2025-12-12T09:49:21+00:00", + "sample document": { + "sample identifier": "7MCC25039NTK151" + }, + "error aggregate document": { + "error document": [ + { + "error": "< 2.50", + "error feature": "asparagine - High with Pre - Dilution" + }, + { + "error": "< 0.50", + "error feature": "aspartate - High with Pre - Dilution" + } + ] + }, + "analyte aggregate document": { + "analyte document": [ + { + "analyte name": "IgG human", + "mass concentration": { + "value": 4.569, + "unit": "g/L" + }, + "custom information document": { + "analyte code": "IGGHD", + "detection kit": "High with Pre - Dilution", + "detection kit range": "0.002665 - 0.05335 mmol/L ", + "record type": "40" + } + }, + { + "analyte name": "asparagine", + "molar concentration": { + "value": -0.0, + "unit": "mmol/L" + }, + "custom information document": { + "analyte code": "ASNHD", + "detection kit": "High with Pre - Dilution", + "detection kit range": "2.5 - 45 mmol/L", + "flag": "< 2.50", + "record type": "40" + } + }, + { + "analyte name": "aspartate", + "molar concentration": { + "value": -0.0, + "unit": "mmol/L" + }, + "custom information document": { + "analyte code": "ASPD", + "detection kit": "High with Pre - Dilution", + "detection kit range": "1 - 45 mmol/L", + "flag": "< 0.50", + "record type": "40" + } + }, + { + "analyte name": "glucose", + "molar concentration": { + "value": 13.26, + "unit": "mmol/L" + }, + "custom information document": { + "analyte code": "GLC3D", + "detection kit": "High with Pre - Dilution", + "detection kit range": "1.11 - 416.3 mmol/L", + "record type": "40" + } + }, + { + "analyte name": "lactate", + "molar concentration": { + "value": 4.46, + "unit": "mmol/L" + }, + "custom information document": { + "analyte code": "LAC2D", + "detection kit": "High with Pre - Dilution", + "detection kit range": "0.444 - 155.5 mmol/L ", + "record type": "40" + } + }, + { + "analyte name": "ldh", + "molar concentration": { + "value": 23.9311, + "unit": "mmol/L" + }, + "custom information document": { + "analyte code": "LDH2D", + "detection kit": "High with Pre - Dilution", + "detection kit range": "200 - 10000 U/L", + "record type": "40" + } + } + ] + } + } + ], + "data processing time": "2026-06-02T15:13:31+00:00" + }, + "analyst": "laboper" + } + ] + } +} diff --git a/tests/parsers/roche_cedex_bioht/testdata/roche_cedex_bioht_v7_dilution_and_replicates.txt b/tests/parsers/roche_cedex_bioht/testdata/roche_cedex_bioht_v7_dilution_and_replicates.txt new file mode 100644 index 000000000..61cf08159 --- /dev/null +++ b/tests/parsers/roche_cedex_bioht/testdata/roche_cedex_bioht_v7_dilution_and_replicates.txt @@ -0,0 +1,17 @@ +0 2026-06-02 15:13:31 #ARC-FILE# 1.1a 2026-06-02 CEDEX BIO HT 600106 7.0.0.2410 (2410) laboper +40 2026-05-11 09:26:45 10AMB26002BR01 SAM IGGHD g/L 3.933 0.06548 R +40 2026-05-11 09:27:53 10AMB26002BR01 SAM IGGHD 1/5 g/L v 6.174 0.02527 R +40 2026-05-11 09:28:56 10AMB26002BR01 SAM LDH2D U/L 1953 0.01526 R +40 2026-02-16 09:46:38 11AMB26001NBR04 SAM LDH2D U/L 6975 0.05563 R +40 2026-02-16 09:59:54 11AMB26001NBR04 SAM IGGHD g/L v 7.549 0.02961 R +40 2026-02-16 10:06:37 11AMB26001NBR04 SAM GLC3D mmol/L 15.17 0.04908 R +40 2026-02-16 10:38:06 11AMB26001NBR04 SAM IGGHD 1/10 g/L v 6.670 0.01073 R +40 2026-02-16 10:49:48 11AMB26001NBR04 SAM GLC3D mmol/L 15.43 0.04989 R +40 2026-02-16 10:50:09 11AMB26001NBR04 SAM IGGHD 1/2 g/L v 9.051 0.06987 R +40 2026-02-16 10:50:42 11AMB26001NBR04 SAM LDH2D U/L 6977 0.05565 R +40 2025-12-12 09:49:21 7MCC25039NTK151 SAM ASNHD mmol/L < TEST RNG < 2.50 0.00149 R +40 2025-12-12 09:49:49 7MCC25039NTK151 SAM ASPD mmol/l < TEST RNG < 0.50 0.00121 R +40 2025-12-12 09:50:08 7MCC25039NTK151 SAM GLC3D mmol/L 13.26 0.04359 R +40 2025-12-12 09:50:28 7MCC25039NTK151 SAM IGGHD g/L 4.569 0.07033 R +40 2025-12-12 09:50:50 7MCC25039NTK151 SAM LAC2D mmol/L 4.46 0.03702 R +40 2025-12-12 09:51:00 7MCC25039NTK151 SAM LDH2D U/L 1433 0.01177 R