fix: Molecular Devices SoftMax Pro - account for partial plate reads (#1013)

Co-authored-by: james-leinas <157071641+james-leinas@users.noreply.github.com>

Author: joshua-benchling

src/allotropy/parsers/moldev_softmax_pro/softmax_pro_data_creator.py

@@ -11,14 +11,19 @@
 )
 from allotropy.allotrope.models.shared.definitions.units import UNITLESS
 from allotropy.allotrope.schema_mappers.adm.plate_reader.rec._2025._03.plate_reader import (
+    ErrorDocument,
     Measurement,
     MeasurementGroup,
     MeasurementType,
     Metadata,
 )
 from allotropy.allotrope.schema_mappers.data_cube import DataCube, DataCubeComponent
 from allotropy.exceptions import AllotropeConversionError
-from allotropy.parsers.constants import DEFAULT_EPOCH_TIMESTAMP, NOT_APPLICABLE
+from allotropy.parsers.constants import (
+    DEFAULT_EPOCH_TIMESTAMP,
+    NEGATIVE_ZERO,
+    NOT_APPLICABLE,
+)
 from allotropy.parsers.moldev_softmax_pro.constants import DEVICE_TYPE
 from allotropy.parsers.moldev_softmax_pro.softmax_pro_structure import (
     DataElement,
@@ -74,15 +79,23 @@ def _get_data_cube(
             )
         ],
         dimensions=[data_element.elapsed_time],
-        measures=[data_element.kinetic_measures],
+        measures=[
+            [
+                value if value is not None else NEGATIVE_ZERO
+                for value in data_element.kinetic_measures
+            ]
+        ],
     )
 
 
 def _get_spectrum_data_cube(
     plate_block: PlateBlock, data_elements: list[DataElement]
-) -> DataCube:
+) -> DataCube | None:
     wavelengths = [data_element.wavelength for data_element in data_elements]
     values = [data_element.value for data_element in data_elements]
+    if all(value is None for value in values):
+        # Ignore the wells completely from the ASM if all values are None
+        return None
 
     return DataCube(
         label=f"{plate_block.header.concept}-spectrum",
@@ -99,20 +112,36 @@ def _get_spectrum_data_cube(
             )
         ],
         dimensions=[wavelengths],
-        measures=[values],
+        measures=[[value if value is not None else NEGATIVE_ZERO for value in values]],
     )
 
 
 def _create_spectrum_measurement(
     plate_block: PlateBlock, data_elements: list[DataElement]
-) -> Measurement:
+) -> Measurement | None:
     measurement_type = plate_block.measurement_type
     first_data_element = data_elements[0]
+    spectrum_data_cube = _get_spectrum_data_cube(plate_block, data_elements)
+    if not spectrum_data_cube:
+        return None
+
+    # Collect error documents and update error_feature to include wavelength with unit
+    error_documents = []
+    for data_element in data_elements:
+        for error_doc in data_element.error_document:
+            if error_doc.error_feature == plate_block.header.read_mode:
+                updated_error_doc = ErrorDocument(
+                    error=error_doc.error, error_feature=f"{data_element.wavelength}nm"
+                )
+                error_documents.append(updated_error_doc)
+            else:
+                # Keep original error_feature for non-spectrum data cube errors
+                error_documents.append(error_doc)
 
     return Measurement(
         type_=measurement_type,
         identifier=first_data_element.uuid,
-        spectrum_data_cube=_get_spectrum_data_cube(plate_block, data_elements),
+        spectrum_data_cube=spectrum_data_cube,
         compartment_temperature=first_data_element.temperature or None,
         location_identifier=first_data_element.position,
         well_plate_identifier=plate_block.header.name,
@@ -136,7 +165,7 @@ def _create_spectrum_measurement(
         total_measurement_time_setting=plate_block.header.read_time,
         read_interval_setting=plate_block.header.read_interval,
         number_of_scans_setting=plate_block.header.kinetic_points,
-        error_document=first_data_element.error_document,
+        error_document=error_documents,
         measurement_custom_info=first_data_element.custom_info,
     )
 
@@ -154,24 +183,39 @@ def _create_measurements(plate_block: PlateBlock, position: str) -> list[Measure
         MeasurementType.EMISSION_LUMINESCENCE_CUBE_SPECTRUM,
         MeasurementType.EXCITATION_LUMINESCENCE_CUBE_SPECTRUM,
     ):
-        return [_create_spectrum_measurement(plate_block, data_elements)]
+        measurement = _create_spectrum_measurement(plate_block, data_elements)
+        if not measurement:
+            return []
+        return [measurement]
 
     return [
         Measurement(
             type_=measurement_type,
             identifier=data_element.uuid,
             absorbance=(
-                data_element.value
+                (
+                    data_element.value
+                    if data_element.value is not None
+                    else NEGATIVE_ZERO
+                )
                 if measurement_type == MeasurementType.ULTRAVIOLET_ABSORBANCE
                 else None
             ),
             fluorescence=(
-                data_element.value
+                (
+                    data_element.value
+                    if data_element.value is not None
+                    else NEGATIVE_ZERO
+                )
                 if measurement_type == MeasurementType.FLUORESCENCE
                 else None
             ),
             luminescence=(
-                data_element.value
+                (
+                    data_element.value
+                    if data_element.value is not None
+                    else NEGATIVE_ZERO
+                )
                 if measurement_type == MeasurementType.LUMINESCENCE
                 else None
             ),

src/allotropy/parsers/moldev_softmax_pro/softmax_pro_structure.py

@@ -373,7 +373,7 @@ class DataElement:
     temperature: float | None
     wavelength: float
     position: str
-    value: float
+    value: float | None
     error_document: list[ErrorDocument]
     custom_info: dict[str, str] = field(default_factory=dict)
     elapsed_time: list[float] = field(default_factory=list)
@@ -410,21 +410,19 @@ def create(
             for row_idx, *row_data in df_data.itertuples()
             for col, raw_value in zip(df_data.columns, row_data, strict=True)
         }
-
         data_elements = {}
         for position, raw_value in data.items():
             value = try_non_nan_float_or_none(raw_value)
             if value is None and elapsed_time is not None:
                 msg = f"Missing kinetic measurement for well position {position} at {elapsed_time}s."
                 raise AllotropeConversionError(msg)
-
             data_elements[str(position)] = DataElement(
                 uuid=random_uuid_str(),
                 plate=header.name,
                 temperature=temperature,
                 wavelength=wavelength,
                 position=str(position),
-                value=NEGATIVE_ZERO if value is None else value,
+                value=value,
                 error_document=(
                     [ErrorDocument(str(raw_value), header.read_mode)]
                     if value is None
@@ -475,6 +473,13 @@ def get_measurement_section(
             reader.lines_as_df(lines=lines, sep="\t"),
             msg="unable to find data from plate block.",
         )
+
+        # Truncate columns Series to match the actual number of data columns
+        if len(columns) >= data.shape[1]:
+            columns = columns.iloc[: data.shape[1]]
+        elif len(columns) < data.shape[1]:
+            data = data.loc[:, : len(columns) - 1]
+
         set_columns(data, columns)
         return data
 
@@ -559,7 +564,7 @@ def _update_kinetic_data(
 
 @dataclass(frozen=True)
 class SpectrumRawPlateData(RawData):
-    maximum_wavelength_signal: dict[str, float]
+    maximum_wavelength_signal: dict[str, float | None]
 
     @staticmethod
     def create(reader: CsvReader, header: PlateHeader) -> SpectrumRawPlateData:
@@ -594,9 +599,7 @@ def create(reader: CsvReader, header: PlateHeader) -> SpectrumRawPlateData:
             reader, columns, rows
         ).iloc[:, 2:]
         signal_data = {
-            f"{num_to_chars(row_idx)}{col}": try_float(
-                str(raw_value), "wavelength signal"
-            )
+            f"{num_to_chars(row_idx)}{col}": try_float_or_none(str(raw_value))
             for row_idx, *row_data in max_wavelength_signal_data.itertuples()
             for col, raw_value in zip(
                 max_wavelength_signal_data.columns, row_data, strict=True
@@ -665,8 +668,14 @@ def create(
         )
 
     def iter_data_elements(self, position: str) -> Iterator[DataElement]:
-        for wavelength_data in self.raw_data.wavelength_data:
-            yield wavelength_data.data_elements[position]
+        for plate_wavelength_data in self.raw_data.wavelength_data:
+            yield plate_wavelength_data.data_elements[position]
+
+    def position_exists(self, position: str) -> bool:
+        for plate_wavelength_data in self.raw_data.wavelength_data:
+            if position in plate_wavelength_data.data_elements:
+                return True
+        return False
 
 
 @dataclass(frozen=True)
@@ -693,7 +702,7 @@ def create(
                 temperature=temperature,
                 wavelength=wavelength,
                 position=str(position),
-                value=NEGATIVE_ZERO if value is None else value,
+                value=value,
                 error_document=error_document,
             )
 
@@ -799,10 +808,19 @@ def create(
         )
 
     def iter_data_elements(self, position: str) -> Iterator[DataElement]:
-        for wavelength_data in self.raw_data.wavelength_data:
-            for measurement_data in wavelength_data.measurement_data:
+        for time_wavelength_data in self.raw_data.wavelength_data:
+            for measurement_data in time_wavelength_data.measurement_data:
                 yield measurement_data.data_elements[position]
 
+    def position_exists(self, position: str) -> bool:
+        for time_wavelength_data in self.raw_data.wavelength_data:
+            if any(
+                position in measurement.data_elements
+                for measurement in time_wavelength_data.measurement_data
+            ):
+                return True
+        return False
+
 
 @dataclass(frozen=True)
 class PlateBlock(ABC, Block):
@@ -915,7 +933,9 @@ def iter_wells(self) -> Iterator[str]:
         cols, rows = NUM_WELLS_TO_PLATE_DIMENSIONS[self.header.num_wells]
         for row in range(rows):
             for col in range(1, cols + 1):
-                yield f"{num_to_chars(row)}{col}"
+                position = f"{num_to_chars(row)}{col}"
+                if self.block_data.position_exists(position):
+                    yield position
 
     def iter_data_elements(self, position: str | list[str]) -> Iterator[DataElement]:
         position = [position] if isinstance(position, str) else position
@@ -1279,12 +1299,26 @@ def create(reader: CsvReader) -> StructureData:
                         # experiment, there is no way at the moment to link the group data with a plate.
                         continue
                     plate_block = block_list.plate_blocks[group_data_element.plate]
+                    is_spectrum = (
+                        plate_block.header.read_type == ReadType.SPECTRUM.value
+                    )
+                    processed_positions = set()
+
                     for data_element in plate_block.iter_data_elements(
                         group_data_element.positions
                     ):
                         data_element.group_id = group_block.group_data.name
                         data_element.sample_id = group_data_element.sample
-                        data_element.error_document += group_data_element.errors
+
+                        # For spectrum measurements, only add errors to the first DataElement per well
+                        if (
+                            not is_spectrum
+                            or data_element.position not in processed_positions
+                        ):
+                            data_element.error_document += group_data_element.errors
+                            if is_spectrum:
+                                processed_positions.add(data_element.position)
+
                         data_element.custom_info.update(group_data_element.custom_info)
 
         return StructureData(

tests/parsers/moldev_softmax_pro/testdata/MD_SMP_absorbance_endpoint_partial_plate_example02.json

@@ -20337,7 +20337,7 @@
             "UNC path": "tests/parsers/moldev_softmax_pro/testdata/MD_SMP_absorbance_endpoint_partial_plate_example02.txt",
             "file name": "MD_SMP_absorbance_endpoint_partial_plate_example02.txt",
             "ASM converter name": "allotropy_molecular_devices_softmax_pro",
-            "ASM converter version": "0.1.92",
+            "ASM converter version": "0.1.93",
             "software name": "SoftMax Pro"
         },
         "device system document": {

tests/parsers/moldev_softmax_pro/testdata/MD_SMP_absorbance_endpoint_partial_plate_example05.json

@@ -15965,7 +15965,7 @@
             "UNC path": "tests/parsers/moldev_softmax_pro/testdata/MD_SMP_absorbance_endpoint_partial_plate_example05.txt",
             "file name": "MD_SMP_absorbance_endpoint_partial_plate_example05.txt",
             "ASM converter name": "allotropy_molecular_devices_softmax_pro",
-            "ASM converter version": "0.1.92",
+            "ASM converter version": "0.1.93",
             "software name": "SoftMax Pro"
         },
         "device system document": {

tests/parsers/moldev_softmax_pro/testdata/MD_SMP_fluorescence_endpoint_partial_plate_example02.json

@@ -18557,7 +18557,7 @@
             "UNC path": "tests/parsers/moldev_softmax_pro/testdata/MD_SMP_fluorescence_endpoint_partial_plate_example02.txt",
             "file name": "MD_SMP_fluorescence_endpoint_partial_plate_example02.txt",
             "ASM converter name": "allotropy_molecular_devices_softmax_pro",
-            "ASM converter version": "0.1.92",
+            "ASM converter version": "0.1.93",
             "software name": "SoftMax Pro"
         },
         "device system document": {

tests/parsers/moldev_softmax_pro/testdata/MD_SMP_luminescence_endpoint_partial_plate_example02.json

@@ -17117,7 +17117,7 @@
             "UNC path": "tests/parsers/moldev_softmax_pro/testdata/MD_SMP_luminescence_endpoint_partial_plate_example02.txt",
             "file name": "MD_SMP_luminescence_endpoint_partial_plate_example02.txt",
             "ASM converter name": "allotropy_molecular_devices_softmax_pro",
-            "ASM converter version": "0.1.92",
+            "ASM converter version": "0.1.93",
             "software name": "SoftMax Pro"
         },
         "device system document": {