Merge branch 'main' into felipenarv/qiacuity_dpcr_get_unread

Author: felipenarv

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this packages will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.1.108] - 2025-10-23
+
+### Added
+
+- BMG Labtech MARS - Support for CSVs generated by Mars V3.40 R2 (#1105)
+
 ## [0.1.107] - 2025-10-09
 
 ### Added

src/allotropy/__about__.py

@@ -1 +1 @@
-__version__ = "0.1.107"
+__version__ = "0.1.108"

src/allotropy/allotrope/schema_mappers/adm/plate_reader/rec/_2025/_03/plate_reader.py

@@ -220,7 +220,8 @@ class Metadata:
     equipment_serial_number: str | None = None
     product_manufacturer: str | None = None
     file_name: str | None = None
-    custom_info_doc: dict[str, Any] | None = None
+    metadata_custom_info: dict[str, Any] | None = None
+    device_control_custom_info: dict[str, Any] | None = None
 
 
 @dataclass(frozen=True)
@@ -253,7 +254,7 @@ def map_model(self, data: Data) -> Model:
                         ASM_converter_name=self.converter_name,
                         ASM_converter_version=ASM_CONVERTER_VERSION,
                     ),
-                    data.metadata.custom_info_doc,
+                    data.metadata.metadata_custom_info,
                 ),
                 plate_reader_document=[
                     self._get_technique_document(measurement_group)

src/allotropy/parsers/bmg_mars/bmg_mars_reader.py

@@ -1,3 +1,4 @@
+import csv
 from io import StringIO
 
 import pandas as pd
@@ -16,6 +17,15 @@ class BmgMarsReader:
     header_content: str
 
     def __init__(self, named_file_contents: NamedFileContents) -> None:
+        try:
+            self._parse_file(named_file_contents)
+        except pd.errors.ParserError:
+            # Fix malformed CSV files with uneven columns and retry
+            named_file_contents = self._normalize_csv_columns(named_file_contents)
+            self._parse_file(named_file_contents)
+
+    def _parse_file(self, named_file_contents: NamedFileContents) -> None:
+        """Parse the BMG MARS file and populate header and data attributes."""
         reader = CsvReader(read_to_lines(named_file_contents))
         lines = list(reader.pop_until_inclusive("^,?Raw Data"))
         # Store the header contents so we can parse some values that don't have key/value
@@ -28,7 +38,6 @@ def __init__(self, named_file_contents: NamedFileContents) -> None:
             axis="index"
         )
         new = df["value"].str.split(": ", expand=True, n=1)
-
         # Handle the case where no ": " delimiter is found, resulting in a DataFrame with only one column
         if new.shape[1] < 2:
             msg = "Unable to parse header data: no key-value pairs found with expected format."
@@ -41,3 +50,67 @@ def __init__(self, named_file_contents: NamedFileContents) -> None:
             reader.pop_csv_block_as_df(header=0, index_col=0),
             msg="Unable to parse dataset from file.",
         )
+
+    def _normalize_csv_columns(
+        self, named_file_contents: NamedFileContents
+    ) -> NamedFileContents:
+        """Fix malformed CSV files with uneven number of columns per row.
+
+        BMG MARS software <= v4.0 can produce CSV files where some rows have
+        more or fewer columns than others. This method normalizes all rows
+        to have the same number of columns by padding with empty strings,
+        allowing the csv reader to parse the file correctly.
+
+        Args:
+            named_file_contents: The named file contents to fix
+
+        Returns:
+            Fixed NamedFileContents with consistent column counts
+        """
+        lines = read_to_lines(named_file_contents)
+
+        if not lines:
+            return named_file_contents
+
+        # Parse all rows and find the maximum number of columns
+        csv_rows = []
+        max_columns = 0
+
+        for line in lines:
+            if line.strip():
+                reader = csv.reader([line])
+                try:
+                    row = next(reader)
+                    csv_rows.append(row)
+                    max_columns = max(max_columns, len(row))
+                except (csv.Error, StopIteration):
+                    # If parsing fails, keep the original line
+                    csv_rows.append([line])
+                    max_columns = max(max_columns, 1)
+            else:
+                pass  # skip empty lines
+
+        # Normalize all rows to have the same number of columns
+        normalized_rows = []
+        for row in csv_rows:
+            normalized_row = row.copy()
+            if len(normalized_row) < max_columns:
+                # Pad with empty strings
+                normalized_row.extend([""] * (max_columns - len(normalized_row)))
+            elif len(normalized_row) > max_columns:
+                # Truncate if somehow longer (shouldn't happen with max calculation)
+                normalized_row = normalized_row[:max_columns]
+            normalized_rows.append(normalized_row)
+
+        # Convert back to CSV string
+        output = StringIO()
+        writer = csv.writer(output)
+        writer.writerows(normalized_rows)
+        output.seek(0)
+
+        # Create a new NamedFileContents with the fixed content
+        return NamedFileContents(
+            contents=output,
+            original_file_path=named_file_contents.original_file_path,
+            encoding="utf-8",
+        )

src/allotropy/parsers/thermo_skanit/thermo_skanit_structure.py

@@ -80,22 +80,66 @@ def create_metadata(
         )
         general_info_data = ThermoSkanItMetadata._get_general_info_data(general_info_df)
         path = Path(file_path)
+
+        device_identifier = str(instrument_info_data.pop("device_identifier"))
+        model_number = str(instrument_info_data.pop("model_number"))
+
+        equipment_serial_number = None
+        if "equipment_serial_number" in instrument_info_data:
+            equipment_serial_number = str(
+                instrument_info_data.pop("equipment_serial_number")
+            )
+
+        software_name = None
+        if general_info_data.get("software_name") is not None:
+            software_name = str(general_info_data.pop("software_name"))
+        else:
+            general_info_data.pop("software_name", None)
+
+        software_version = None
+        if general_info_data.get("software_version") is not None:
+            software_version = str(general_info_data.pop("software_version"))
+        else:
+            general_info_data.pop("software_version", None)
+
+        # Merge custom_info dicts, ensuring we only unpack dict types
+        instrument_custom_info = instrument_info_data.get("custom_info", {})
+        device_control_custom_info_raw = instrument_info_data.get(
+            "device_control_custom_info"
+        )
+        general_custom_info = general_info_data.get("custom_info", {})
+
+        custom_info: dict[str, Any] = {}
+        if isinstance(instrument_custom_info, dict):
+            custom_info.update(instrument_custom_info)
+        if isinstance(general_custom_info, dict):
+            custom_info.update(general_custom_info)
+
+        # Ensure device_control_custom_info is the correct type
+        device_control_custom_info = (
+            device_control_custom_info_raw
+            if isinstance(device_control_custom_info_raw, dict)
+            else None
+        )
+
         return Metadata(
             asm_file_identifier=path.with_suffix(".json").name,
             data_system_instance_id=NOT_APPLICABLE,
             file_name=path.name,
             unc_path=file_path,
-            device_identifier=instrument_info_data["device_identifier"],
-            model_number=instrument_info_data["model_number"],
-            equipment_serial_number=instrument_info_data.get("equipment_serial_number"),
-            software_name=general_info_data["software_name"],
-            software_version=general_info_data["software_version"],
+            device_identifier=device_identifier,
+            model_number=model_number,
+            equipment_serial_number=equipment_serial_number,
+            software_name=software_name,
+            software_version=software_version,
+            metadata_custom_info=custom_info,
+            device_control_custom_info=device_control_custom_info,
         )
 
     @staticmethod
     def _get_general_info_data(
         general_info_df: pd.DataFrame | None,
-    ) -> dict[str, str | None]:
+    ) -> dict[str, str | None | dict[str, str]]:
         if general_info_df is None:
             return dict.fromkeys(GENERAL_INFO_KEYS, None)
 
@@ -106,15 +150,21 @@ def _get_general_info_data(
         software_name, software_version_txt = software_info.split(",", 1)
         match = re.search(r"\d+(?:\.\d+)*", software_version_txt)
         software_version = match.group() if match else None
+        unread_data = general_info_data.get_unread(skip={"Filter", "Position"})
+        # Filter out "nan" keys from custom info and ensure string values
+        custom_info = {
+            k: str(v) for k, v in unread_data.items() if k != "nan" and v is not None
+        }
         return {
             "software_name": software_name,
             "software_version": software_version,
+            "custom_info": custom_info,
         }
 
     @staticmethod
     def _get_instrument_data(
         instrument_info_df: pd.DataFrame | None,
-    ) -> dict[str, str]:
+    ) -> dict[str, str | None | dict[str, str] | dict[str, float | str | None]]:
         if instrument_info_df is None:
             return {
                 "device_identifier": NOT_APPLICABLE,
@@ -139,11 +189,45 @@ def _get_instrument_data(
             "model_number": ("Name", NOT_APPLICABLE),
             "equipment_serial_number": ("Serial number", None),
         }
-        return {
+        # Get the raw data first
+        raw_data = {
             key: value
             for key, (label, default) in lookups.items()
             if (value := instrument_info_data.get(str, label, default)) is not None
         }
+        device_control_custom_info = instrument_info_data.get_custom_keys(
+            {
+                "Optical response compensation",
+                "Plate adapter number",
+                "Plate adapter name",
+            }
+        )
+        unread_data = instrument_info_data.get_unread(
+            skip={
+                "Wavelength",
+                "Incubator",
+                "Date and time of definition",
+                "Top optics",
+                "Bottom optics",
+                "Dispenser 1",
+                "Dispenser 2",
+                "Module's name",
+                "Module's serial number",
+                "Instrument modules",
+                "Gas control",
+                "Bandwidth",
+                "Position",
+                "Filter",
+            }
+        )
+        custom_info = {
+            k: str(v) for k, v in unread_data.items() if k != "nan" and v is not None
+        }
+        return {
+            **raw_data,
+            "custom_info": custom_info,
+            "device_control_custom_info": device_control_custom_info,
+        }
 
 
 @dataclass
@@ -158,6 +242,7 @@ def create(
         well_plate_identifier: str | None,
         error_documents: list[ErrorDocument] | None = None,
         experimental_data_identifier: str | None = None,
+        device_control_custom_info: dict[str, Any] | None = None,
     ) -> Measurement:
         measurement_type_str = MEASUREMENT_TYPES[type_]
         error_docs = error_documents or []
@@ -184,6 +269,7 @@ def create(
             else None,
             experimental_data_identifier=experimental_data_identifier,
             error_document=error_docs if error_docs else None,
+            device_control_custom_info=device_control_custom_info,
         )
 
     @staticmethod
@@ -202,14 +288,19 @@ def create(
         sheet_df: pd.DataFrame,
         type_: MeasurementType,
         session_info_df: pd.DataFrame | None,
+        device_control_custom_info: dict[str, Any] | None = None,
     ) -> list[MeasurementGroup]:
         plates = ThermoSkanItMeasurementGroups.identify_data_and_sample_dfs(sheet_df)
 
         session_name = exec_time = None
+        custom_info = {}
         if session_info_df is not None:
             session_info_data = df_to_series_data(parse_header_row(session_info_df.T))
             session_name = session_info_data.get(str, "Session name")
             exec_time = session_info_data.get(str, "Execution time")
+            unread_data = session_info_data.get_unread(skip={"Executed with"})
+            # Filter out "nan" keys from custom info
+            custom_info = {k: v for k, v in unread_data.items() if k != "nan"}
 
         if not exec_time:
             exec_time = sheet_df.iloc[1].iloc[0]
@@ -273,6 +364,7 @@ def create(
                         experimental_data_identifier=session_name.replace(".skax", "")
                         if session_name
                         else None,
+                        device_control_custom_info=device_control_custom_info,
                     )
 
                     if well_key not in well_measurements:
@@ -288,6 +380,7 @@ def create(
                     measurements=measurements,
                     plate_well_count=plate_well_counts[plate_id],
                     measurement_time=exec_time,
+                    custom_info=custom_info,
                 )
             )
 
@@ -458,10 +551,15 @@ def create(sheet_data: dict[str, pd.DataFrame], file_path: str) -> Data:
             general_info_df=clean_data.get("General information"),
             file_path=file_path,
         )
+
+        # Extract device control custom info from metadata
+        device_control_custom_info = metadata.device_control_custom_info
+
         measurement_groups = ThermoSkanItMeasurementGroups.create(
             sheet_df=measurement_df,
             session_info_df=clean_data.get("Session information"),
             type_=_type,
+            device_control_custom_info=device_control_custom_info,
         )
 
         return Data(

src/allotropy/parsers/unchained_labs_lunatic_stunner/unchained_labs_lunatic_stunner_structure.py

@@ -463,7 +463,7 @@ def create_metadata(header: SeriesData, file_path: str) -> Metadata:
         unc_path=file_path,
         asm_file_identifier=asm_file_identifier.name,
         data_system_instance_id=NOT_APPLICABLE,
-        custom_info_doc=_filter_empty_string_values(
+        metadata_custom_info=_filter_empty_string_values(
             header.get_unread(
                 # Skip already mapped columns from well plate data (repeated in CSV no header cases)
                 skip={

tests/parsers/bmg_mars/testdata/MARS V3.40 V2 Output Missing line end separators.csv

@@ -0,0 +1,25 @@
+User: USER,Path: C:\Program Files (x86)\BMG\PHERAstar\User\Data,Test run no.: 4
+Test name: Transcreener ADP2 FI,Date: 29/02/2016,Time: 14:34:46
+
+ID1: black 384w small volume,ID2: 20 ul
+Fluorescence (FI)
+
+Raw Data (580/620)
+
+,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24
+A,25104,25203,24886,24098,24466,26571,24056,24156,24130,24089,21634,21969,22160,23823,23019,22230,22992,23852,23657,24361
+B,33863,33227,34758,33603,34186,34318,34196,35279,35608,37342,35468,34056,34031,33235,33242,33304,35286,34482,34378,33774
+C,45480,42551,41891,44773,43821,45338,43328,42901,44137,46425,39341,39162,39791,41566,41265,43889,40461,41195,40761,40262
+D,50970,50862,50279,50797,51063,50202,50808,50966,50097,49497,50325,48821,48954,49208,49181,49016,49404,49271,48856,47830
+E,53997,52824,53844,50679,55121,54280,53994,54413,54396,53971,53444,50748,53143,53916,53624,53156,53531,53493,53216,53154
+F,55774,56191,56164,56462,55583,56253,56300,55464,56719,54927,56253,54441,58358,56279,55870,55886,55465,55897,55737,53798
+G,57378,57180,57681,58158,57949,57397,57546,57895,58077,57617,57827,57072,58385,58021,58624,57891,57835,58321,58046,55748
+H,59660,58847,59844,60655,61534,60652,60090,60958,59683,59631,59656,58141,59493,59990,59664,59956,59928,60272,59780,58810
+I,63183,57522,58573,60240,60618,60777,61149,61504,61682,60194,61840,61912,61256,63237,61643,60385,60806,60566,60231,58425
+J,62375,61292,62851,65025,64750,65573,58147,64039,63962,64279,64763,60451,64021,64272,65055,63228,61028,61503,61899,60751
+K,63255,63635,63048,63799,65845,66218,65161,63773,64427,62842,65316,64136,65885,65606,65812,66198,62934,63653,64950,62926
+L,66220,62882,65007,64797,65385,66359,66227,67120,65709,64827,65253,61630,66214,65540,65774,67329,65779,65519,65672,63404
+M,63792,64859,66378,66579,67777,67381,67099,67971,66080,64760,67145,65628,66682,66075,66363,65642,67384,65322,63597,64895
+N,64497,64888,64056,65284,66138,66751,66896,66044,66878,62160,64748,64183,65883,65153,66413,63789,63507,62883,63665,63882
+O,68426,60666,65656,66138,63289,107,103,98,100,94,,59
+P,63471,62852,62886,63555,63035,79,90,100,102,103,,62