feat: MSD Workbench - Update support for CSV format and Calc Conc CV handling (#1157)

Changes:
- Updated msd_workbench_reader.py to detect and handle both CSV formats:
  - Old format: Plate ID with trailing commas
- New format: Plate ID without commas (line 1 = plate ID, line 2 =
headers)
- Added support for "Calc. Conc. CV" calculated data column
- Improved column name handling to support both "Dilution Factor" and
"Dilution"
- "Calc. Conc. CV" now properly appears in calculated data documents
- "Dilution" field now correctly consumed by dilution factor setting

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: james-leinas

src/allotropy/calcdocs/msd_workbench/extractor.py

@@ -39,5 +39,6 @@ def to_element(
                 "percent_recovery_mean": calc_data_structure.percent_recovery_mean,
                 "calc_concentration": calc_data_structure.calc_concentration,
                 "calc_conc_mean": calc_data_structure.calc_conc_mean,
+                "calc_conc_cv": calc_data_structure.calc_conc_cv,
             },
         )

src/allotropy/parsers/msd_workbench/calculdated_data_structure.py

@@ -17,3 +17,4 @@ class CalculatedDataMeasurementStructure:
     percent_recovery_mean: float | None
     calc_concentration: float | None
     calc_conc_mean: float | None
+    calc_conc_cv: float | None

src/allotropy/parsers/msd_workbench/msd_workbench_calculated_data_mapping.py

@@ -44,6 +44,7 @@ class CalculatedDataColumns(Enum):
     PERCENT_RECOVERY_MEAN = "% Recovery Mean"
     CALC_CONCENTRATION = "Calc. Concentration"
     CALC_CONC_MEAN = "Calc. Conc. Mean"
+    CALC_CONC_CV = "Calc. Conc. CV"
 
 
 def create_calculated_data_groups(
@@ -84,6 +85,9 @@ def create_calculated_data_groups(
                     calc_conc_mean=row_series.get(
                         float, CalculatedDataColumns.CALC_CONC_MEAN.value
                     ),
+                    calc_conc_cv=row_series.get(
+                        float, CalculatedDataColumns.CALC_CONC_CV.value
+                    ),
                 )
             )
             # we do not need additional data for the calculated data documents
@@ -156,6 +160,12 @@ def create_calculated_data_groups(
         view_data=assay_view,
         source_configs=(calc_concentration_config,),
     )
+    calc_conc_cv_config = CalculatedDataConfig(
+        name=CalculatedDataColumns.CALC_CONC_CV.value,
+        value="calc_conc_cv",
+        view_data=assay_view,
+        source_configs=(calc_concentration_config,),
+    )
     configs = CalcDocsConfig(
         [
             mean_config,
@@ -167,6 +177,7 @@ def create_calculated_data_groups(
             percent_recovery_mean_config,
             calc_concentration_config,
             calc_conc_mean_config,
+            calc_conc_cv_config,
         ]
     )
     calc_docs = [

src/allotropy/parsers/msd_workbench/msd_workbench_reader.py

@@ -18,18 +18,58 @@ def __init__(self, named_file_contents: NamedFileContents) -> None:
                 sheet_name="Workbench data",
                 engine="openpyxl",
             )
+            data = data.where(pd.notna(data), None)
+            data.index = pd.Index(data.index.to_series().ffill())
+            data.index = data.index.astype(str).str.strip()
+            data.columns = data.columns.astype(str).str.strip()
+            first_row = str(data.iloc[0].index[0])
+            if "Plate" not in first_row:
+                msg = "Plate ID not found in the first row of the data"
+                raise AllotropeConversionError(msg)
+            self.well_plate_id = first_row.split("_")[-1].strip()
+            # Set the first row as the header
+            data.columns = pd.Index(data.iloc[0])
+            data = data[1:].reset_index(drop=True)
         else:
-            data = read_csv(named_file_contents.contents)
-        data = data.where(pd.notna(data), None)
-        data.index = pd.Index(data.index.to_series().ffill())
-        data.index = data.index.astype(str).str.strip()
-        data.columns = data.columns.astype(str).str.strip()
-        first_row = str(data.iloc[0].index[0])
-        if "Plate" not in first_row:
-            msg = "Plate ID not found in the first row of the data"
-            raise AllotropeConversionError(msg)
-        self.well_plate_id = first_row.split("_")[-1].strip()
-        # Set the first row as the header
-        data.columns = pd.Index(data.iloc[0])
-        data = data[1:].reset_index(drop=True)
+            # First, peek at the first line to check format
+            named_file_contents.contents.seek(0)
+            first_line = named_file_contents.contents.readline()
+            if isinstance(first_line, bytes):
+                first_line = first_line.decode("utf-8")
+            named_file_contents.contents.seek(0)
+
+            # Check if first line has commas (old format) or not (new format)
+            if "," in first_line:
+                # Old format: Plate_ID,,,,,,,,
+                data = read_csv(named_file_contents.contents)
+                data = data.where(pd.notna(data), None)
+                data.index = pd.Index(data.index.to_series().ffill())
+                data.index = data.index.astype(str).str.strip()
+                data.columns = data.columns.astype(str).str.strip()
+                first_row = str(data.iloc[0].index[0])
+                if "Plate" not in first_row:
+                    msg = "Plate ID not found in the first row of the data"
+                    raise AllotropeConversionError(msg)
+                self.well_plate_id = first_row.split("_")[-1].strip()
+                # Set the first row as the header
+                data.columns = pd.Index(data.iloc[0])
+                data = data[1:].reset_index(drop=True)
+            else:
+                # New format: Plate_ID (no commas on first line)
+                # Extract plate ID from first line
+                plate_id_line = first_line.strip()
+                if "Plate" not in plate_id_line:
+                    msg = "Plate ID not found in the first row of the data"
+                    raise AllotropeConversionError(msg)
+                self.well_plate_id = plate_id_line.split("_")[-1].strip()
+
+                # Now read the CSV starting from the second line (which has headers)
+                named_file_contents.contents.seek(0)
+                # Skip the first line
+                named_file_contents.contents.readline()
+                # Read rest of file with second line as header
+                data = read_csv(named_file_contents.contents)
+                data = data.where(pd.notna(data), None)
+                data.columns = data.columns.astype(str).str.strip()
+
         self.plate_data = data

src/allotropy/parsers/msd_workbench/msd_workbench_structure.py

@@ -84,6 +84,11 @@ def map_measurement(row: SeriesData) -> Measurement:
             "detection range": row.get(str, "Detection Range"),
             "assay identifier": row.get(str, "Assay"),
         }
+        # Try both "Dilution Factor" and "Dilution" for backwards compatibility
+        dilution_value = row.get(int, "Dilution Factor")
+        if dilution_value is None:
+            dilution_value = row.get(int, "Dilution")
+
         return Measurement(
             type_=MeasurementType.LUMINESCENCE,
             identifier=random_uuid_str(),
@@ -99,7 +104,7 @@ def map_measurement(row: SeriesData) -> Measurement:
                 row[str, "Sample"][0].lower()
             ),
             sample_custom_info={
-                "dilution factor setting": row.get(int, "Dilution Factor"),
+                "dilution factor setting": dilution_value,
             },
             measurement_custom_info={
                 **custom_info,

tests/parsers/msd_workbench/testdata/msd_workbench_test_2.csv

@@ -0,0 +1,50 @@
+Plate_TEST001
+Sample,Assay,Well,Spot,Dilution,Concentration,Signal,Adjusted Signal,Mean,Adj. Sig. Mean,CV,% Recovery,% Recovery Mean,Calc. Concentration,Calc. Conc. Mean,Calc. Conc. CV
+S001,Test Assay 1.0,A01,1,,100.0,1000.0,1000.0,1010.0,1010.0,1.5,102.5,101.0,102.0,101.0,1.2
+S001,Test Assay 1.0,A02,1,,100.0,1020.0,1020.0,1010.0,1010.0,1.5,99.5,101.0,100.0,101.0,1.2
+U001,Test Assay 1.0,A03,1,1.0,,2000.0,2000.0,2010.0,2010.0,0.8,,,200.0,201.0,0.6
+U001,Test Assay 1.0,A04,1,1.0,,2020.0,2020.0,2010.0,2010.0,0.8,,,202.0,201.0,0.6
+U009,Test Assay 1.0,A05,1,1.0,,3000.0,3000.0,3010.0,3010.0,1.2,,,300.0,301.0,1.0
+U009,Test Assay 1.0,A06,1,1.0,,3020.0,3020.0,3010.0,3010.0,1.2,,,302.0,301.0,1.0
+U017,Test Assay 1.0,A07,1,1.0,,4000.0,4000.0,4010.0,4010.0,0.5,,,400.0,401.0,0.4
+U017,Test Assay 1.0,A08,1,1.0,,4020.0,4020.0,4010.0,4010.0,0.5,,,402.0,401.0,0.4
+S002,Test Assay 1.0,B01,1,,50.0,500.0,500.0,510.0,510.0,1.8,99.0,98.5,49.5,49.0,1.5
+S002,Test Assay 1.0,B02,1,,50.0,520.0,520.0,510.0,510.0,1.8,98.0,98.5,48.5,49.0,1.5
+U002,Test Assay 1.0,B03,1,1.0,,1500.0,1500.0,1510.0,1510.0,1.0,,,150.0,151.0,0.9
+U002,Test Assay 1.0,B04,1,1.0,,1520.0,1520.0,1510.0,1510.0,1.0,,,152.0,151.0,0.9
+U010,Test Assay 1.0,B05,1,1.0,,2500.0,2500.0,2510.0,2510.0,0.6,,,250.0,251.0,0.5
+U010,Test Assay 1.0,B06,1,1.0,,2520.0,2520.0,2510.0,2510.0,0.6,,,252.0,251.0,0.5
+U018,Test Assay 1.0,B07,1,1.0,,3500.0,3500.0,3510.0,3510.0,0.4,,,350.0,351.0,0.3
+U018,Test Assay 1.0,B08,1,1.0,,3520.0,3520.0,3510.0,3510.0,0.4,,,352.0,351.0,0.3
+S003,Test Assay 1.0,C01,1,,25.0,250.0,250.0,255.0,255.0,0.1,98.0,98.5,24.5,24.7,0.1
+S003,Test Assay 1.0,C02,1,,25.0,260.0,260.0,255.0,255.0,0.1,99.0,98.5,24.9,24.7,0.1
+U003,Test Assay 1.0,C03,1,1.0,,1000.0,1000.0,1010.0,1010.0,0.7,,,100.0,101.0,0.6
+U003,Test Assay 1.0,C04,1,1.0,,1020.0,1020.0,1010.0,1010.0,0.7,,,102.0,101.0,0.6
+U011,Test Assay 1.0,C05,1,1.0,,2000.0,2000.0,2010.0,2010.0,0.9,,,200.0,201.0,0.8
+U011,Test Assay 1.0,C06,1,1.0,,2020.0,2020.0,2010.0,2010.0,0.9,,,202.0,201.0,0.8
+U019,Test Assay 1.0,C07,1,1.0,,3000.0,3000.0,3010.0,3010.0,0.2,,,300.0,301.0,0.2
+U019,Test Assay 1.0,C08,1,1.0,,3020.0,3020.0,3010.0,3010.0,0.2,,,302.0,301.0,0.2
+S004,Test Assay 1.0,D01,1,,12.5,125.0,125.0,130.0,130.0,2.5,103.0,100.5,12.9,12.6,3.0
+S004,Test Assay 1.0,D02,1,,12.5,135.0,135.0,130.0,130.0,2.5,98.0,100.5,12.3,12.6,3.0
+U004,Test Assay 1.0,D03,1,1.0,,500.0,500.0,510.0,510.0,0.3,,,50.0,51.0,0.2
+U004,Test Assay 1.0,D04,1,1.0,,520.0,520.0,510.0,510.0,0.3,,,52.0,51.0,0.2
+U012,Test Assay 1.0,D05,1,1.0,,1500.0,1500.0,1510.0,1510.0,1.3,,,150.0,151.0,1.2
+U012,Test Assay 1.0,D06,1,1.0,,1520.0,1520.0,1510.0,1510.0,1.3,,,152.0,151.0,1.2
+U020,Test Assay 1.0,D07,1,1.0,,2500.0,2500.0,2510.0,2510.0,1.6,,,250.0,251.0,2.5
+U020,Test Assay 1.0,D08,1,1.0,,2520.0,2520.0,2510.0,2510.0,1.6,,,252.0,251.0,2.5
+S005,Test Assay 1.0,E01,1,,6.25,62.5,62.5,65.0,65.0,2.2,98.5,102.0,6.15,6.35,4.5
+S005,Test Assay 1.0,E02,1,,6.25,67.5,67.5,65.0,65.0,2.2,105.5,102.0,6.55,6.35,4.5
+U005,Test Assay 1.0,E03,1,1.0,,250.0,250.0,255.0,255.0,0.2,,,25.0,25.5,0.1
+U005,Test Assay 1.0,E04,1,1.0,,260.0,260.0,255.0,255.0,0.2,,,26.0,25.5,0.1
+S006,Test Assay 1.0,F01,1,,3.125,31.25,31.25,32.5,32.5,1.0,103.5,101.0,3.24,3.16,2.8
+S006,Test Assay 1.0,F02,1,,3.125,33.75,33.75,32.5,32.5,1.0,98.5,101.0,3.08,3.16,2.8
+U006,Test Assay 1.0,F03,1,1.0,,125.0,125.0,127.5,127.5,0.05,,,12.5,12.75,0.04
+U006,Test Assay 1.0,F04,1,1.0,,130.0,130.0,127.5,127.5,0.05,,,13.0,12.75,0.04
+S007,Test Assay 1.0,G01,1,,1.5625,15.625,15.625,16.25,16.25,4.2,130.0,109.0,2.03,1.88,20.0
+S007,Test Assay 1.0,G02,1,,1.5625,16.875,16.875,16.25,16.25,4.2,88.0,109.0,1.73,1.88,20.0
+U007,Test Assay 1.0,G03,1,1.0,,62.5,62.5,65.0,65.0,0.2,,,6.25,6.5,0.3
+U007,Test Assay 1.0,G04,1,1.0,,67.5,67.5,65.0,65.0,0.2,,,6.75,6.5,0.3
+S008,Test Assay 1.0,H01,1,,0.0,7.8,7.8,8.1,8.1,2.0,,NaN,NaN,NaN,NaN
+S008,Test Assay 1.0,H02,1,,0.0,8.4,8.4,8.1,8.1,2.0,,NaN,NaN,NaN,NaN
+U008,Test Assay 1.0,H03,1,1.0,,31.25,31.25,32.5,32.5,1.5,,,3.125,3.25,2.5
+U008,Test Assay 1.0,H04,1,1.0,,33.75,33.75,32.5,32.5,1.5,,,3.375,3.25,2.5