Skip to content

feat: MSD Workbench - Update support for CSV format and Calc Conc CV handling #1157

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nathan-stender merged 1 commit into from
Mar 26, 2026
Merged

feat: MSD Workbench - Update support for CSV format and Calc Conc CV handling #1157

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/allotropy/calcdocs/msd_workbench/extractor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,5 +39,6 @@ def to_element(
"percent_recovery_mean": calc_data_structure.percent_recovery_mean,
"calc_concentration": calc_data_structure.calc_concentration,
"calc_conc_mean": calc_data_structure.calc_conc_mean,
"calc_conc_cv": calc_data_structure.calc_conc_cv,
},
)
1 change: 1 addition & 0 deletions src/allotropy/parsers/msd_workbench/calculdated_data_structure.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,3 +17,4 @@ class CalculatedDataMeasurementStructure:
percent_recovery_mean: float | None
calc_concentration: float | None
calc_conc_mean: float | None
calc_conc_cv: float | None
11 changes: 11 additions & 0 deletions src/allotropy/parsers/msd_workbench/msd_workbench_calculated_data_mapping.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ class CalculatedDataColumns(Enum):
PERCENT_RECOVERY_MEAN = "% Recovery Mean"
CALC_CONCENTRATION = "Calc. Concentration"
CALC_CONC_MEAN = "Calc. Conc. Mean"
CALC_CONC_CV = "Calc. Conc. CV"


def create_calculated_data_groups(
Expand Down Expand Up @@ -84,6 +85,9 @@ def create_calculated_data_groups(
calc_conc_mean=row_series.get(
float, CalculatedDataColumns.CALC_CONC_MEAN.value
),
calc_conc_cv=row_series.get(
float, CalculatedDataColumns.CALC_CONC_CV.value
),
)
)
# we do not need additional data for the calculated data documents
Expand Down Expand Up @@ -156,6 +160,12 @@ def create_calculated_data_groups(
view_data=assay_view,
source_configs=(calc_concentration_config,),
)
calc_conc_cv_config = CalculatedDataConfig(
name=CalculatedDataColumns.CALC_CONC_CV.value,
value="calc_conc_cv",
view_data=assay_view,
source_configs=(calc_concentration_config,),
)
configs = CalcDocsConfig(
[
mean_config,
Expand All @@ -167,6 +177,7 @@ def create_calculated_data_groups(
percent_recovery_mean_config,
calc_concentration_config,
calc_conc_mean_config,
calc_conc_cv_config,
]
)
calc_docs = [
Expand Down
66 changes: 53 additions & 13 deletions src/allotropy/parsers/msd_workbench/msd_workbench_reader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,18 +18,58 @@ def __init__(self, named_file_contents: NamedFileContents) -> None:
sheet_name="Workbench data",
engine="openpyxl",
)
data = data.where(pd.notna(data), None)
data.index = pd.Index(data.index.to_series().ffill())
data.index = data.index.astype(str).str.strip()
data.columns = data.columns.astype(str).str.strip()
first_row = str(data.iloc[0].index[0])
if "Plate" not in first_row:
msg = "Plate ID not found in the first row of the data"
raise AllotropeConversionError(msg)
self.well_plate_id = first_row.split("_")[-1].strip()
# Set the first row as the header
data.columns = pd.Index(data.iloc[0])
data = data[1:].reset_index(drop=True)
else:
data = read_csv(named_file_contents.contents)
data = data.where(pd.notna(data), None)
data.index = pd.Index(data.index.to_series().ffill())
data.index = data.index.astype(str).str.strip()
data.columns = data.columns.astype(str).str.strip()
first_row = str(data.iloc[0].index[0])
if "Plate" not in first_row:
msg = "Plate ID not found in the first row of the data"
raise AllotropeConversionError(msg)
self.well_plate_id = first_row.split("_")[-1].strip()
# Set the first row as the header
data.columns = pd.Index(data.iloc[0])
data = data[1:].reset_index(drop=True)
# First, peek at the first line to check format
named_file_contents.contents.seek(0)
first_line = named_file_contents.contents.readline()
if isinstance(first_line, bytes):
first_line = first_line.decode("utf-8")
named_file_contents.contents.seek(0)

# Check if first line has commas (old format) or not (new format)
if "," in first_line:
# Old format: Plate_ID,,,,,,,,
data = read_csv(named_file_contents.contents)
data = data.where(pd.notna(data), None)
data.index = pd.Index(data.index.to_series().ffill())
data.index = data.index.astype(str).str.strip()
data.columns = data.columns.astype(str).str.strip()
first_row = str(data.iloc[0].index[0])
if "Plate" not in first_row:
msg = "Plate ID not found in the first row of the data"
raise AllotropeConversionError(msg)
self.well_plate_id = first_row.split("_")[-1].strip()
# Set the first row as the header
data.columns = pd.Index(data.iloc[0])
data = data[1:].reset_index(drop=True)
else:
# New format: Plate_ID (no commas on first line)
# Extract plate ID from first line
plate_id_line = first_line.strip()
if "Plate" not in plate_id_line:
msg = "Plate ID not found in the first row of the data"
raise AllotropeConversionError(msg)
self.well_plate_id = plate_id_line.split("_")[-1].strip()

# Now read the CSV starting from the second line (which has headers)
named_file_contents.contents.seek(0)
# Skip the first line
named_file_contents.contents.readline()
# Read rest of file with second line as header
data = read_csv(named_file_contents.contents)
data = data.where(pd.notna(data), None)
data.columns = data.columns.astype(str).str.strip()

self.plate_data = data
7 changes: 6 additions & 1 deletion src/allotropy/parsers/msd_workbench/msd_workbench_structure.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,11 @@ def map_measurement(row: SeriesData) -> Measurement:
"detection range": row.get(str, "Detection Range"),
"assay identifier": row.get(str, "Assay"),
}
# Try both "Dilution Factor" and "Dilution" for backwards compatibility
dilution_value = row.get(int, "Dilution Factor")
Comment thread
nathan-stender marked this conversation as resolved.
if dilution_value is None:
dilution_value = row.get(int, "Dilution")

return Measurement(
type_=MeasurementType.LUMINESCENCE,
identifier=random_uuid_str(),
Expand All @@ -99,7 +104,7 @@ def map_measurement(row: SeriesData) -> Measurement:
row[str, "Sample"][0].lower()
),
sample_custom_info={
"dilution factor setting": row.get(int, "Dilution Factor"),
"dilution factor setting": dilution_value,
},
measurement_custom_info={
**custom_info,
Expand Down
50 changes: 50 additions & 0 deletions tests/parsers/msd_workbench/testdata/msd_workbench_test_2.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
Plate_TEST001
Sample,Assay,Well,Spot,Dilution,Concentration,Signal,Adjusted Signal,Mean,Adj. Sig. Mean,CV,% Recovery,% Recovery Mean,Calc. Concentration,Calc. Conc. Mean,Calc. Conc. CV
S001,Test Assay 1.0,A01,1,,100.0,1000.0,1000.0,1010.0,1010.0,1.5,102.5,101.0,102.0,101.0,1.2
S001,Test Assay 1.0,A02,1,,100.0,1020.0,1020.0,1010.0,1010.0,1.5,99.5,101.0,100.0,101.0,1.2
U001,Test Assay 1.0,A03,1,1.0,,2000.0,2000.0,2010.0,2010.0,0.8,,,200.0,201.0,0.6
U001,Test Assay 1.0,A04,1,1.0,,2020.0,2020.0,2010.0,2010.0,0.8,,,202.0,201.0,0.6
U009,Test Assay 1.0,A05,1,1.0,,3000.0,3000.0,3010.0,3010.0,1.2,,,300.0,301.0,1.0
U009,Test Assay 1.0,A06,1,1.0,,3020.0,3020.0,3010.0,3010.0,1.2,,,302.0,301.0,1.0
U017,Test Assay 1.0,A07,1,1.0,,4000.0,4000.0,4010.0,4010.0,0.5,,,400.0,401.0,0.4
U017,Test Assay 1.0,A08,1,1.0,,4020.0,4020.0,4010.0,4010.0,0.5,,,402.0,401.0,0.4
S002,Test Assay 1.0,B01,1,,50.0,500.0,500.0,510.0,510.0,1.8,99.0,98.5,49.5,49.0,1.5
S002,Test Assay 1.0,B02,1,,50.0,520.0,520.0,510.0,510.0,1.8,98.0,98.5,48.5,49.0,1.5
U002,Test Assay 1.0,B03,1,1.0,,1500.0,1500.0,1510.0,1510.0,1.0,,,150.0,151.0,0.9
U002,Test Assay 1.0,B04,1,1.0,,1520.0,1520.0,1510.0,1510.0,1.0,,,152.0,151.0,0.9
U010,Test Assay 1.0,B05,1,1.0,,2500.0,2500.0,2510.0,2510.0,0.6,,,250.0,251.0,0.5
U010,Test Assay 1.0,B06,1,1.0,,2520.0,2520.0,2510.0,2510.0,0.6,,,252.0,251.0,0.5
U018,Test Assay 1.0,B07,1,1.0,,3500.0,3500.0,3510.0,3510.0,0.4,,,350.0,351.0,0.3
U018,Test Assay 1.0,B08,1,1.0,,3520.0,3520.0,3510.0,3510.0,0.4,,,352.0,351.0,0.3
S003,Test Assay 1.0,C01,1,,25.0,250.0,250.0,255.0,255.0,0.1,98.0,98.5,24.5,24.7,0.1
S003,Test Assay 1.0,C02,1,,25.0,260.0,260.0,255.0,255.0,0.1,99.0,98.5,24.9,24.7,0.1
U003,Test Assay 1.0,C03,1,1.0,,1000.0,1000.0,1010.0,1010.0,0.7,,,100.0,101.0,0.6
U003,Test Assay 1.0,C04,1,1.0,,1020.0,1020.0,1010.0,1010.0,0.7,,,102.0,101.0,0.6
U011,Test Assay 1.0,C05,1,1.0,,2000.0,2000.0,2010.0,2010.0,0.9,,,200.0,201.0,0.8
U011,Test Assay 1.0,C06,1,1.0,,2020.0,2020.0,2010.0,2010.0,0.9,,,202.0,201.0,0.8
U019,Test Assay 1.0,C07,1,1.0,,3000.0,3000.0,3010.0,3010.0,0.2,,,300.0,301.0,0.2
U019,Test Assay 1.0,C08,1,1.0,,3020.0,3020.0,3010.0,3010.0,0.2,,,302.0,301.0,0.2
S004,Test Assay 1.0,D01,1,,12.5,125.0,125.0,130.0,130.0,2.5,103.0,100.5,12.9,12.6,3.0
S004,Test Assay 1.0,D02,1,,12.5,135.0,135.0,130.0,130.0,2.5,98.0,100.5,12.3,12.6,3.0
U004,Test Assay 1.0,D03,1,1.0,,500.0,500.0,510.0,510.0,0.3,,,50.0,51.0,0.2
U004,Test Assay 1.0,D04,1,1.0,,520.0,520.0,510.0,510.0,0.3,,,52.0,51.0,0.2
U012,Test Assay 1.0,D05,1,1.0,,1500.0,1500.0,1510.0,1510.0,1.3,,,150.0,151.0,1.2
U012,Test Assay 1.0,D06,1,1.0,,1520.0,1520.0,1510.0,1510.0,1.3,,,152.0,151.0,1.2
U020,Test Assay 1.0,D07,1,1.0,,2500.0,2500.0,2510.0,2510.0,1.6,,,250.0,251.0,2.5
U020,Test Assay 1.0,D08,1,1.0,,2520.0,2520.0,2510.0,2510.0,1.6,,,252.0,251.0,2.5
S005,Test Assay 1.0,E01,1,,6.25,62.5,62.5,65.0,65.0,2.2,98.5,102.0,6.15,6.35,4.5
S005,Test Assay 1.0,E02,1,,6.25,67.5,67.5,65.0,65.0,2.2,105.5,102.0,6.55,6.35,4.5
U005,Test Assay 1.0,E03,1,1.0,,250.0,250.0,255.0,255.0,0.2,,,25.0,25.5,0.1
U005,Test Assay 1.0,E04,1,1.0,,260.0,260.0,255.0,255.0,0.2,,,26.0,25.5,0.1
S006,Test Assay 1.0,F01,1,,3.125,31.25,31.25,32.5,32.5,1.0,103.5,101.0,3.24,3.16,2.8
S006,Test Assay 1.0,F02,1,,3.125,33.75,33.75,32.5,32.5,1.0,98.5,101.0,3.08,3.16,2.8
U006,Test Assay 1.0,F03,1,1.0,,125.0,125.0,127.5,127.5,0.05,,,12.5,12.75,0.04
U006,Test Assay 1.0,F04,1,1.0,,130.0,130.0,127.5,127.5,0.05,,,13.0,12.75,0.04
S007,Test Assay 1.0,G01,1,,1.5625,15.625,15.625,16.25,16.25,4.2,130.0,109.0,2.03,1.88,20.0
S007,Test Assay 1.0,G02,1,,1.5625,16.875,16.875,16.25,16.25,4.2,88.0,109.0,1.73,1.88,20.0
U007,Test Assay 1.0,G03,1,1.0,,62.5,62.5,65.0,65.0,0.2,,,6.25,6.5,0.3
U007,Test Assay 1.0,G04,1,1.0,,67.5,67.5,65.0,65.0,0.2,,,6.75,6.5,0.3
S008,Test Assay 1.0,H01,1,,0.0,7.8,7.8,8.1,8.1,2.0,,NaN,NaN,NaN,NaN
S008,Test Assay 1.0,H02,1,,0.0,8.4,8.4,8.1,8.1,2.0,,NaN,NaN,NaN,NaN
U008,Test Assay 1.0,H03,1,1.0,,31.25,31.25,32.5,32.5,1.5,,,3.125,3.25,2.5
U008,Test Assay 1.0,H04,1,1.0,,33.75,33.75,32.5,32.5,1.5,,,3.375,3.25,2.5
Loading
Loading