Skip to content

Commit b0b6720

Browse files
fix: Handle trailing tabs and quoted fields in Agilent Gen5 parser (#1241)
## Summary - Add chardet encoding detection for non-UTF-8 Gen5 files (e.g. files with degree symbol `°` encoded as 0xB0 in Windows-1252) - Strip trailing tabs from all lines at read time, preventing extra DataFrame columns/rows - Handle quoted fields in procedure details (e.g. `"Optics: Bottom, Gain: 75"`) by stripping quotes before parsing - Extract section names from first non-empty tab-delimited field, handling files with leading tabs - Detect and strip extra leading tab from data sections (Results, Layout, kinetic Time data) - Merge section data when a blank line separates header from content (e.g. `Layout\n\n<data>`) These issues all originate from a specific Gen5 export variant (Synergy H1 reader) that pads every line with trailing tabs, prepends an extra tab to data rows, and quotes fields containing commas. ## Test plan - [x] All 65 existing Agilent Gen5 tests pass - [x] All 318 vendor discovery tests pass - [x] 710 non-parser tests pass - [x] Target file (`PR391_020 plate1.txt`) parses successfully with 96 well documents 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 5dd7670 commit b0b6720

3 files changed

Lines changed: 47 additions & 9 deletions

File tree

src/allotropy/parsers/agilent_gen5/agilent_gen5_parser.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
Data,
66
Mapper,
77
)
8+
from allotropy.constants import CHARDET_ENCODING
89
from allotropy.named_file_contents import NamedFileContents
910
from allotropy.parsers.agilent_gen5.agilent_gen5_reader import AgilentGen5Reader
1011
from allotropy.parsers.agilent_gen5.agilent_gen5_structure import (
@@ -35,6 +36,12 @@ def sniff(cls, named_file_contents: NamedFileContents) -> bool:
3536
return False
3637

3738
def create_data(self, named_file_contents: NamedFileContents) -> Data:
39+
if named_file_contents.encoding is None:
40+
named_file_contents = NamedFileContents(
41+
named_file_contents.contents,
42+
named_file_contents.original_file_path,
43+
CHARDET_ENCODING,
44+
)
3845
reader = AgilentGen5Reader(named_file_contents)
3946
context = reader.extract_data_context(named_file_contents.original_file_path)
4047

src/allotropy/parsers/agilent_gen5/agilent_gen5_reader.py

Lines changed: 39 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
MULTIPLATE_FILE_ERROR,
2121
NO_PLATE_DATA_ERROR,
2222
)
23-
from allotropy.parsers.lines_reader import SectionLinesReader
23+
from allotropy.parsers.lines_reader import read_to_lines, SectionLinesReader
2424
from allotropy.parsers.utils.pandas import df_to_series_data, read_csv, SeriesData
2525
from allotropy.parsers.utils.values import assert_not_none
2626

@@ -31,7 +31,8 @@ class AgilentGen5Reader:
3131
SUPPORTED_EXTENSIONS = "txt"
3232

3333
def __init__(self, named_file_contents: NamedFileContents) -> None:
34-
reader = SectionLinesReader.create(named_file_contents)
34+
lines = [line.rstrip("\t") for line in read_to_lines(named_file_contents)]
35+
reader = SectionLinesReader(lines)
3536

3637
plate_readers = list(reader.iter_sections("^Software Version"))
3738
if not plate_readers:
@@ -75,7 +76,14 @@ def __init__(self, named_file_contents: NamedFileContents) -> None:
7576
last_read_label: str | None = None
7677

7778
while lines := list(plate_reader.pop_until_empty()):
78-
section_name = lines[0].split("\t")[0].strip(":")
79+
section_name = self._get_section_name(lines[0])
80+
81+
# Check if this is a single-line section header with data in the next block
82+
if len(lines) == 1 and section_name in ("Layout", "Results"):
83+
plate_reader.drop_empty()
84+
next_lines = list(plate_reader.pop_until_empty())
85+
if next_lines:
86+
lines = [lines[0], *next_lines]
7987

8088
# Check if this is a "Read X:label" section (single line, starts with "Read ")
8189
# If so, store the label and immediately read the next section
@@ -87,27 +95,50 @@ def __init__(self, named_file_contents: NamedFileContents) -> None:
8795
if not lines:
8896
# No data section after the label, just continue
8997
continue
90-
section_name = lines[0].split("\t")[0].strip(":")
98+
section_name = self._get_section_name(lines[0])
9199

92100
# If this is a Time section, associate it with a read label if we have one
93101
if section_name == "Time":
94102
if last_read_label:
95103
# Format: "Read X:label" -> Time section
96-
self.kinetic_sections[last_read_label] = lines
104+
self.kinetic_sections[last_read_label] = self._strip_leading_tab(
105+
lines
106+
)
97107
last_read_label = None
98108
elif last_section_name:
99109
# Format: "{label}" section followed by Time section
100110
# Store with just the label as key for kinetic files
101-
self.kinetic_sections[last_section_name] = lines
111+
self.kinetic_sections[last_section_name] = self._strip_leading_tab(
112+
lines
113+
)
102114
else:
103115
# Standalone Time section (shouldn't happen, but store it anyway)
104-
self.sections[section_name] = lines
116+
self.sections[section_name] = self._strip_leading_tab(lines)
105117
else:
106-
self.sections[section_name] = lines
118+
self.sections[section_name] = self._strip_leading_tab(lines)
107119
last_section_name = section_name
108120

109121
plate_reader.drop_empty()
110122

123+
@staticmethod
124+
def _get_section_name(line: str) -> str:
125+
fields = line.split("\t")
126+
for field in fields:
127+
name = field.strip('"').strip(":")
128+
if name:
129+
return name
130+
return ""
131+
132+
@staticmethod
133+
def _strip_leading_tab(lines: list[str]) -> list[str]:
134+
if len(lines) < 2:
135+
return lines
136+
non_header = lines[1:] if not lines[0].startswith("\t") else lines
137+
non_empty = [line for line in non_header if line.strip()]
138+
if non_empty and all(line.startswith("\t") for line in non_empty):
139+
return [line[1:] if line.startswith("\t") else line for line in lines]
140+
return lines
141+
111142
def get_required_section(self, section_name: str) -> list[str]:
112143
"""Get a required section, raises error if not found."""
113144
section = self.sections.get(section_name)

src/allotropy/parsers/agilent_gen5/agilent_gen5_structure.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -240,7 +240,7 @@ def create(cls, lines: list[str], read_mode: ReadMode) -> DeviceControlData:
240240
device_control_data = DeviceControlData()
241241

242242
for line in lines:
243-
strp_line = str(line.strip())
243+
strp_line = str(line.strip().strip('"').strip())
244244
if strp_line == "Fluorescence Polarization":
245245
device_control_data.is_polarization = True
246246
continue

0 commit comments

Comments
 (0)