From 69fb8e905d677ee1f6ec14f7e88a5fda4d06020c Mon Sep 17 00:00:00 2001 From: Nathan Stender Date: Fri, 15 Aug 2025 14:27:46 -0400 Subject: [PATCH 01/13] Add UNREAD_DATA_HANDLED --- src/allotropy/parser_factory.py | 4 ++++ src/allotropy/parsers/agilent_gen5/agilent_gen5_parser.py | 1 + .../parsers/agilent_gen5_image/agilent_gen5_image_parser.py | 1 + .../parsers/agilent_openlab_cds/agilent_openlab_cds_parser.py | 1 + .../agilent_tapestation_analysis_parser.py | 1 + .../parsers/appbio_absolute_q/appbio_absolute_q_parser.py | 1 + .../parsers/appbio_quantstudio/appbio_quantstudio_parser.py | 1 + .../appbio_quantstudio_designandanalysis_parser.py | 1 + .../bd_biosciences_facsdiva/bd_biosciences_facsdiva_parser.py | 1 + .../beckman_coulter_biomek/beckman_coulter_biomek_parser.py | 1 + .../beckman_echo_plate_reformat_parser.py | 1 + .../parsers/beckman_pharmspec/beckman_pharmspec_parser.py | 1 + .../parsers/beckman_vi_cell_blu/vi_cell_blu_parser.py | 1 + src/allotropy/parsers/beckman_vi_cell_xr/vi_cell_xr_parser.py | 1 + .../benchling_chromeleon/benchling_chromeleon_parser.py | 1 + .../parsers/benchling_empower/benchling_empower_parser.py | 1 + .../biorad_bioplex_manager/biorad_bioplex_manager_parser.py | 1 + .../bmg_labtech_smart_control_parser.py | 1 + src/allotropy/parsers/bmg_mars/bmg_mars_parser.py | 1 + src/allotropy/parsers/cfxmaestro/cfxmaestro_parser.py | 1 + .../parsers/chemometec_nc_view/chemometec_nc_view_parser.py | 1 + .../parsers/chemometec_nucleoview/nucleoview_parser.py | 1 + src/allotropy/parsers/ctl_immunospot/ctl_immunospot_parser.py | 1 + .../cytiva_biacore_t200_control_parser.py | 1 + src/allotropy/parsers/cytiva_unicorn/cytiva_unicorn_parser.py | 1 + .../example_weyland_yutani/example_weyland_yutani_parser.py | 1 + src/allotropy/parsers/flowjo/flowjo_parser.py | 1 + .../parsers/luminex_intelliflex/luminex_intelliflex_parser.py | 1 + .../parsers/luminex_xponent/luminex_xponent_parser.py | 1 + src/allotropy/parsers/mabtech_apex/mabtech_apex_parser.py | 1 + .../parsers/methodical_mind/methodical_mind_parser.py | 1 + .../parsers/moldev_softmax_pro/softmax_pro_parser.py | 1 + src/allotropy/parsers/msd_workbench/msd_workbench_parser.py | 1 + src/allotropy/parsers/novabio_flex2/novabio_flex2_parser.py | 1 + .../perkin_elmer_envision/perkin_elmer_envision_parser.py | 1 + src/allotropy/parsers/qiacuity_dpcr/qiacuity_dpcr_parser.py | 1 + src/allotropy/parsers/revvity_kaleido/kaleido_parser.py | 1 + src/allotropy/parsers/revvity_matrix/revvity_matrix_parser.py | 1 + .../parsers/roche_cedex_bioht/roche_cedex_bioht_parser.py | 1 + .../parsers/roche_cedex_hires/roche_cedex_hires_parser.py | 1 + src/allotropy/parsers/tecan_magellan/tecan_magellan_parser.py | 1 + .../thermo_fisher_genesys30/thermo_fisher_genesys30_parser.py | 1 + .../thermo_fisher_genesys_on_board_parser.py | 1 + .../thermo_fisher_nanodrop_8000/nanodrop_8000_parser.py | 1 + .../thermo_fisher_nanodrop_eight/nanodrop_eight_parser.py | 2 +- .../thermo_fisher_nanodrop_one_parser.py | 1 + .../thermo_fisher_qubit4/thermo_fisher_qubit4_parser.py | 1 + .../thermo_fisher_qubit_flex_parser.py | 1 + .../thermo_fisher_visionlite_parser.py | 1 + src/allotropy/parsers/thermo_skanit/thermo_skanit_parser.py | 1 + .../unchained_labs_lunatic/unchained_labs_lunatic_parser.py | 1 + src/allotropy/parsers/vendor_parser.py | 2 ++ tests/to_allotrope_test.py | 2 +- 53 files changed, 57 insertions(+), 2 deletions(-) diff --git a/src/allotropy/parser_factory.py b/src/allotropy/parser_factory.py index bb6eebee4e..7e7294b5a7 100644 --- a/src/allotropy/parser_factory.py +++ b/src/allotropy/parser_factory.py @@ -182,6 +182,10 @@ class Vendor(Enum): def display_name(self) -> str: return self.get_parser().DISPLAY_NAME + @property + def unread_data_handled(self) -> str: + return self.get_parser().UNREAD_DATA_HANDLED + @property def release_state(self) -> ReleaseState: return self.get_parser().RELEASE_STATE diff --git a/src/allotropy/parsers/agilent_gen5/agilent_gen5_parser.py b/src/allotropy/parsers/agilent_gen5/agilent_gen5_parser.py index a35cf91921..5005c9f771 100644 --- a/src/allotropy/parsers/agilent_gen5/agilent_gen5_parser.py +++ b/src/allotropy/parsers/agilent_gen5/agilent_gen5_parser.py @@ -24,6 +24,7 @@ class AgilentGen5Parser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = AgilentGen5Reader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = True def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = AgilentGen5Reader(named_file_contents) diff --git a/src/allotropy/parsers/agilent_gen5_image/agilent_gen5_image_parser.py b/src/allotropy/parsers/agilent_gen5_image/agilent_gen5_image_parser.py index 3bfb60bee6..784e9b50fb 100644 --- a/src/allotropy/parsers/agilent_gen5_image/agilent_gen5_image_parser.py +++ b/src/allotropy/parsers/agilent_gen5_image/agilent_gen5_image_parser.py @@ -25,6 +25,7 @@ class AgilentGen5ImageParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = AgilentGen5Reader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = True def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = AgilentGen5Reader(named_file_contents) diff --git a/src/allotropy/parsers/agilent_openlab_cds/agilent_openlab_cds_parser.py b/src/allotropy/parsers/agilent_openlab_cds/agilent_openlab_cds_parser.py index fbc64a6d7d..2fbb4f1b01 100644 --- a/src/allotropy/parsers/agilent_openlab_cds/agilent_openlab_cds_parser.py +++ b/src/allotropy/parsers/agilent_openlab_cds/agilent_openlab_cds_parser.py @@ -24,6 +24,7 @@ class AgilentOpenLabCDSParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = "rslt" SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: structured_data = decode_data(named_file_contents.get_bytes_stream()) diff --git a/src/allotropy/parsers/agilent_tapestation_analysis/agilent_tapestation_analysis_parser.py b/src/allotropy/parsers/agilent_tapestation_analysis/agilent_tapestation_analysis_parser.py index 9f2799c62e..300745474f 100644 --- a/src/allotropy/parsers/agilent_tapestation_analysis/agilent_tapestation_analysis_parser.py +++ b/src/allotropy/parsers/agilent_tapestation_analysis/agilent_tapestation_analysis_parser.py @@ -22,6 +22,7 @@ class AgilentTapestationAnalysisParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = "xml" SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = True def create_data(self, named_file_contents: NamedFileContents) -> Data: try: diff --git a/src/allotropy/parsers/appbio_absolute_q/appbio_absolute_q_parser.py b/src/allotropy/parsers/appbio_absolute_q/appbio_absolute_q_parser.py index cda1e68065..3977ff0ffc 100644 --- a/src/allotropy/parsers/appbio_absolute_q/appbio_absolute_q_parser.py +++ b/src/allotropy/parsers/appbio_absolute_q/appbio_absolute_q_parser.py @@ -25,6 +25,7 @@ class AppbioAbsoluteQParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = AppbioAbsoluteQReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = True def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = AppbioAbsoluteQReader(named_file_contents) diff --git a/src/allotropy/parsers/appbio_quantstudio/appbio_quantstudio_parser.py b/src/allotropy/parsers/appbio_quantstudio/appbio_quantstudio_parser.py index d0937e9e5a..f1b13a3499 100644 --- a/src/allotropy/parsers/appbio_quantstudio/appbio_quantstudio_parser.py +++ b/src/allotropy/parsers/appbio_quantstudio/appbio_quantstudio_parser.py @@ -29,6 +29,7 @@ class AppBioQuantStudioParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = AppBioQuantStudioReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def parse_data( self, reader: AppBioQuantStudioReader, original_file_path: str diff --git a/src/allotropy/parsers/appbio_quantstudio_designandanalysis/appbio_quantstudio_designandanalysis_parser.py b/src/allotropy/parsers/appbio_quantstudio_designandanalysis/appbio_quantstudio_designandanalysis_parser.py index fc117f40fe..8f13ff016f 100644 --- a/src/allotropy/parsers/appbio_quantstudio_designandanalysis/appbio_quantstudio_designandanalysis_parser.py +++ b/src/allotropy/parsers/appbio_quantstudio_designandanalysis/appbio_quantstudio_designandanalysis_parser.py @@ -29,6 +29,7 @@ class AppBioQuantStudioDesignandanalysisParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = DesignQuantstudioReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def parse_rt_pcr( self, reader: DesignQuantstudioReader, original_file_path: str diff --git a/src/allotropy/parsers/bd_biosciences_facsdiva/bd_biosciences_facsdiva_parser.py b/src/allotropy/parsers/bd_biosciences_facsdiva/bd_biosciences_facsdiva_parser.py index 1a470a3433..dda2ca6942 100644 --- a/src/allotropy/parsers/bd_biosciences_facsdiva/bd_biosciences_facsdiva_parser.py +++ b/src/allotropy/parsers/bd_biosciences_facsdiva/bd_biosciences_facsdiva_parser.py @@ -24,6 +24,7 @@ class BDFACSDivaParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = "xml" SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: try: diff --git a/src/allotropy/parsers/beckman_coulter_biomek/beckman_coulter_biomek_parser.py b/src/allotropy/parsers/beckman_coulter_biomek/beckman_coulter_biomek_parser.py index cb7697299f..7eb5401cf8 100644 --- a/src/allotropy/parsers/beckman_coulter_biomek/beckman_coulter_biomek_parser.py +++ b/src/allotropy/parsers/beckman_coulter_biomek/beckman_coulter_biomek_parser.py @@ -23,6 +23,7 @@ class BeckmanCoulterBiomekParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = BeckmanCoulterBiomekReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = BeckmanCoulterBiomekReader(named_file_contents) diff --git a/src/allotropy/parsers/beckman_echo_plate_reformat/beckman_echo_plate_reformat_parser.py b/src/allotropy/parsers/beckman_echo_plate_reformat/beckman_echo_plate_reformat_parser.py index daf6a3b13f..57022093fc 100644 --- a/src/allotropy/parsers/beckman_echo_plate_reformat/beckman_echo_plate_reformat_parser.py +++ b/src/allotropy/parsers/beckman_echo_plate_reformat/beckman_echo_plate_reformat_parser.py @@ -25,6 +25,7 @@ class BeckmanEchoPlateReformatParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = BeckmanEchoPlateReformatReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = BeckmanEchoPlateReformatReader(named_file_contents) diff --git a/src/allotropy/parsers/beckman_pharmspec/beckman_pharmspec_parser.py b/src/allotropy/parsers/beckman_pharmspec/beckman_pharmspec_parser.py index acfb3a64da..c461a73f6e 100644 --- a/src/allotropy/parsers/beckman_pharmspec/beckman_pharmspec_parser.py +++ b/src/allotropy/parsers/beckman_pharmspec/beckman_pharmspec_parser.py @@ -25,6 +25,7 @@ class PharmSpecParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = BeckmanPharmspecReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = BeckmanPharmspecReader(named_file_contents) diff --git a/src/allotropy/parsers/beckman_vi_cell_blu/vi_cell_blu_parser.py b/src/allotropy/parsers/beckman_vi_cell_blu/vi_cell_blu_parser.py index 549321e7e4..a1ab9e37d4 100644 --- a/src/allotropy/parsers/beckman_vi_cell_blu/vi_cell_blu_parser.py +++ b/src/allotropy/parsers/beckman_vi_cell_blu/vi_cell_blu_parser.py @@ -23,6 +23,7 @@ class ViCellBluParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = ViCellBluReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = True def create_data(self, named_file_contents: NamedFileContents) -> Data: return Data( diff --git a/src/allotropy/parsers/beckman_vi_cell_xr/vi_cell_xr_parser.py b/src/allotropy/parsers/beckman_vi_cell_xr/vi_cell_xr_parser.py index 05f11a8694..aa8d6cfaab 100644 --- a/src/allotropy/parsers/beckman_vi_cell_xr/vi_cell_xr_parser.py +++ b/src/allotropy/parsers/beckman_vi_cell_xr/vi_cell_xr_parser.py @@ -23,6 +23,7 @@ class ViCellXRParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = "txt,xls,xlsx" SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = True def create_data(self, named_file_contents: NamedFileContents) -> Data: reader_data = create_reader_data(named_file_contents) diff --git a/src/allotropy/parsers/benchling_chromeleon/benchling_chromeleon_parser.py b/src/allotropy/parsers/benchling_chromeleon/benchling_chromeleon_parser.py index c747aa5cd3..cc137f13fa 100644 --- a/src/allotropy/parsers/benchling_chromeleon/benchling_chromeleon_parser.py +++ b/src/allotropy/parsers/benchling_chromeleon/benchling_chromeleon_parser.py @@ -23,6 +23,7 @@ class BenchlingChromeleonParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = BenchlingChromeleonReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = BenchlingChromeleonReader(named_file_contents) diff --git a/src/allotropy/parsers/benchling_empower/benchling_empower_parser.py b/src/allotropy/parsers/benchling_empower/benchling_empower_parser.py index 371d56785f..589907b679 100644 --- a/src/allotropy/parsers/benchling_empower/benchling_empower_parser.py +++ b/src/allotropy/parsers/benchling_empower/benchling_empower_parser.py @@ -23,6 +23,7 @@ class BenchlingEmpowerParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = BenchlingEmpowerReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = True def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = BenchlingEmpowerReader(named_file_contents) diff --git a/src/allotropy/parsers/biorad_bioplex_manager/biorad_bioplex_manager_parser.py b/src/allotropy/parsers/biorad_bioplex_manager/biorad_bioplex_manager_parser.py index b61e2dab28..c2975fc263 100644 --- a/src/allotropy/parsers/biorad_bioplex_manager/biorad_bioplex_manager_parser.py +++ b/src/allotropy/parsers/biorad_bioplex_manager/biorad_bioplex_manager_parser.py @@ -27,6 +27,7 @@ class BioradBioplexParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = "xml" SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = BioradBioplexReader(named_file_contents) diff --git a/src/allotropy/parsers/bmg_labtech_smart_control/bmg_labtech_smart_control_parser.py b/src/allotropy/parsers/bmg_labtech_smart_control/bmg_labtech_smart_control_parser.py index 459d3c5058..71cd65df8d 100644 --- a/src/allotropy/parsers/bmg_labtech_smart_control/bmg_labtech_smart_control_parser.py +++ b/src/allotropy/parsers/bmg_labtech_smart_control/bmg_labtech_smart_control_parser.py @@ -27,6 +27,7 @@ class BmgLabtechSmartControlParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = BmgLabtechSmartControlReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = BmgLabtechSmartControlReader(named_file_contents) diff --git a/src/allotropy/parsers/bmg_mars/bmg_mars_parser.py b/src/allotropy/parsers/bmg_mars/bmg_mars_parser.py index b36817ef99..003b57d323 100644 --- a/src/allotropy/parsers/bmg_mars/bmg_mars_parser.py +++ b/src/allotropy/parsers/bmg_mars/bmg_mars_parser.py @@ -23,6 +23,7 @@ class BmgMarsParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = "csv" SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = BmgMarsReader(named_file_contents) diff --git a/src/allotropy/parsers/cfxmaestro/cfxmaestro_parser.py b/src/allotropy/parsers/cfxmaestro/cfxmaestro_parser.py index f902044524..a17870543b 100644 --- a/src/allotropy/parsers/cfxmaestro/cfxmaestro_parser.py +++ b/src/allotropy/parsers/cfxmaestro/cfxmaestro_parser.py @@ -18,6 +18,7 @@ class CfxmaestroParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = CFXMaestroReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = True def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = CFXMaestroReader(named_file_contents) diff --git a/src/allotropy/parsers/chemometec_nc_view/chemometec_nc_view_parser.py b/src/allotropy/parsers/chemometec_nc_view/chemometec_nc_view_parser.py index 6f79b992cc..56ea8f4ed8 100644 --- a/src/allotropy/parsers/chemometec_nc_view/chemometec_nc_view_parser.py +++ b/src/allotropy/parsers/chemometec_nc_view/chemometec_nc_view_parser.py @@ -24,6 +24,7 @@ class ChemometecNcViewParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = ChemometecNcViewReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = ChemometecNcViewReader(named_file_contents) diff --git a/src/allotropy/parsers/chemometec_nucleoview/nucleoview_parser.py b/src/allotropy/parsers/chemometec_nucleoview/nucleoview_parser.py index d58ddc806b..370214dd39 100644 --- a/src/allotropy/parsers/chemometec_nucleoview/nucleoview_parser.py +++ b/src/allotropy/parsers/chemometec_nucleoview/nucleoview_parser.py @@ -21,6 +21,7 @@ class ChemometecNucleoviewParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = NucleoviewReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = True def create_data(self, named_file_contents: NamedFileContents) -> Data: df = NucleoviewReader.read(named_file_contents.contents) diff --git a/src/allotropy/parsers/ctl_immunospot/ctl_immunospot_parser.py b/src/allotropy/parsers/ctl_immunospot/ctl_immunospot_parser.py index b28ed624d2..4f908762bc 100644 --- a/src/allotropy/parsers/ctl_immunospot/ctl_immunospot_parser.py +++ b/src/allotropy/parsers/ctl_immunospot/ctl_immunospot_parser.py @@ -20,6 +20,7 @@ class CtlImmunospotParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = CtlImmunospotReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = CtlImmunospotReader(named_file_contents) diff --git a/src/allotropy/parsers/cytiva_biacore_t200_control/cytiva_biacore_t200_control_parser.py b/src/allotropy/parsers/cytiva_biacore_t200_control/cytiva_biacore_t200_control_parser.py index 49a50a6b1a..445b2216b5 100644 --- a/src/allotropy/parsers/cytiva_biacore_t200_control/cytiva_biacore_t200_control_parser.py +++ b/src/allotropy/parsers/cytiva_biacore_t200_control/cytiva_biacore_t200_control_parser.py @@ -27,6 +27,7 @@ class CytivaBiacoreT200ControlParser(VendorParser[MapperData, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = "blr" SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> MapperData: data = Data.create(decode_data(named_file_contents)) diff --git a/src/allotropy/parsers/cytiva_unicorn/cytiva_unicorn_parser.py b/src/allotropy/parsers/cytiva_unicorn/cytiva_unicorn_parser.py index f133160330..5dd9df0cb0 100644 --- a/src/allotropy/parsers/cytiva_unicorn/cytiva_unicorn_parser.py +++ b/src/allotropy/parsers/cytiva_unicorn/cytiva_unicorn_parser.py @@ -25,6 +25,7 @@ class CytivaUnicornParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = "zip" SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = True def create_data(self, named_file_contents: NamedFileContents) -> Data: handler = UnicornZipHandler(named_file_contents.get_bytes_stream()) diff --git a/src/allotropy/parsers/example_weyland_yutani/example_weyland_yutani_parser.py b/src/allotropy/parsers/example_weyland_yutani/example_weyland_yutani_parser.py index 3a7ac5abf7..0143317a69 100644 --- a/src/allotropy/parsers/example_weyland_yutani/example_weyland_yutani_parser.py +++ b/src/allotropy/parsers/example_weyland_yutani/example_weyland_yutani_parser.py @@ -26,6 +26,7 @@ class ExampleWeylandYutaniParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.WORKING_DRAFT SUPPORTED_EXTENSIONS = ExampleWeylandYutaniReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = ExampleWeylandYutaniReader(named_file_contents) diff --git a/src/allotropy/parsers/flowjo/flowjo_parser.py b/src/allotropy/parsers/flowjo/flowjo_parser.py index 30543dfd12..337f18d44b 100644 --- a/src/allotropy/parsers/flowjo/flowjo_parser.py +++ b/src/allotropy/parsers/flowjo/flowjo_parser.py @@ -60,6 +60,7 @@ class FlowjoParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = "wsp" SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: try: diff --git a/src/allotropy/parsers/luminex_intelliflex/luminex_intelliflex_parser.py b/src/allotropy/parsers/luminex_intelliflex/luminex_intelliflex_parser.py index 5850ec56fc..0f43cb3b99 100644 --- a/src/allotropy/parsers/luminex_intelliflex/luminex_intelliflex_parser.py +++ b/src/allotropy/parsers/luminex_intelliflex/luminex_intelliflex_parser.py @@ -26,6 +26,7 @@ class LuminexIntelliflexParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = LuminexXponentReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = LuminexXponentReader(named_file_contents) diff --git a/src/allotropy/parsers/luminex_xponent/luminex_xponent_parser.py b/src/allotropy/parsers/luminex_xponent/luminex_xponent_parser.py index f2e38f04f0..33d156c845 100644 --- a/src/allotropy/parsers/luminex_xponent/luminex_xponent_parser.py +++ b/src/allotropy/parsers/luminex_xponent/luminex_xponent_parser.py @@ -23,6 +23,7 @@ class LuminexXponentParser(VendorParser[MapperData, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = LuminexXponentReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> MapperData: reader = LuminexXponentReader(named_file_contents) diff --git a/src/allotropy/parsers/mabtech_apex/mabtech_apex_parser.py b/src/allotropy/parsers/mabtech_apex/mabtech_apex_parser.py index f6f3c5ab96..9b7db7ff46 100644 --- a/src/allotropy/parsers/mabtech_apex/mabtech_apex_parser.py +++ b/src/allotropy/parsers/mabtech_apex/mabtech_apex_parser.py @@ -25,6 +25,7 @@ class MabtechApexParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = MabtechApexReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = MabtechApexReader.create(named_file_contents) diff --git a/src/allotropy/parsers/methodical_mind/methodical_mind_parser.py b/src/allotropy/parsers/methodical_mind/methodical_mind_parser.py index 38cbce4427..50154cdf8b 100644 --- a/src/allotropy/parsers/methodical_mind/methodical_mind_parser.py +++ b/src/allotropy/parsers/methodical_mind/methodical_mind_parser.py @@ -26,6 +26,7 @@ class MethodicalMindParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = "txt" SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = MethodicalMindReader(named_file_contents) diff --git a/src/allotropy/parsers/moldev_softmax_pro/softmax_pro_parser.py b/src/allotropy/parsers/moldev_softmax_pro/softmax_pro_parser.py index 3096b49f69..3000c96898 100644 --- a/src/allotropy/parsers/moldev_softmax_pro/softmax_pro_parser.py +++ b/src/allotropy/parsers/moldev_softmax_pro/softmax_pro_parser.py @@ -22,6 +22,7 @@ class SoftmaxproParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = "txt" SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: lines = read_to_lines(named_file_contents) diff --git a/src/allotropy/parsers/msd_workbench/msd_workbench_parser.py b/src/allotropy/parsers/msd_workbench/msd_workbench_parser.py index 4c15d4ed84..56ebce7d2d 100644 --- a/src/allotropy/parsers/msd_workbench/msd_workbench_parser.py +++ b/src/allotropy/parsers/msd_workbench/msd_workbench_parser.py @@ -37,6 +37,7 @@ class MSDWorkbenchParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = MSDWorkbenchReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: if named_file_contents.extension == "txt": diff --git a/src/allotropy/parsers/novabio_flex2/novabio_flex2_parser.py b/src/allotropy/parsers/novabio_flex2/novabio_flex2_parser.py index e5b0a28c46..f0c82d7aab 100644 --- a/src/allotropy/parsers/novabio_flex2/novabio_flex2_parser.py +++ b/src/allotropy/parsers/novabio_flex2/novabio_flex2_parser.py @@ -25,6 +25,7 @@ class NovaBioFlexParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = "csv" SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: try: diff --git a/src/allotropy/parsers/perkin_elmer_envision/perkin_elmer_envision_parser.py b/src/allotropy/parsers/perkin_elmer_envision/perkin_elmer_envision_parser.py index 293ff81fac..13addd73b9 100644 --- a/src/allotropy/parsers/perkin_elmer_envision/perkin_elmer_envision_parser.py +++ b/src/allotropy/parsers/perkin_elmer_envision/perkin_elmer_envision_parser.py @@ -24,6 +24,7 @@ class PerkinElmerEnvisionParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = "csv" SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: lines = read_to_lines(named_file_contents) diff --git a/src/allotropy/parsers/qiacuity_dpcr/qiacuity_dpcr_parser.py b/src/allotropy/parsers/qiacuity_dpcr/qiacuity_dpcr_parser.py index 34149837df..928dc47f8a 100644 --- a/src/allotropy/parsers/qiacuity_dpcr/qiacuity_dpcr_parser.py +++ b/src/allotropy/parsers/qiacuity_dpcr/qiacuity_dpcr_parser.py @@ -22,6 +22,7 @@ class QiacuitydPCRParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = QiacuitydPCRReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = QiacuitydPCRReader(named_file_contents) diff --git a/src/allotropy/parsers/revvity_kaleido/kaleido_parser.py b/src/allotropy/parsers/revvity_kaleido/kaleido_parser.py index 2b2782bbda..b854a2f8ab 100644 --- a/src/allotropy/parsers/revvity_kaleido/kaleido_parser.py +++ b/src/allotropy/parsers/revvity_kaleido/kaleido_parser.py @@ -21,6 +21,7 @@ class KaleidoParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SCHEMA_MAPPER = Mapper SUPPORTED_EXTENSIONS = "csv" + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: data = create_data(CsvReader(read_to_lines(named_file_contents))) diff --git a/src/allotropy/parsers/revvity_matrix/revvity_matrix_parser.py b/src/allotropy/parsers/revvity_matrix/revvity_matrix_parser.py index dc2e0b81d8..8cfaa1fdb1 100644 --- a/src/allotropy/parsers/revvity_matrix/revvity_matrix_parser.py +++ b/src/allotropy/parsers/revvity_matrix/revvity_matrix_parser.py @@ -26,6 +26,7 @@ class RevvityMatrixParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = RevvityMatrixReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = RevvityMatrixReader(named_file_contents) diff --git a/src/allotropy/parsers/roche_cedex_bioht/roche_cedex_bioht_parser.py b/src/allotropy/parsers/roche_cedex_bioht/roche_cedex_bioht_parser.py index c78226697b..391342de35 100644 --- a/src/allotropy/parsers/roche_cedex_bioht/roche_cedex_bioht_parser.py +++ b/src/allotropy/parsers/roche_cedex_bioht/roche_cedex_bioht_parser.py @@ -24,6 +24,7 @@ class RocheCedexBiohtParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = RocheCedexBiohtReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = RocheCedexBiohtReader(named_file_contents.contents) diff --git a/src/allotropy/parsers/roche_cedex_hires/roche_cedex_hires_parser.py b/src/allotropy/parsers/roche_cedex_hires/roche_cedex_hires_parser.py index d54af8ee5d..7ed63b3eeb 100644 --- a/src/allotropy/parsers/roche_cedex_hires/roche_cedex_hires_parser.py +++ b/src/allotropy/parsers/roche_cedex_hires/roche_cedex_hires_parser.py @@ -24,6 +24,7 @@ class RocheCedexHiResParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = RocheCedexHiResReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = RocheCedexHiResReader(named_file_contents) diff --git a/src/allotropy/parsers/tecan_magellan/tecan_magellan_parser.py b/src/allotropy/parsers/tecan_magellan/tecan_magellan_parser.py index 2c4390aadd..3b6a4cd4f5 100644 --- a/src/allotropy/parsers/tecan_magellan/tecan_magellan_parser.py +++ b/src/allotropy/parsers/tecan_magellan/tecan_magellan_parser.py @@ -27,6 +27,7 @@ class TecanMagellanParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = TecanMagellanReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = TecanMagellanReader(named_file_contents) diff --git a/src/allotropy/parsers/thermo_fisher_genesys30/thermo_fisher_genesys30_parser.py b/src/allotropy/parsers/thermo_fisher_genesys30/thermo_fisher_genesys30_parser.py index d827d47c4e..f3897f57eb 100644 --- a/src/allotropy/parsers/thermo_fisher_genesys30/thermo_fisher_genesys30_parser.py +++ b/src/allotropy/parsers/thermo_fisher_genesys30/thermo_fisher_genesys30_parser.py @@ -23,6 +23,7 @@ class ThermoFisherGenesys30Parser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = ThermoFisherGenesys30Reader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = ThermoFisherGenesys30Reader(named_file_contents) diff --git a/src/allotropy/parsers/thermo_fisher_genesys_on_board/thermo_fisher_genesys_on_board_parser.py b/src/allotropy/parsers/thermo_fisher_genesys_on_board/thermo_fisher_genesys_on_board_parser.py index 246b7dd238..663793cdba 100644 --- a/src/allotropy/parsers/thermo_fisher_genesys_on_board/thermo_fisher_genesys_on_board_parser.py +++ b/src/allotropy/parsers/thermo_fisher_genesys_on_board/thermo_fisher_genesys_on_board_parser.py @@ -25,6 +25,7 @@ class ThermoFisherGenesysOnBoardParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = ThermoFisherVisionliteReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: return VisionLiteData.create( diff --git a/src/allotropy/parsers/thermo_fisher_nanodrop_8000/nanodrop_8000_parser.py b/src/allotropy/parsers/thermo_fisher_nanodrop_8000/nanodrop_8000_parser.py index da19c8a7f1..7e7115799e 100644 --- a/src/allotropy/parsers/thermo_fisher_nanodrop_8000/nanodrop_8000_parser.py +++ b/src/allotropy/parsers/thermo_fisher_nanodrop_8000/nanodrop_8000_parser.py @@ -24,6 +24,7 @@ class Nanodrop8000Parser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = Nanodrop8000Reader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = Nanodrop8000Reader(named_file_contents) diff --git a/src/allotropy/parsers/thermo_fisher_nanodrop_eight/nanodrop_eight_parser.py b/src/allotropy/parsers/thermo_fisher_nanodrop_eight/nanodrop_eight_parser.py index b28418eed4..aa1902b816 100644 --- a/src/allotropy/parsers/thermo_fisher_nanodrop_eight/nanodrop_eight_parser.py +++ b/src/allotropy/parsers/thermo_fisher_nanodrop_eight/nanodrop_eight_parser.py @@ -25,8 +25,8 @@ class NanodropEightParser(VendorParser[Data, Model]): DISPLAY_NAME = "Thermo Fisher Scientific NanoDrop Eight" RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = NanodropEightReader.SUPPORTED_EXTENSIONS - SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = NanodropEightReader(named_file_contents) diff --git a/src/allotropy/parsers/thermo_fisher_nanodrop_one/thermo_fisher_nanodrop_one_parser.py b/src/allotropy/parsers/thermo_fisher_nanodrop_one/thermo_fisher_nanodrop_one_parser.py index 98cfa8b948..69c8a861ae 100644 --- a/src/allotropy/parsers/thermo_fisher_nanodrop_one/thermo_fisher_nanodrop_one_parser.py +++ b/src/allotropy/parsers/thermo_fisher_nanodrop_one/thermo_fisher_nanodrop_one_parser.py @@ -25,6 +25,7 @@ class ThermoFisherNanodropOneParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = "csv,xlsx" SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def read_data( self, named_file_contents: NamedFileContents diff --git a/src/allotropy/parsers/thermo_fisher_qubit4/thermo_fisher_qubit4_parser.py b/src/allotropy/parsers/thermo_fisher_qubit4/thermo_fisher_qubit4_parser.py index 1da7a58b72..613ead00c9 100644 --- a/src/allotropy/parsers/thermo_fisher_qubit4/thermo_fisher_qubit4_parser.py +++ b/src/allotropy/parsers/thermo_fisher_qubit4/thermo_fisher_qubit4_parser.py @@ -23,6 +23,7 @@ class ThermoFisherQubit4Parser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = ThermoFisherQubit4Reader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: return Data( diff --git a/src/allotropy/parsers/thermo_fisher_qubit_flex/thermo_fisher_qubit_flex_parser.py b/src/allotropy/parsers/thermo_fisher_qubit_flex/thermo_fisher_qubit_flex_parser.py index 99a891b799..4e9460f5cd 100644 --- a/src/allotropy/parsers/thermo_fisher_qubit_flex/thermo_fisher_qubit_flex_parser.py +++ b/src/allotropy/parsers/thermo_fisher_qubit_flex/thermo_fisher_qubit_flex_parser.py @@ -25,6 +25,7 @@ class ThermoFisherQubitFlexParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = ThermoFisherQubit4Reader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: df = ThermoFisherQubitFlexReader.read(named_file_contents) diff --git a/src/allotropy/parsers/thermo_fisher_visionlite/thermo_fisher_visionlite_parser.py b/src/allotropy/parsers/thermo_fisher_visionlite/thermo_fisher_visionlite_parser.py index 814ae7d68c..b8d9826e47 100644 --- a/src/allotropy/parsers/thermo_fisher_visionlite/thermo_fisher_visionlite_parser.py +++ b/src/allotropy/parsers/thermo_fisher_visionlite/thermo_fisher_visionlite_parser.py @@ -22,6 +22,7 @@ class ThermoFisherVisionliteParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = ThermoFisherVisionliteReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: return VisionLiteData.create( diff --git a/src/allotropy/parsers/thermo_skanit/thermo_skanit_parser.py b/src/allotropy/parsers/thermo_skanit/thermo_skanit_parser.py index 2cef70d849..801a1fa3b8 100644 --- a/src/allotropy/parsers/thermo_skanit/thermo_skanit_parser.py +++ b/src/allotropy/parsers/thermo_skanit/thermo_skanit_parser.py @@ -17,6 +17,7 @@ class ThermoSkanItParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = "xlsx" SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: contents = read_multisheet_excel( diff --git a/src/allotropy/parsers/unchained_labs_lunatic/unchained_labs_lunatic_parser.py b/src/allotropy/parsers/unchained_labs_lunatic/unchained_labs_lunatic_parser.py index 9d1a811746..d67767003b 100644 --- a/src/allotropy/parsers/unchained_labs_lunatic/unchained_labs_lunatic_parser.py +++ b/src/allotropy/parsers/unchained_labs_lunatic/unchained_labs_lunatic_parser.py @@ -24,6 +24,7 @@ class UnchainedLabsLunaticParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = UnchainedLabsLunaticReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = UnchainedLabsLunaticReader(named_file_contents) diff --git a/src/allotropy/parsers/vendor_parser.py b/src/allotropy/parsers/vendor_parser.py index f6a2e93467..18caa5942d 100644 --- a/src/allotropy/parsers/vendor_parser.py +++ b/src/allotropy/parsers/vendor_parser.py @@ -25,6 +25,8 @@ class VendorParser(ABC, Generic[Data, Model]): SUPPORTED_EXTENSIONS: str # The schema mapper to use for mapping to ASM SCHEMA_MAPPER: Callable[..., SchemaMapper[Data, Model]] + # Marks if the parser has already been migrated to handle unread data and should warn if data is unread + UNREAD_DATA_HANDLED: bool = False timestamp_parser: TimestampParser diff --git a/tests/to_allotrope_test.py b/tests/to_allotrope_test.py index 044d84485b..ff70bda5d6 100644 --- a/tests/to_allotrope_test.py +++ b/tests/to_allotrope_test.py @@ -46,7 +46,7 @@ class ParserTest: def test_positive_cases( self, test_file_path: Path, *, overwrite: bool, warn_unread_keys: bool ) -> None: - if warn_unread_keys: + if warn_unread_keys or self.VENDOR.unread_data_handled: os.environ["WARN_UNUSED_KEYS"] = "1" # Special case when input files are json, the are placed in an input/ folder and the results are put # in a corresponding output/ folder. From 98bdd47a845ceea18b6de17bcbbf31ae97cecbca Mon Sep 17 00:00:00 2001 From: Nathan Stender Date: Fri, 15 Aug 2025 14:28:41 -0400 Subject: [PATCH 02/13] Fix mypy --- src/allotropy/parser_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allotropy/parser_factory.py b/src/allotropy/parser_factory.py index 7e7294b5a7..3db547a021 100644 --- a/src/allotropy/parser_factory.py +++ b/src/allotropy/parser_factory.py @@ -183,7 +183,7 @@ def display_name(self) -> str: return self.get_parser().DISPLAY_NAME @property - def unread_data_handled(self) -> str: + def unread_data_handled(self) -> bool: return self.get_parser().UNREAD_DATA_HANDLED @property From e8eaac49448c4ca186f684ae5bd38b97e8004f6c Mon Sep 17 00:00:00 2001 From: Nathan Stender Date: Mon, 18 Aug 2025 10:21:17 -0400 Subject: [PATCH 03/13] Add more entries --- .../agilent_tapestation_analysis_parser.py | 2 +- ...io_quantstudio_designandanalysis_parser.py | 2 +- .../beckman_echo_plate_reformat_parser.py | 2 +- .../beckman_vi_cell_xr/vi_cell_xr_reader.py | 4 +- .../bmg_labtech_smart_control_parser.py | 2 +- .../cfxmaestro/cfxmaestro_structure.py | 7 +--- src/allotropy/parsers/flowjo/flowjo_parser.py | 2 +- .../mabtech_apex/mabtech_apex_parser.py | 2 +- .../methodical_mind/methodical_mind_parser.py | 2 +- .../msd_workbench/msd_workbench_parser.py | 2 +- .../revvity_matrix/revvity_matrix_parser.py | 2 +- .../roche_cedex_hires_parser.py | 2 +- .../thermo_fisher_nanodrop_one_parser.py | 2 +- .../thermo_fisher_qubit4_parser.py | 2 +- .../unchained_labs_lunatic_parser.py | 2 +- src/allotropy/parsers/utils/json.py | 33 ++++++++------- src/allotropy/parsers/utils/pandas.py | 9 ++-- .../parsers/utils/strict_xml_element.py | 41 +++++++++---------- tests/to_allotrope_test.py | 24 ++++++++--- 19 files changed, 76 insertions(+), 68 deletions(-) diff --git a/src/allotropy/parsers/agilent_tapestation_analysis/agilent_tapestation_analysis_parser.py b/src/allotropy/parsers/agilent_tapestation_analysis/agilent_tapestation_analysis_parser.py index 300745474f..e62f4cd5e6 100644 --- a/src/allotropy/parsers/agilent_tapestation_analysis/agilent_tapestation_analysis_parser.py +++ b/src/allotropy/parsers/agilent_tapestation_analysis/agilent_tapestation_analysis_parser.py @@ -22,7 +22,7 @@ class AgilentTapestationAnalysisParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = "xml" SCHEMA_MAPPER = Mapper - UNREAD_DATA_HANDLED = True + UNREAD_DATA_HANDLED = False def create_data(self, named_file_contents: NamedFileContents) -> Data: try: diff --git a/src/allotropy/parsers/appbio_quantstudio_designandanalysis/appbio_quantstudio_designandanalysis_parser.py b/src/allotropy/parsers/appbio_quantstudio_designandanalysis/appbio_quantstudio_designandanalysis_parser.py index 8f13ff016f..63ef575fcb 100644 --- a/src/allotropy/parsers/appbio_quantstudio_designandanalysis/appbio_quantstudio_designandanalysis_parser.py +++ b/src/allotropy/parsers/appbio_quantstudio_designandanalysis/appbio_quantstudio_designandanalysis_parser.py @@ -29,7 +29,7 @@ class AppBioQuantStudioDesignandanalysisParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = DesignQuantstudioReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper - UNREAD_DATA_HANDLED = False + UNREAD_DATA_HANDLED = True def parse_rt_pcr( self, reader: DesignQuantstudioReader, original_file_path: str diff --git a/src/allotropy/parsers/beckman_echo_plate_reformat/beckman_echo_plate_reformat_parser.py b/src/allotropy/parsers/beckman_echo_plate_reformat/beckman_echo_plate_reformat_parser.py index 57022093fc..3427500a30 100644 --- a/src/allotropy/parsers/beckman_echo_plate_reformat/beckman_echo_plate_reformat_parser.py +++ b/src/allotropy/parsers/beckman_echo_plate_reformat/beckman_echo_plate_reformat_parser.py @@ -25,7 +25,7 @@ class BeckmanEchoPlateReformatParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = BeckmanEchoPlateReformatReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper - UNREAD_DATA_HANDLED = False + UNREAD_DATA_HANDLED = True def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = BeckmanEchoPlateReformatReader(named_file_contents) diff --git a/src/allotropy/parsers/beckman_vi_cell_xr/vi_cell_xr_reader.py b/src/allotropy/parsers/beckman_vi_cell_xr/vi_cell_xr_reader.py index b5246f2236..147343e422 100644 --- a/src/allotropy/parsers/beckman_vi_cell_xr/vi_cell_xr_reader.py +++ b/src/allotropy/parsers/beckman_vi_cell_xr/vi_cell_xr_reader.py @@ -124,7 +124,9 @@ def _get_file_info(self) -> SeriesData: usecols=[0], ).squeeze() info.index = pd.Index(["model", "filepath", "serial"]) - return SeriesData(info) + data = SeriesData(info) + data.mark_read("filepath") + return data class ViCellXRTXTReader: diff --git a/src/allotropy/parsers/bmg_labtech_smart_control/bmg_labtech_smart_control_parser.py b/src/allotropy/parsers/bmg_labtech_smart_control/bmg_labtech_smart_control_parser.py index 71cd65df8d..359a46565c 100644 --- a/src/allotropy/parsers/bmg_labtech_smart_control/bmg_labtech_smart_control_parser.py +++ b/src/allotropy/parsers/bmg_labtech_smart_control/bmg_labtech_smart_control_parser.py @@ -27,7 +27,7 @@ class BmgLabtechSmartControlParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = BmgLabtechSmartControlReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper - UNREAD_DATA_HANDLED = False + UNREAD_DATA_HANDLED = True def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = BmgLabtechSmartControlReader(named_file_contents) diff --git a/src/allotropy/parsers/cfxmaestro/cfxmaestro_structure.py b/src/allotropy/parsers/cfxmaestro/cfxmaestro_structure.py index fde78b6dcb..a56754d3dd 100644 --- a/src/allotropy/parsers/cfxmaestro/cfxmaestro_structure.py +++ b/src/allotropy/parsers/cfxmaestro/cfxmaestro_structure.py @@ -64,12 +64,7 @@ def create_measurement_group( set(constants.PROCESSED_DATA_DOCUMENT_CUSTOM_KEYS) ) # these fields are not need in the asm - data.mark_read( - { - "Cq Mean", - "Unnamed: 0", - } - ) + data.mark_read({"Cq Mean", "Unnamed: 0"}) measurements.append( Measurement( # Measurement metadata diff --git a/src/allotropy/parsers/flowjo/flowjo_parser.py b/src/allotropy/parsers/flowjo/flowjo_parser.py index 337f18d44b..309a3483c6 100644 --- a/src/allotropy/parsers/flowjo/flowjo_parser.py +++ b/src/allotropy/parsers/flowjo/flowjo_parser.py @@ -60,7 +60,7 @@ class FlowjoParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = "wsp" SCHEMA_MAPPER = Mapper - UNREAD_DATA_HANDLED = False + UNREAD_DATA_HANDLED = True def create_data(self, named_file_contents: NamedFileContents) -> Data: try: diff --git a/src/allotropy/parsers/mabtech_apex/mabtech_apex_parser.py b/src/allotropy/parsers/mabtech_apex/mabtech_apex_parser.py index 9b7db7ff46..0de2ab3eb8 100644 --- a/src/allotropy/parsers/mabtech_apex/mabtech_apex_parser.py +++ b/src/allotropy/parsers/mabtech_apex/mabtech_apex_parser.py @@ -25,7 +25,7 @@ class MabtechApexParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = MabtechApexReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper - UNREAD_DATA_HANDLED = False + UNREAD_DATA_HANDLED = True def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = MabtechApexReader.create(named_file_contents) diff --git a/src/allotropy/parsers/methodical_mind/methodical_mind_parser.py b/src/allotropy/parsers/methodical_mind/methodical_mind_parser.py index 50154cdf8b..09ac40b8fd 100644 --- a/src/allotropy/parsers/methodical_mind/methodical_mind_parser.py +++ b/src/allotropy/parsers/methodical_mind/methodical_mind_parser.py @@ -26,7 +26,7 @@ class MethodicalMindParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = "txt" SCHEMA_MAPPER = Mapper - UNREAD_DATA_HANDLED = False + UNREAD_DATA_HANDLED = True def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = MethodicalMindReader(named_file_contents) diff --git a/src/allotropy/parsers/msd_workbench/msd_workbench_parser.py b/src/allotropy/parsers/msd_workbench/msd_workbench_parser.py index 56ebce7d2d..7761ab1b3c 100644 --- a/src/allotropy/parsers/msd_workbench/msd_workbench_parser.py +++ b/src/allotropy/parsers/msd_workbench/msd_workbench_parser.py @@ -37,7 +37,7 @@ class MSDWorkbenchParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = MSDWorkbenchReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper - UNREAD_DATA_HANDLED = False + UNREAD_DATA_HANDLED = True def create_data(self, named_file_contents: NamedFileContents) -> Data: if named_file_contents.extension == "txt": diff --git a/src/allotropy/parsers/revvity_matrix/revvity_matrix_parser.py b/src/allotropy/parsers/revvity_matrix/revvity_matrix_parser.py index 8cfaa1fdb1..bf27b0ccf1 100644 --- a/src/allotropy/parsers/revvity_matrix/revvity_matrix_parser.py +++ b/src/allotropy/parsers/revvity_matrix/revvity_matrix_parser.py @@ -26,7 +26,7 @@ class RevvityMatrixParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = RevvityMatrixReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper - UNREAD_DATA_HANDLED = False + UNREAD_DATA_HANDLED = True def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = RevvityMatrixReader(named_file_contents) diff --git a/src/allotropy/parsers/roche_cedex_hires/roche_cedex_hires_parser.py b/src/allotropy/parsers/roche_cedex_hires/roche_cedex_hires_parser.py index 7ed63b3eeb..c3584827f5 100644 --- a/src/allotropy/parsers/roche_cedex_hires/roche_cedex_hires_parser.py +++ b/src/allotropy/parsers/roche_cedex_hires/roche_cedex_hires_parser.py @@ -24,7 +24,7 @@ class RocheCedexHiResParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = RocheCedexHiResReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper - UNREAD_DATA_HANDLED = False + UNREAD_DATA_HANDLED = True def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = RocheCedexHiResReader(named_file_contents) diff --git a/src/allotropy/parsers/thermo_fisher_nanodrop_one/thermo_fisher_nanodrop_one_parser.py b/src/allotropy/parsers/thermo_fisher_nanodrop_one/thermo_fisher_nanodrop_one_parser.py index 69c8a861ae..f12eb900c2 100644 --- a/src/allotropy/parsers/thermo_fisher_nanodrop_one/thermo_fisher_nanodrop_one_parser.py +++ b/src/allotropy/parsers/thermo_fisher_nanodrop_one/thermo_fisher_nanodrop_one_parser.py @@ -25,7 +25,7 @@ class ThermoFisherNanodropOneParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = "csv,xlsx" SCHEMA_MAPPER = Mapper - UNREAD_DATA_HANDLED = False + UNREAD_DATA_HANDLED = True def read_data( self, named_file_contents: NamedFileContents diff --git a/src/allotropy/parsers/thermo_fisher_qubit4/thermo_fisher_qubit4_parser.py b/src/allotropy/parsers/thermo_fisher_qubit4/thermo_fisher_qubit4_parser.py index 613ead00c9..6cdbe30997 100644 --- a/src/allotropy/parsers/thermo_fisher_qubit4/thermo_fisher_qubit4_parser.py +++ b/src/allotropy/parsers/thermo_fisher_qubit4/thermo_fisher_qubit4_parser.py @@ -23,7 +23,7 @@ class ThermoFisherQubit4Parser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = ThermoFisherQubit4Reader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper - UNREAD_DATA_HANDLED = False + UNREAD_DATA_HANDLED = True def create_data(self, named_file_contents: NamedFileContents) -> Data: return Data( diff --git a/src/allotropy/parsers/unchained_labs_lunatic/unchained_labs_lunatic_parser.py b/src/allotropy/parsers/unchained_labs_lunatic/unchained_labs_lunatic_parser.py index d67767003b..3d19244208 100644 --- a/src/allotropy/parsers/unchained_labs_lunatic/unchained_labs_lunatic_parser.py +++ b/src/allotropy/parsers/unchained_labs_lunatic/unchained_labs_lunatic_parser.py @@ -24,7 +24,7 @@ class UnchainedLabsLunaticParser(VendorParser[Data, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = UnchainedLabsLunaticReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper - UNREAD_DATA_HANDLED = False + UNREAD_DATA_HANDLED = True def create_data(self, named_file_contents: NamedFileContents) -> Data: reader = UnchainedLabsLunaticReader(named_file_contents) diff --git a/src/allotropy/parsers/utils/json.py b/src/allotropy/parsers/utils/json.py index 8f7a654836..56c972d0f9 100644 --- a/src/allotropy/parsers/utils/json.py +++ b/src/allotropy/parsers/utils/json.py @@ -49,24 +49,23 @@ def __del__(self) -> None: return # NOTE: this will be turned on by default when all callers have been updated to pass the warning. if unread_keys := set(self.data.keys()) - self.read_keys: - if os.getenv("WARN_UNUSED_KEYS"): - # Find the creation point in the stack (skip the JsonData.__init__ frame) - creation_point = None - for frame in reversed(self.creation_stack): - if frame.name != "__init__" or "json.py" not in frame.filename: - creation_point = ( - f"{frame.filename}:{frame.lineno} in {frame.name}" - ) - break - - creation_info = ( - f" (created at {creation_point})" if creation_point else "" - ) + # Find the creation point in the stack (skip the JsonData.__init__ frame) + creation_point = None + for frame in reversed(self.creation_stack): + if frame.name != "__init__" or "json.py" not in frame.filename: + creation_point = ( + f"{frame.filename}:{frame.lineno} in {frame.name}" + ) + break + + creation_info = ( + f" (created at {creation_point})" if creation_point else "" + ) - warnings.warn( - f"JsonData went out of scope without reading all keys{creation_info}, unread: {sorted(unread_keys)}.", - stacklevel=2, - ) + warnings.warn( + f"JsonData went out of scope without reading all keys {creation_info}, unread: {sorted(unread_keys)}.", + stacklevel=2, + ) def _get_custom_key(self, key: str) -> float | str | None: if (float_value := self.get(float, key)) is not None: diff --git a/src/allotropy/parsers/utils/pandas.py b/src/allotropy/parsers/utils/pandas.py index 5639be3506..a2253cca41 100644 --- a/src/allotropy/parsers/utils/pandas.py +++ b/src/allotropy/parsers/utils/pandas.py @@ -262,11 +262,10 @@ def __del__(self) -> None: return # NOTE: this will be turned on by default when all callers have been updated to pass the warning. if unread_keys := set(self.series.index.to_list()) - self.read_keys: - if os.getenv("WARN_UNUSED_KEYS"): - warnings.warn( - f"SeriesData went out of scope without reading all keys, unread: {sorted(unread_keys)}.", - stacklevel=2, - ) + warnings.warn( + f"SeriesData went out of scope without reading all keys, unread: {sorted(unread_keys)}.", + stacklevel=2, + ) def _get_custom_key(self, key: str) -> float | str | None: if (float_value := self.get(float, key)) is not None: diff --git a/src/allotropy/parsers/utils/strict_xml_element.py b/src/allotropy/parsers/utils/strict_xml_element.py index ab8b8e8a8c..1b81921ec2 100644 --- a/src/allotropy/parsers/utils/strict_xml_element.py +++ b/src/allotropy/parsers/utils/strict_xml_element.py @@ -82,27 +82,26 @@ def __del__(self) -> None: # Only warn if we've read at least one key but haven't read all keys # This avoids warnings for elements that were never accessed at all if unread_keys and self.read_keys: - if os.getenv("WARN_UNUSED_KEYS"): - # Find the creation point in the stack (skip the StrictXmlElement.__init__ frame) - creation_point = None - for frame in reversed(self.creation_stack): - if ( - frame.name != "__init__" - or "strict_xml_element.py" not in frame.filename - ): - creation_point = ( - f"{frame.filename}:{frame.lineno} in {frame.name}" - ) - break - - creation_info = ( - f" (created at {creation_point})" if creation_point else "" - ) - - warnings.warn( - f"StrictXmlElement went out of scope without reading all keys{creation_info}, unread: {sorted(unread_keys)}.", - stacklevel=2, - ) + # Find the creation point in the stack (skip the StrictXmlElement.__init__ frame) + creation_point = None + for frame in reversed(self.creation_stack): + if ( + frame.name != "__init__" + or "strict_xml_element.py" not in frame.filename + ): + creation_point = ( + f"{frame.filename}:{frame.lineno} in {frame.name}" + ) + break + + creation_info = ( + f" (created at {creation_point})" if creation_point else "" + ) + + warnings.warn( + f"StrictXmlElement went out of scope without reading all keys {creation_info}, unread: {sorted(unread_keys)}.", + stacklevel=2, + ) def _get_all_available_keys(self) -> set[str]: """Get all available keys (attributes and child elements) from the XML element.""" diff --git a/tests/to_allotrope_test.py b/tests/to_allotrope_test.py index ff70bda5d6..b21e5c3eb6 100644 --- a/tests/to_allotrope_test.py +++ b/tests/to_allotrope_test.py @@ -1,6 +1,7 @@ import os from pathlib import Path import re +import warnings import pytest @@ -46,8 +47,6 @@ class ParserTest: def test_positive_cases( self, test_file_path: Path, *, overwrite: bool, warn_unread_keys: bool ) -> None: - if warn_unread_keys or self.VENDOR.unread_data_handled: - os.environ["WARN_UNUSED_KEYS"] = "1" # Special case when input files are json, the are placed in an input/ folder and the results are put # in a corresponding output/ folder. if test_file_path.parts[-2] == "input": @@ -56,9 +55,24 @@ def test_positive_cases( ).with_suffix(".json") else: expected_filepath = test_file_path.with_suffix(".json") - allotrope_dict = from_file( - str(test_file_path), self.VENDOR, encoding=CHARDET_ENCODING - ) + + if warn_unread_keys or self.VENDOR.unread_data_handled: + print("HERE") + print(warn_unread_keys) + print(self.VENDOR.unread_data_handled) + assert False + allotrope_dict = from_file( + str(test_file_path), self.VENDOR, encoding=CHARDET_ENCODING + ) + else: + with warnings.catch_warnings(record=True): + warnings.filterwarnings("ignore") + # warnings.filterwarnings("ignore", "StrictXmlElement went out of scope without reading all keys") + # warnings.filterwarnings("ignore", "JsonData went out of scope without reading all keys") + allotrope_dict = from_file( + str(test_file_path), self.VENDOR, encoding=CHARDET_ENCODING + ) + # If expected output does not exist, assume this is a new file and write it. overwrite = overwrite or not expected_filepath.exists() validate_contents( From 1ba4c62838ddcef8f88fd2806b2b95878fcad9ff Mon Sep 17 00:00:00 2001 From: Nathan Stender Date: Mon, 18 Aug 2025 10:21:51 -0400 Subject: [PATCH 04/13] Fix lint --- src/allotropy/parsers/utils/json.py | 9 ++------- src/allotropy/parsers/utils/pandas.py | 1 - src/allotropy/parsers/utils/strict_xml_element.py | 9 ++------- tests/to_allotrope_test.py | 3 +-- 4 files changed, 5 insertions(+), 17 deletions(-) diff --git a/src/allotropy/parsers/utils/json.py b/src/allotropy/parsers/utils/json.py index 56c972d0f9..b23c5af3b4 100644 --- a/src/allotropy/parsers/utils/json.py +++ b/src/allotropy/parsers/utils/json.py @@ -2,7 +2,6 @@ from collections.abc import Callable, Iterable from enum import Enum -import os import re import traceback from typing import Any, Literal, overload, TypeVar @@ -53,14 +52,10 @@ def __del__(self) -> None: creation_point = None for frame in reversed(self.creation_stack): if frame.name != "__init__" or "json.py" not in frame.filename: - creation_point = ( - f"{frame.filename}:{frame.lineno} in {frame.name}" - ) + creation_point = f"{frame.filename}:{frame.lineno} in {frame.name}" break - creation_info = ( - f" (created at {creation_point})" if creation_point else "" - ) + creation_info = f" (created at {creation_point})" if creation_point else "" warnings.warn( f"JsonData went out of scope without reading all keys {creation_info}, unread: {sorted(unread_keys)}.", diff --git a/src/allotropy/parsers/utils/pandas.py b/src/allotropy/parsers/utils/pandas.py index a2253cca41..bbae763719 100644 --- a/src/allotropy/parsers/utils/pandas.py +++ b/src/allotropy/parsers/utils/pandas.py @@ -2,7 +2,6 @@ from collections.abc import Callable, Iterable from enum import Enum -import os import re from typing import Any, IO, Literal, overload, TypeVar import unicodedata diff --git a/src/allotropy/parsers/utils/strict_xml_element.py b/src/allotropy/parsers/utils/strict_xml_element.py index 1b81921ec2..dcb6d284c5 100644 --- a/src/allotropy/parsers/utils/strict_xml_element.py +++ b/src/allotropy/parsers/utils/strict_xml_element.py @@ -1,6 +1,5 @@ from __future__ import annotations -import os import re import traceback import warnings @@ -89,14 +88,10 @@ def __del__(self) -> None: frame.name != "__init__" or "strict_xml_element.py" not in frame.filename ): - creation_point = ( - f"{frame.filename}:{frame.lineno} in {frame.name}" - ) + creation_point = f"{frame.filename}:{frame.lineno} in {frame.name}" break - creation_info = ( - f" (created at {creation_point})" if creation_point else "" - ) + creation_info = f" (created at {creation_point})" if creation_point else "" warnings.warn( f"StrictXmlElement went out of scope without reading all keys {creation_info}, unread: {sorted(unread_keys)}.", diff --git a/tests/to_allotrope_test.py b/tests/to_allotrope_test.py index b21e5c3eb6..fb0bab0fb7 100644 --- a/tests/to_allotrope_test.py +++ b/tests/to_allotrope_test.py @@ -1,4 +1,3 @@ -import os from pathlib import Path import re import warnings @@ -60,7 +59,7 @@ def test_positive_cases( print("HERE") print(warn_unread_keys) print(self.VENDOR.unread_data_handled) - assert False + raise AssertionError() allotrope_dict = from_file( str(test_file_path), self.VENDOR, encoding=CHARDET_ENCODING ) From 48609614e55cdf7dcc38201f51982683ee0f36df Mon Sep 17 00:00:00 2001 From: Nathan Stender Date: Tue, 19 Aug 2025 18:16:10 -0400 Subject: [PATCH 05/13] Undo changes to base --- src/allotropy/parsers/utils/json.py | 32 +++++++++------- src/allotropy/parsers/utils/pandas.py | 10 +++-- .../parsers/utils/strict_xml_element.py | 38 +++++++++++-------- .../luminex_xponent_structure_test.py | 7 ++++ tests/to_allotrope_test.py | 25 +++--------- 5 files changed, 60 insertions(+), 52 deletions(-) diff --git a/src/allotropy/parsers/utils/json.py b/src/allotropy/parsers/utils/json.py index b23c5af3b4..8f7a654836 100644 --- a/src/allotropy/parsers/utils/json.py +++ b/src/allotropy/parsers/utils/json.py @@ -2,6 +2,7 @@ from collections.abc import Callable, Iterable from enum import Enum +import os import re import traceback from typing import Any, Literal, overload, TypeVar @@ -48,19 +49,24 @@ def __del__(self) -> None: return # NOTE: this will be turned on by default when all callers have been updated to pass the warning. if unread_keys := set(self.data.keys()) - self.read_keys: - # Find the creation point in the stack (skip the JsonData.__init__ frame) - creation_point = None - for frame in reversed(self.creation_stack): - if frame.name != "__init__" or "json.py" not in frame.filename: - creation_point = f"{frame.filename}:{frame.lineno} in {frame.name}" - break - - creation_info = f" (created at {creation_point})" if creation_point else "" - - warnings.warn( - f"JsonData went out of scope without reading all keys {creation_info}, unread: {sorted(unread_keys)}.", - stacklevel=2, - ) + if os.getenv("WARN_UNUSED_KEYS"): + # Find the creation point in the stack (skip the JsonData.__init__ frame) + creation_point = None + for frame in reversed(self.creation_stack): + if frame.name != "__init__" or "json.py" not in frame.filename: + creation_point = ( + f"{frame.filename}:{frame.lineno} in {frame.name}" + ) + break + + creation_info = ( + f" (created at {creation_point})" if creation_point else "" + ) + + warnings.warn( + f"JsonData went out of scope without reading all keys{creation_info}, unread: {sorted(unread_keys)}.", + stacklevel=2, + ) def _get_custom_key(self, key: str) -> float | str | None: if (float_value := self.get(float, key)) is not None: diff --git a/src/allotropy/parsers/utils/pandas.py b/src/allotropy/parsers/utils/pandas.py index bbae763719..5639be3506 100644 --- a/src/allotropy/parsers/utils/pandas.py +++ b/src/allotropy/parsers/utils/pandas.py @@ -2,6 +2,7 @@ from collections.abc import Callable, Iterable from enum import Enum +import os import re from typing import Any, IO, Literal, overload, TypeVar import unicodedata @@ -261,10 +262,11 @@ def __del__(self) -> None: return # NOTE: this will be turned on by default when all callers have been updated to pass the warning. if unread_keys := set(self.series.index.to_list()) - self.read_keys: - warnings.warn( - f"SeriesData went out of scope without reading all keys, unread: {sorted(unread_keys)}.", - stacklevel=2, - ) + if os.getenv("WARN_UNUSED_KEYS"): + warnings.warn( + f"SeriesData went out of scope without reading all keys, unread: {sorted(unread_keys)}.", + stacklevel=2, + ) def _get_custom_key(self, key: str) -> float | str | None: if (float_value := self.get(float, key)) is not None: diff --git a/src/allotropy/parsers/utils/strict_xml_element.py b/src/allotropy/parsers/utils/strict_xml_element.py index dcb6d284c5..ab8b8e8a8c 100644 --- a/src/allotropy/parsers/utils/strict_xml_element.py +++ b/src/allotropy/parsers/utils/strict_xml_element.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import re import traceback import warnings @@ -81,22 +82,27 @@ def __del__(self) -> None: # Only warn if we've read at least one key but haven't read all keys # This avoids warnings for elements that were never accessed at all if unread_keys and self.read_keys: - # Find the creation point in the stack (skip the StrictXmlElement.__init__ frame) - creation_point = None - for frame in reversed(self.creation_stack): - if ( - frame.name != "__init__" - or "strict_xml_element.py" not in frame.filename - ): - creation_point = f"{frame.filename}:{frame.lineno} in {frame.name}" - break - - creation_info = f" (created at {creation_point})" if creation_point else "" - - warnings.warn( - f"StrictXmlElement went out of scope without reading all keys {creation_info}, unread: {sorted(unread_keys)}.", - stacklevel=2, - ) + if os.getenv("WARN_UNUSED_KEYS"): + # Find the creation point in the stack (skip the StrictXmlElement.__init__ frame) + creation_point = None + for frame in reversed(self.creation_stack): + if ( + frame.name != "__init__" + or "strict_xml_element.py" not in frame.filename + ): + creation_point = ( + f"{frame.filename}:{frame.lineno} in {frame.name}" + ) + break + + creation_info = ( + f" (created at {creation_point})" if creation_point else "" + ) + + warnings.warn( + f"StrictXmlElement went out of scope without reading all keys{creation_info}, unread: {sorted(unread_keys)}.", + stacklevel=2, + ) def _get_all_available_keys(self) -> set[str]: """Get all available keys (attributes and child elements) from the XML element.""" diff --git a/tests/parsers/luminex_xponent/luminex_xponent_structure_test.py b/tests/parsers/luminex_xponent/luminex_xponent_structure_test.py index af01cabae9..5916b70edd 100644 --- a/tests/parsers/luminex_xponent/luminex_xponent_structure_test.py +++ b/tests/parsers/luminex_xponent/luminex_xponent_structure_test.py @@ -27,6 +27,7 @@ MeasurementList, ) from allotropy.parsers.utils.pandas import SeriesData +from tests.to_allotrope_test import suppress_unread_warn def get_result_lines() -> list[str]: @@ -58,6 +59,7 @@ def get_result_lines() -> list[str]: ] +@suppress_unread_warn() def test_create_header() -> None: data = pd.DataFrame.from_dict( { @@ -101,6 +103,7 @@ def test_create_header() -> None: "ProtocolPlate", ], ) +@suppress_unread_warn() def test_create_heder_without_required_col(required_col: str) -> None: data = pd.DataFrame.from_dict( { @@ -131,6 +134,7 @@ def test_create_heder_without_required_col(required_col: str) -> None: ) +@suppress_unread_warn() def test_create_calibration_item() -> None: name = "Device Calibration" report = "Passed" @@ -141,6 +145,7 @@ def test_create_calibration_item() -> None: ) == Calibration(name, time, report) +@suppress_unread_warn() def test_create_calibration_item_invalid_line_format() -> None: bad_line = "Bad line." data = SeriesData(pd.Series([bad_line])) @@ -151,6 +156,7 @@ def test_create_calibration_item_invalid_line_format() -> None: create_calibration(data) +@suppress_unread_warn() def test_create_calibration_item_invalid_calibration_result() -> None: bad_result = "bad_result" error = f"Invalid calibration result format, got: ['{bad_result}']" @@ -158,6 +164,7 @@ def test_create_calibration_item_invalid_calibration_result() -> None: create_calibration(SeriesData(pd.Series(["Last CalReport", bad_result]))) +@suppress_unread_warn() def test_create_measurement_list() -> None: results_data = LuminexXponentReader._get_results(CsvReader(get_result_lines())) with mock.patch( diff --git a/tests/to_allotrope_test.py b/tests/to_allotrope_test.py index fb0bab0fb7..044d84485b 100644 --- a/tests/to_allotrope_test.py +++ b/tests/to_allotrope_test.py @@ -1,6 +1,6 @@ +import os from pathlib import Path import re -import warnings import pytest @@ -46,6 +46,8 @@ class ParserTest: def test_positive_cases( self, test_file_path: Path, *, overwrite: bool, warn_unread_keys: bool ) -> None: + if warn_unread_keys: + os.environ["WARN_UNUSED_KEYS"] = "1" # Special case when input files are json, the are placed in an input/ folder and the results are put # in a corresponding output/ folder. if test_file_path.parts[-2] == "input": @@ -54,24 +56,9 @@ def test_positive_cases( ).with_suffix(".json") else: expected_filepath = test_file_path.with_suffix(".json") - - if warn_unread_keys or self.VENDOR.unread_data_handled: - print("HERE") - print(warn_unread_keys) - print(self.VENDOR.unread_data_handled) - raise AssertionError() - allotrope_dict = from_file( - str(test_file_path), self.VENDOR, encoding=CHARDET_ENCODING - ) - else: - with warnings.catch_warnings(record=True): - warnings.filterwarnings("ignore") - # warnings.filterwarnings("ignore", "StrictXmlElement went out of scope without reading all keys") - # warnings.filterwarnings("ignore", "JsonData went out of scope without reading all keys") - allotrope_dict = from_file( - str(test_file_path), self.VENDOR, encoding=CHARDET_ENCODING - ) - + allotrope_dict = from_file( + str(test_file_path), self.VENDOR, encoding=CHARDET_ENCODING + ) # If expected output does not exist, assume this is a new file and write it. overwrite = overwrite or not expected_filepath.exists() validate_contents( From 7dcfb420a4825eb0275f7cfb205a58686a141efb Mon Sep 17 00:00:00 2001 From: Nathan Stender Date: Tue, 19 Aug 2025 22:26:37 -0400 Subject: [PATCH 06/13] Add script --- scripts/check_warn_unread.py | 65 +++++++++++++++++++ .../luminex_xponent_structure_test.py | 7 -- tests/to_allotrope_test.py | 7 +- 3 files changed, 71 insertions(+), 8 deletions(-) create mode 100755 scripts/check_warn_unread.py diff --git a/scripts/check_warn_unread.py b/scripts/check_warn_unread.py new file mode 100755 index 0000000000..18ce6c3175 --- /dev/null +++ b/scripts/check_warn_unread.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +import multiprocessing +import os +import subprocess + +import click + + +def _check_parser(parser: str) -> bool: + result = subprocess.run( + [ + "hatch", + "run", + "test", + f"tests/parsers/{parser}/to_allotrope_test.py::TestParser::test_positive_cases", + ], + capture_output=True, + text=True, + check=True, + ) + return "warning" not in result.stdout + + +def _is_valid_parser(parser: str): + if parser.startswith("."): + return False + if parser.startswith("_"): + return False + if parser == "utils": + return False + return True + + +def _check_warn_unread(parser: str | None = None) -> bool: + parsers = sorted(os.listdir("tests/parsers/")) + if parser: + parsers = [parser] + + parsers = [parser for parser in parsers if _is_valid_parser(parser)] + + if len(parsers) > 1: + with multiprocessing.Pool() as pool: + results = pool.map(_check_parser, parsers) + else: + results = [_check_parser(parsers[0])] + + failing = [ + parser for parser, success in zip(parsers, results, strict=True) if not success + ] + if failing: + print(f"Parsers with unread data warnings: {failing}") + else: + print("Success!") + + return not failing + + +@click.command() +@click.option("--parser", "-p", help="Parser to test, if blank, will test all") +def check_warn_unread(parser: str | None = None) -> bool: + _check_warn_unread(parser) + + +if __name__ == "__main__": + check_warn_unread() diff --git a/tests/parsers/luminex_xponent/luminex_xponent_structure_test.py b/tests/parsers/luminex_xponent/luminex_xponent_structure_test.py index 5916b70edd..af01cabae9 100644 --- a/tests/parsers/luminex_xponent/luminex_xponent_structure_test.py +++ b/tests/parsers/luminex_xponent/luminex_xponent_structure_test.py @@ -27,7 +27,6 @@ MeasurementList, ) from allotropy.parsers.utils.pandas import SeriesData -from tests.to_allotrope_test import suppress_unread_warn def get_result_lines() -> list[str]: @@ -59,7 +58,6 @@ def get_result_lines() -> list[str]: ] -@suppress_unread_warn() def test_create_header() -> None: data = pd.DataFrame.from_dict( { @@ -103,7 +101,6 @@ def test_create_header() -> None: "ProtocolPlate", ], ) -@suppress_unread_warn() def test_create_heder_without_required_col(required_col: str) -> None: data = pd.DataFrame.from_dict( { @@ -134,7 +131,6 @@ def test_create_heder_without_required_col(required_col: str) -> None: ) -@suppress_unread_warn() def test_create_calibration_item() -> None: name = "Device Calibration" report = "Passed" @@ -145,7 +141,6 @@ def test_create_calibration_item() -> None: ) == Calibration(name, time, report) -@suppress_unread_warn() def test_create_calibration_item_invalid_line_format() -> None: bad_line = "Bad line." data = SeriesData(pd.Series([bad_line])) @@ -156,7 +151,6 @@ def test_create_calibration_item_invalid_line_format() -> None: create_calibration(data) -@suppress_unread_warn() def test_create_calibration_item_invalid_calibration_result() -> None: bad_result = "bad_result" error = f"Invalid calibration result format, got: ['{bad_result}']" @@ -164,7 +158,6 @@ def test_create_calibration_item_invalid_calibration_result() -> None: create_calibration(SeriesData(pd.Series(["Last CalReport", bad_result]))) -@suppress_unread_warn() def test_create_measurement_list() -> None: results_data = LuminexXponentReader._get_results(CsvReader(get_result_lines())) with mock.patch( diff --git a/tests/to_allotrope_test.py b/tests/to_allotrope_test.py index 044d84485b..d8326feb47 100644 --- a/tests/to_allotrope_test.py +++ b/tests/to_allotrope_test.py @@ -9,6 +9,7 @@ from allotropy.parser_factory import Vendor from allotropy.testing.utils import from_file, validate_contents from allotropy.to_allotrope import allotrope_from_file, allotrope_model_from_file +from scripts.check_warn_unread import _check_warn_unread INVALID_FILE_PATH = "not/a/path" EXPECTED_ERROR_MESSAGE = f"File not found: {INVALID_FILE_PATH}" @@ -46,7 +47,7 @@ class ParserTest: def test_positive_cases( self, test_file_path: Path, *, overwrite: bool, warn_unread_keys: bool ) -> None: - if warn_unread_keys: + if warn_unread_keys or self.VENDOR.unread_data_handled: os.environ["WARN_UNUSED_KEYS"] = "1" # Special case when input files are json, the are placed in an input/ folder and the results are put # in a corresponding output/ folder. @@ -66,3 +67,7 @@ def test_positive_cases( expected_filepath, write_actual_to_expected_on_fail=overwrite, ) + + def test_no_unread_warnings(self) -> None: + parser = str(self.__class__).split(".")[2] + assert _check_warn_unread(parser) From 9b2dd76223210552921abfb548365fa0467d8c13 Mon Sep 17 00:00:00 2001 From: Nathan Stender Date: Tue, 19 Aug 2025 23:43:43 -0400 Subject: [PATCH 07/13] Fix lint --- scripts/check_warn_unread.py | 65 ------------------- .../bmg_labtech_smart_control_structure.py | 2 +- src/allotropy/parsers/utils/json.py | 32 ++++----- src/allotropy/parsers/utils/pandas.py | 17 ++--- .../parsers/utils/strict_xml_element.py | 38 +++++------ ...allotropy_test.py => to_allotrope_test.py} | 0 ...allotropy_test.py => to_allotrope_test.py} | 0 tests/to_allotrope_test.py | 51 +++++++++++---- 8 files changed, 79 insertions(+), 126 deletions(-) delete mode 100755 scripts/check_warn_unread.py rename tests/parsers/roche_cedex_hires/{to_allotropy_test.py => to_allotrope_test.py} (100%) rename tests/parsers/thermo_fisher_qubit4/{to_allotropy_test.py => to_allotrope_test.py} (100%) diff --git a/scripts/check_warn_unread.py b/scripts/check_warn_unread.py deleted file mode 100755 index 18ce6c3175..0000000000 --- a/scripts/check_warn_unread.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python3 -import multiprocessing -import os -import subprocess - -import click - - -def _check_parser(parser: str) -> bool: - result = subprocess.run( - [ - "hatch", - "run", - "test", - f"tests/parsers/{parser}/to_allotrope_test.py::TestParser::test_positive_cases", - ], - capture_output=True, - text=True, - check=True, - ) - return "warning" not in result.stdout - - -def _is_valid_parser(parser: str): - if parser.startswith("."): - return False - if parser.startswith("_"): - return False - if parser == "utils": - return False - return True - - -def _check_warn_unread(parser: str | None = None) -> bool: - parsers = sorted(os.listdir("tests/parsers/")) - if parser: - parsers = [parser] - - parsers = [parser for parser in parsers if _is_valid_parser(parser)] - - if len(parsers) > 1: - with multiprocessing.Pool() as pool: - results = pool.map(_check_parser, parsers) - else: - results = [_check_parser(parsers[0])] - - failing = [ - parser for parser, success in zip(parsers, results, strict=True) if not success - ] - if failing: - print(f"Parsers with unread data warnings: {failing}") - else: - print("Success!") - - return not failing - - -@click.command() -@click.option("--parser", "-p", help="Parser to test, if blank, will test all") -def check_warn_unread(parser: str | None = None) -> bool: - _check_warn_unread(parser) - - -if __name__ == "__main__": - check_warn_unread() diff --git a/src/allotropy/parsers/bmg_labtech_smart_control/bmg_labtech_smart_control_structure.py b/src/allotropy/parsers/bmg_labtech_smart_control/bmg_labtech_smart_control_structure.py index e1bf8c4fe6..6212a58d34 100644 --- a/src/allotropy/parsers/bmg_labtech_smart_control/bmg_labtech_smart_control_structure.py +++ b/src/allotropy/parsers/bmg_labtech_smart_control/bmg_labtech_smart_control_structure.py @@ -181,7 +181,7 @@ def create_calculated_data_documents( measurements = [group.measurements[0] for group in measurement_groups] calc_data_measurements = [] for idx, measurement in enumerate(measurements): - corrected_value = SeriesData(reader.data.iloc[idx]).get( + corrected_value = SeriesData(reader.data.iloc[idx], ignore_unread=True).get( float, "Blank corrected based on Raw Data (480-14/520-30)" ) calc_data_measurements.append( diff --git a/src/allotropy/parsers/utils/json.py b/src/allotropy/parsers/utils/json.py index 8f7a654836..4b8e3e6a3b 100644 --- a/src/allotropy/parsers/utils/json.py +++ b/src/allotropy/parsers/utils/json.py @@ -2,7 +2,6 @@ from collections.abc import Callable, Iterable from enum import Enum -import os import re import traceback from typing import Any, Literal, overload, TypeVar @@ -49,24 +48,19 @@ def __del__(self) -> None: return # NOTE: this will be turned on by default when all callers have been updated to pass the warning. if unread_keys := set(self.data.keys()) - self.read_keys: - if os.getenv("WARN_UNUSED_KEYS"): - # Find the creation point in the stack (skip the JsonData.__init__ frame) - creation_point = None - for frame in reversed(self.creation_stack): - if frame.name != "__init__" or "json.py" not in frame.filename: - creation_point = ( - f"{frame.filename}:{frame.lineno} in {frame.name}" - ) - break - - creation_info = ( - f" (created at {creation_point})" if creation_point else "" - ) - - warnings.warn( - f"JsonData went out of scope without reading all keys{creation_info}, unread: {sorted(unread_keys)}.", - stacklevel=2, - ) + # Find the creation point in the stack (skip the JsonData.__init__ frame) + creation_point = None + for frame in reversed(self.creation_stack): + if frame.name != "__init__" or "json.py" not in frame.filename: + creation_point = f"{frame.filename}:{frame.lineno} in {frame.name}" + break + + creation_info = f" (created at {creation_point})" if creation_point else "" + + warnings.warn( + f"JsonData went out of scope without reading all keys{creation_info}, unread: {sorted(unread_keys)}.", + stacklevel=2, + ) def _get_custom_key(self, key: str) -> float | str | None: if (float_value := self.get(float, key)) is not None: diff --git a/src/allotropy/parsers/utils/pandas.py b/src/allotropy/parsers/utils/pandas.py index 5639be3506..140fde06bd 100644 --- a/src/allotropy/parsers/utils/pandas.py +++ b/src/allotropy/parsers/utils/pandas.py @@ -2,7 +2,6 @@ from collections.abc import Callable, Iterable from enum import Enum -import os import re from typing import Any, IO, Literal, overload, TypeVar import unicodedata @@ -252,21 +251,23 @@ def _validate_raw(v: Any, mode: ValidateRawMode | None) -> Any: return None if (v is None or pd.isna(v)) else v return v - def __init__(self, series: pd.Series[Any] | None = None) -> None: + def __init__( + self, series: pd.Series[Any] | None = None, ignore_unread: bool | None = None + ) -> None: self.series = pd.Series() if series is None else series self.read_keys: set[str] = set() self.errored = False + self.ignore_unread = ignore_unread or False def __del__(self) -> None: - if self.errored: + if self.errored or self.ignore_unread: return # NOTE: this will be turned on by default when all callers have been updated to pass the warning. if unread_keys := set(self.series.index.to_list()) - self.read_keys: - if os.getenv("WARN_UNUSED_KEYS"): - warnings.warn( - f"SeriesData went out of scope without reading all keys, unread: {sorted(unread_keys)}.", - stacklevel=2, - ) + warnings.warn( + f"SeriesData went out of scope without reading all keys, unread: {sorted(unread_keys)}.", + stacklevel=2, + ) def _get_custom_key(self, key: str) -> float | str | None: if (float_value := self.get(float, key)) is not None: diff --git a/src/allotropy/parsers/utils/strict_xml_element.py b/src/allotropy/parsers/utils/strict_xml_element.py index ab8b8e8a8c..10b56e9edb 100644 --- a/src/allotropy/parsers/utils/strict_xml_element.py +++ b/src/allotropy/parsers/utils/strict_xml_element.py @@ -1,6 +1,5 @@ from __future__ import annotations -import os import re import traceback import warnings @@ -82,27 +81,22 @@ def __del__(self) -> None: # Only warn if we've read at least one key but haven't read all keys # This avoids warnings for elements that were never accessed at all if unread_keys and self.read_keys: - if os.getenv("WARN_UNUSED_KEYS"): - # Find the creation point in the stack (skip the StrictXmlElement.__init__ frame) - creation_point = None - for frame in reversed(self.creation_stack): - if ( - frame.name != "__init__" - or "strict_xml_element.py" not in frame.filename - ): - creation_point = ( - f"{frame.filename}:{frame.lineno} in {frame.name}" - ) - break - - creation_info = ( - f" (created at {creation_point})" if creation_point else "" - ) - - warnings.warn( - f"StrictXmlElement went out of scope without reading all keys{creation_info}, unread: {sorted(unread_keys)}.", - stacklevel=2, - ) + # Find the creation point in the stack (skip the StrictXmlElement.__init__ frame) + creation_point = None + for frame in reversed(self.creation_stack): + if ( + frame.name != "__init__" + or "strict_xml_element.py" not in frame.filename + ): + creation_point = f"{frame.filename}:{frame.lineno} in {frame.name}" + break + + creation_info = f" (created at {creation_point})" if creation_point else "" + + warnings.warn( + f"StrictXmlElement went out of scope without reading all keys{creation_info}, unread: {sorted(unread_keys)}.", + stacklevel=2, + ) def _get_all_available_keys(self) -> set[str]: """Get all available keys (attributes and child elements) from the XML element.""" diff --git a/tests/parsers/roche_cedex_hires/to_allotropy_test.py b/tests/parsers/roche_cedex_hires/to_allotrope_test.py similarity index 100% rename from tests/parsers/roche_cedex_hires/to_allotropy_test.py rename to tests/parsers/roche_cedex_hires/to_allotrope_test.py diff --git a/tests/parsers/thermo_fisher_qubit4/to_allotropy_test.py b/tests/parsers/thermo_fisher_qubit4/to_allotrope_test.py similarity index 100% rename from tests/parsers/thermo_fisher_qubit4/to_allotropy_test.py rename to tests/parsers/thermo_fisher_qubit4/to_allotrope_test.py diff --git a/tests/to_allotrope_test.py b/tests/to_allotrope_test.py index d8326feb47..27509c6045 100644 --- a/tests/to_allotrope_test.py +++ b/tests/to_allotrope_test.py @@ -1,6 +1,6 @@ -import os from pathlib import Path import re +import warnings import pytest @@ -9,7 +9,6 @@ from allotropy.parser_factory import Vendor from allotropy.testing.utils import from_file, validate_contents from allotropy.to_allotrope import allotrope_from_file, allotrope_model_from_file -from scripts.check_warn_unread import _check_warn_unread INVALID_FILE_PATH = "not/a/path" EXPECTED_ERROR_MESSAGE = f"File not found: {INVALID_FILE_PATH}" @@ -47,8 +46,6 @@ class ParserTest: def test_positive_cases( self, test_file_path: Path, *, overwrite: bool, warn_unread_keys: bool ) -> None: - if warn_unread_keys or self.VENDOR.unread_data_handled: - os.environ["WARN_UNUSED_KEYS"] = "1" # Special case when input files are json, the are placed in an input/ folder and the results are put # in a corresponding output/ folder. if test_file_path.parts[-2] == "input": @@ -57,9 +54,45 @@ def test_positive_cases( ).with_suffix(".json") else: expected_filepath = test_file_path.with_suffix(".json") - allotrope_dict = from_file( - str(test_file_path), self.VENDOR, encoding=CHARDET_ENCODING - ) + + with warnings.catch_warnings(record=True) as captured_warnings: + if not (warn_unread_keys or self.VENDOR.unread_data_handled): + warnings.filterwarnings( + "ignore", + category=UserWarning, + message="JsonData went out of scope without reading all keys", + ) + warnings.filterwarnings( + "ignore", + category=UserWarning, + message="SeriesData went out of scope without reading all keys", + ) + warnings.filterwarnings( + "ignore", + category=UserWarning, + message="StrictXmlElement went out of scope without reading all keys", + ) + allotrope_dict = from_file( + str(test_file_path), self.VENDOR, encoding=CHARDET_ENCODING + ) + + # If parser is marked as having unread data handled, error on any unread data warnings. + if self.VENDOR.unread_data_handled and captured_warnings: + for captured_warning in captured_warnings: + warnings.warn_explicit( + message=captured_warning.message, + category=captured_warning.category, + filename=captured_warning.filename, + lineno=captured_warning.lineno, + source=captured_warning.source, + ) + warning = captured_warning.message + if isinstance( + warning, UserWarning + ) and "went out of scope without reading all keys" in str(warning): + msg = "Parser is marked as UNREAD_DATA_HANDLED, but had unread data warnings!" + raise AssertionError(msg) + # If expected output does not exist, assume this is a new file and write it. overwrite = overwrite or not expected_filepath.exists() validate_contents( @@ -67,7 +100,3 @@ def test_positive_cases( expected_filepath, write_actual_to_expected_on_fail=overwrite, ) - - def test_no_unread_warnings(self) -> None: - parser = str(self.__class__).split(".")[2] - assert _check_warn_unread(parser) From 74ee630de29c50aa2bdc2f409f5fb505123c2359 Mon Sep 17 00:00:00 2001 From: Nathan Stender Date: Wed, 20 Aug 2025 00:11:25 -0400 Subject: [PATCH 08/13] Clean up --- pyproject.toml | 1 + tests/conftest.py | 11 +++++++++++ tests/to_allotrope_test.py | 19 +++++-------------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6ef7c1688d..e61ea3c96e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -230,6 +230,7 @@ cov = [ pythonpath = "src" markers = [ "long: mark a test that takes a long time to run", + "skip_ignore_unread_warnings: mark a test should warn for unread data - only intended to be used in to_allotrope_test.py", ] [tool.ruff] diff --git a/tests/conftest.py b/tests/conftest.py index c01e21d1a8..4afbe303c1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ from pathlib import Path import re from typing import Any +import warnings import pytest from pytest import FixtureRequest, Parser @@ -14,6 +15,16 @@ PARSE_FOLDER = [".rslt"] +# We only want to raise unread data warnings in to_allotrope_test, so ignore by default. +@pytest.fixture(autouse=True) +def ignore_unread_warnings(request): + behavior = ( + "default" if "skip_ignore_unread_warnings" in request.keywords else "ignore" + ) + warnings.filterwarnings(behavior, ".*went out of scope without reading all keys.*") + yield + + def pytest_addoption(parser: Parser) -> None: parser.addoption( "--overwrite", diff --git a/tests/to_allotrope_test.py b/tests/to_allotrope_test.py index 27509c6045..0c1835bc7c 100644 --- a/tests/to_allotrope_test.py +++ b/tests/to_allotrope_test.py @@ -43,6 +43,7 @@ class ParserTest: VENDOR: Vendor # test_file_path is automatically populated with all files in testdata folder next to the test file. + @pytest.mark.skip_ignore_unread_warnings def test_positive_cases( self, test_file_path: Path, *, overwrite: bool, warn_unread_keys: bool ) -> None: @@ -55,22 +56,12 @@ def test_positive_cases( else: expected_filepath = test_file_path.with_suffix(".json") - with warnings.catch_warnings(record=True) as captured_warnings: + with warnings.catch_warnings( + record=self.VENDOR.unread_data_handled + ) as captured_warnings: if not (warn_unread_keys or self.VENDOR.unread_data_handled): warnings.filterwarnings( - "ignore", - category=UserWarning, - message="JsonData went out of scope without reading all keys", - ) - warnings.filterwarnings( - "ignore", - category=UserWarning, - message="SeriesData went out of scope without reading all keys", - ) - warnings.filterwarnings( - "ignore", - category=UserWarning, - message="StrictXmlElement went out of scope without reading all keys", + "ignore", ".*went out of scope without reading all keys.*" ) allotrope_dict = from_file( str(test_file_path), self.VENDOR, encoding=CHARDET_ENCODING From af4c77bf8e1d24304adebc6e5b803864ec496402 Mon Sep 17 00:00:00 2001 From: Nathan Stender Date: Wed, 20 Aug 2025 00:19:08 -0400 Subject: [PATCH 09/13] Fix typing --- src/allotropy/parsers/utils/pandas.py | 2 +- tests/conftest.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/allotropy/parsers/utils/pandas.py b/src/allotropy/parsers/utils/pandas.py index 140fde06bd..3a314c45a9 100644 --- a/src/allotropy/parsers/utils/pandas.py +++ b/src/allotropy/parsers/utils/pandas.py @@ -265,7 +265,7 @@ def __del__(self) -> None: # NOTE: this will be turned on by default when all callers have been updated to pass the warning. if unread_keys := set(self.series.index.to_list()) - self.read_keys: warnings.warn( - f"SeriesData went out of scope without reading all keys, unread: {sorted(unread_keys)}.", + f"SeriesData went out of scope without reading all keys, unread: {sorted(str(k) for k in unread_keys)}.", stacklevel=2, ) diff --git a/tests/conftest.py b/tests/conftest.py index 4afbe303c1..afedb89473 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,8 @@ +from collections.abc import Generator import inspect from pathlib import Path import re -from typing import Any +from typing import Any, Literal import warnings import pytest @@ -17,8 +18,10 @@ # We only want to raise unread data warnings in to_allotrope_test, so ignore by default. @pytest.fixture(autouse=True) -def ignore_unread_warnings(request): - behavior = ( +def ignore_unread_warnings( + request: pytest.FixtureRequest, +) -> Generator[None, None, None]: + behavior: Literal["default", "ignore"] = ( "default" if "skip_ignore_unread_warnings" in request.keywords else "ignore" ) warnings.filterwarnings(behavior, ".*went out of scope without reading all keys.*") From 51aba314527815cdc27834570dd9f83f0c0c770d Mon Sep 17 00:00:00 2001 From: Nathan Stender Date: Thu, 21 Aug 2025 13:06:36 -0400 Subject: [PATCH 10/13] Merge main --- src/allotropy/parsers/luminex_xponent/luminex_xponent_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allotropy/parsers/luminex_xponent/luminex_xponent_parser.py b/src/allotropy/parsers/luminex_xponent/luminex_xponent_parser.py index 33d156c845..794cfc0156 100644 --- a/src/allotropy/parsers/luminex_xponent/luminex_xponent_parser.py +++ b/src/allotropy/parsers/luminex_xponent/luminex_xponent_parser.py @@ -23,7 +23,7 @@ class LuminexXponentParser(VendorParser[MapperData, Model]): RELEASE_STATE = ReleaseState.RECOMMENDED SUPPORTED_EXTENSIONS = LuminexXponentReader.SUPPORTED_EXTENSIONS SCHEMA_MAPPER = Mapper - UNREAD_DATA_HANDLED = False + UNREAD_DATA_HANDLED = True def create_data(self, named_file_contents: NamedFileContents) -> MapperData: reader = LuminexXponentReader(named_file_contents) From 457a56a9e8883ed05876ae307bfdd71f4822eb96 Mon Sep 17 00:00:00 2001 From: Nathan Stender Date: Wed, 27 Aug 2025 11:24:58 -0400 Subject: [PATCH 11/13] Upgrade requirements --- pyproject.toml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e61ea3c96e..e5d4956958 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -174,7 +174,7 @@ extra-dependencies = [ "pandas-stubs == 2.1.4.231227", "types-olefile", # For pytest types - "pytest >= 7.4.0", + "pytest >= 8.4.1", # For check graph generation "networkx-stubs", ] @@ -207,9 +207,10 @@ extra-dependencies = [ "coverage[toml] >= 6.5", "deepdiff >= 6.5.0", "more-itertools >= 10.1.0", - "pytest >= 7.4.0", - "pytest-xdist >= 3.6.1", - "types-olefile" + "pluggy >= 1.6.0", + "pytest >= 8.4.1", + "pytest-xdist >= 3.8.0", + "types-olefile", ] [tool.hatch.envs.test.scripts] From 121e379f910b8643bae3ba628b6d1ef41fe048fc Mon Sep 17 00:00:00 2001 From: Nathan Stender Date: Wed, 27 Aug 2025 11:52:42 -0400 Subject: [PATCH 12/13] Fix parallelism test --- tests/to_allotrope_test.py | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/tests/to_allotrope_test.py b/tests/to_allotrope_test.py index 0c1835bc7c..fae6436319 100644 --- a/tests/to_allotrope_test.py +++ b/tests/to_allotrope_test.py @@ -56,33 +56,30 @@ def test_positive_cases( else: expected_filepath = test_file_path.with_suffix(".json") - with warnings.catch_warnings( - record=self.VENDOR.unread_data_handled - ) as captured_warnings: - if not (warn_unread_keys or self.VENDOR.unread_data_handled): - warnings.filterwarnings( - "ignore", ".*went out of scope without reading all keys.*" - ) + with warnings.catch_warnings(record=True) as captured_warnings: allotrope_dict = from_file( str(test_file_path), self.VENDOR, encoding=CHARDET_ENCODING ) # If parser is marked as having unread data handled, error on any unread data warnings. - if self.VENDOR.unread_data_handled and captured_warnings: - for captured_warning in captured_warnings: - warnings.warn_explicit( - message=captured_warning.message, - category=captured_warning.category, - filename=captured_warning.filename, - lineno=captured_warning.lineno, - source=captured_warning.source, - ) - warning = captured_warning.message - if isinstance( - warning, UserWarning - ) and "went out of scope without reading all keys" in str(warning): + for captured_warning in captured_warnings: + if isinstance( + captured_warning.message, UserWarning + ) and "went out of scope without reading all keys" in str( + captured_warning.message + ): + if self.VENDOR.unread_data_handled: msg = "Parser is marked as UNREAD_DATA_HANDLED, but had unread data warnings!" raise AssertionError(msg) + if not warn_unread_keys: + continue + warnings.warn_explicit( + message=captured_warning.message, + category=captured_warning.category, + filename=captured_warning.filename, + lineno=captured_warning.lineno, + source=captured_warning.source, + ) # If expected output does not exist, assume this is a new file and write it. overwrite = overwrite or not expected_filepath.exists() From 22e5acb1168c3008c19203394283908db8920538 Mon Sep 17 00:00:00 2001 From: Nathan Stender Date: Wed, 27 Aug 2025 13:17:08 -0400 Subject: [PATCH 13/13] Parallelism is defeated --- src/allotropy/parsers/utils/json.py | 5 ++-- src/allotropy/parsers/utils/pandas.py | 5 ++-- .../parsers/utils/strict_xml_element.py | 5 ++-- tests/to_allotrope_test.py | 24 +++++++++++-------- 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/src/allotropy/parsers/utils/json.py b/src/allotropy/parsers/utils/json.py index 4b8e3e6a3b..15781714f9 100644 --- a/src/allotropy/parsers/utils/json.py +++ b/src/allotropy/parsers/utils/json.py @@ -2,6 +2,7 @@ from collections.abc import Callable, Iterable from enum import Enum +import os import re import traceback from typing import Any, Literal, overload, TypeVar @@ -44,7 +45,7 @@ def __init__(self, data: dict[str, Any] | None = None) -> None: self.creation_stack = traceback.extract_stack() def __del__(self) -> None: - if self.errored: + if self.errored or not os.getenv("WARN_UNUSED_KEYS"): return # NOTE: this will be turned on by default when all callers have been updated to pass the warning. if unread_keys := set(self.data.keys()) - self.read_keys: @@ -58,7 +59,7 @@ def __del__(self) -> None: creation_info = f" (created at {creation_point})" if creation_point else "" warnings.warn( - f"JsonData went out of scope without reading all keys{creation_info}, unread: {sorted(unread_keys)}.", + f"JsonData went out of scope without reading all keys with UNREAD_DATA_HANDLED={bool(os.getenv('UNREAD_DATA_HANDLED'))} for {os.getenv('VENDOR')}{creation_info}, unread: {sorted(unread_keys)}.", stacklevel=2, ) diff --git a/src/allotropy/parsers/utils/pandas.py b/src/allotropy/parsers/utils/pandas.py index cf988d61be..e290f7d0ae 100644 --- a/src/allotropy/parsers/utils/pandas.py +++ b/src/allotropy/parsers/utils/pandas.py @@ -2,6 +2,7 @@ from collections.abc import Callable, Iterable from enum import Enum +import os import re from typing import Any, IO, Literal, overload, TypeVar import unicodedata @@ -260,12 +261,12 @@ def __init__( self.ignore_unread = ignore_unread or False def __del__(self) -> None: - if self.errored or self.ignore_unread: + if self.errored or self.ignore_unread or not os.getenv("WARN_UNUSED_KEYS"): return # NOTE: this will be turned on by default when all callers have been updated to pass the warning. if unread_keys := set(self.series.index.to_list()) - self.read_keys: warnings.warn( - f"SeriesData went out of scope without reading all keys, unread: {sorted(str(k) for k in unread_keys)}.", + f"SeriesData went out of scope without reading all keys with UNREAD_DATA_HANDLED={bool(os.getenv('UNREAD_DATA_HANDLED'))} for {os.getenv('VENDOR')}, unread: {sorted(str(k) for k in unread_keys)}.", stacklevel=2, ) diff --git a/src/allotropy/parsers/utils/strict_xml_element.py b/src/allotropy/parsers/utils/strict_xml_element.py index 29b4657c33..00a5794dd4 100644 --- a/src/allotropy/parsers/utils/strict_xml_element.py +++ b/src/allotropy/parsers/utils/strict_xml_element.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import re import traceback import warnings @@ -32,7 +33,7 @@ def __init__( self.mark_read(mark_read or set()) def __del__(self) -> None: - if self.errored: + if self.errored or not os.getenv("WARN_UNUSED_KEYS"): return # NOTE: this will be turned on by default when all callers have been updated to pass the warning. # Only consider attributes as available keys, not child elements @@ -100,7 +101,7 @@ def __del__(self) -> None: creation_info = f" (created at {creation_point})" if creation_point else "" warnings.warn( - f"StrictXmlElement '{self.element.tag}' went out of scope without reading all keys {creation_info}, unread: {sorted(unread_keys)}.", + f"StrictXmlElement '{self.element.tag}' went out of scope without reading all keys with UNREAD_DATA_HANDLED={bool(os.getenv('UNREAD_DATA_HANDLED'))} for {os.getenv('VENDOR')}{creation_info}, unread: {sorted(unread_keys)}.", stacklevel=2, ) diff --git a/tests/to_allotrope_test.py b/tests/to_allotrope_test.py index fae6436319..63d7afd294 100644 --- a/tests/to_allotrope_test.py +++ b/tests/to_allotrope_test.py @@ -1,3 +1,4 @@ +import os from pathlib import Path import re import warnings @@ -47,6 +48,14 @@ class ParserTest: def test_positive_cases( self, test_file_path: Path, *, overwrite: bool, warn_unread_keys: bool ) -> None: + os.environ["VENDOR"] = self.VENDOR.value + os.environ.pop("WARN_UNUSED_KEYS", None) + os.environ.pop("UNREAD_DATA_HANDLED", None) + if warn_unread_keys or self.VENDOR.unread_data_handled: + os.environ["WARN_UNUSED_KEYS"] = "1" + if self.VENDOR.unread_data_handled: + os.environ["UNREAD_DATA_HANDLED"] = "1" + # Special case when input files are json, the are placed in an input/ folder and the results are put # in a corresponding output/ folder. if test_file_path.parts[-2] == "input": @@ -63,16 +72,6 @@ def test_positive_cases( # If parser is marked as having unread data handled, error on any unread data warnings. for captured_warning in captured_warnings: - if isinstance( - captured_warning.message, UserWarning - ) and "went out of scope without reading all keys" in str( - captured_warning.message - ): - if self.VENDOR.unread_data_handled: - msg = "Parser is marked as UNREAD_DATA_HANDLED, but had unread data warnings!" - raise AssertionError(msg) - if not warn_unread_keys: - continue warnings.warn_explicit( message=captured_warning.message, category=captured_warning.category, @@ -80,6 +79,11 @@ def test_positive_cases( lineno=captured_warning.lineno, source=captured_warning.source, ) + if isinstance( + captured_warning.message, UserWarning + ) and "UNREAD_DATA_HANDLED=True" in str(captured_warning.message): + msg = "Parser is marked as UNREAD_DATA_HANDLED, but had unread data warnings!" + raise AssertionError(msg) # If expected output does not exist, assume this is a new file and write it. overwrite = overwrite or not expected_filepath.exists()