diff --git a/pyproject.toml b/pyproject.toml index 75f33ac502..eb3c823bfb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ + "babel >= 2.9.0", "cattrs >= 23.2.0", "chardet >= 5.2.0", "defusedxml >= 0.7.1", diff --git a/src/allotropy/parsers/agilent_gen5/agilent_gen5_structure.py b/src/allotropy/parsers/agilent_gen5/agilent_gen5_structure.py index 832259a592..02d4f747e5 100644 --- a/src/allotropy/parsers/agilent_gen5/agilent_gen5_structure.py +++ b/src/allotropy/parsers/agilent_gen5/agilent_gen5_structure.py @@ -77,7 +77,7 @@ DataSource, Referenceable, ) -from allotropy.parsers.utils.pandas import read_csv, SeriesData +from allotropy.parsers.utils.pandas import read_csv, series_to_float_list, SeriesData from allotropy.parsers.utils.uuids import random_uuid_str from allotropy.parsers.utils.values import ( try_float, @@ -1246,7 +1246,8 @@ def create_spectrum_results( data = read_csv(StringIO("\n".join(wavelengths_sections)), sep="\t") try: - wavelengths = data["Wavelength"].astype(float).tolist() + # Use series_to_float_list for locale support + wavelengths = series_to_float_list(data["Wavelength"], "wavelength") except (ValueError, KeyError): return [], [] diff --git a/src/allotropy/parsers/appbio_absolute_q/appbio_absolute_q_structure.py b/src/allotropy/parsers/appbio_absolute_q/appbio_absolute_q_structure.py index 951befb1c2..998555756f 100644 --- a/src/allotropy/parsers/appbio_absolute_q/appbio_absolute_q_structure.py +++ b/src/allotropy/parsers/appbio_absolute_q/appbio_absolute_q_structure.py @@ -42,7 +42,7 @@ NEGATIVE_ZERO, NOT_APPLICABLE, ) -from allotropy.parsers.utils.pandas import map_rows, SeriesData +from allotropy.parsers.utils.pandas import map_rows, series_to_float_list, SeriesData from allotropy.parsers.utils.uuids import random_uuid_str @@ -233,7 +233,7 @@ def create_cube_well_items(well_data: pd.DataFrame) -> list[WellItem]: cycle_count = DataCubeComponent( FieldComponentDatatype.integer, "cycle count", "#" ) - index_column = well_data["Index"].astype(float).tolist() + index_column = series_to_float_list(well_data["Index"], "index") data = SeriesData(well_data.iloc[0]) well_items: list[WellItem] = [] @@ -291,7 +291,9 @@ def create_cube_well_items(well_data: pd.DataFrame) -> list[WellItem]: ) ], dimensions=[index_column], - measures=[well_data[dye_setting].astype(float).tolist()], + measures=[ + series_to_float_list(well_data[dye_setting], "fluorescence") + ], ), passive_reference_dye_data_cube=DataCube( label="passive reference dye", @@ -305,7 +307,10 @@ def create_cube_well_items(well_data: pd.DataFrame) -> list[WellItem]: ], dimensions=[index_column], measures=[ - well_data[passive_dye_settings[0]].astype(float).tolist() + series_to_float_list( + well_data[passive_dye_settings[0]], + "passive_fluorescence", + ) ], ), errors=errors, diff --git a/src/allotropy/parsers/cytiva_biacore_t200_control/cytiva_biacore_t200_control_data_creator.py b/src/allotropy/parsers/cytiva_biacore_t200_control/cytiva_biacore_t200_control_data_creator.py index 38e4100b16..bcd80402a4 100644 --- a/src/allotropy/parsers/cytiva_biacore_t200_control/cytiva_biacore_t200_control_data_creator.py +++ b/src/allotropy/parsers/cytiva_biacore_t200_control/cytiva_biacore_t200_control_data_creator.py @@ -47,6 +47,7 @@ from allotropy.parsers.utils.calculated_data_documents.definition import ( CalculatedDocument, ) +from allotropy.parsers.utils.pandas import series_to_float_list from allotropy.parsers.utils.values import quantity_or_none, try_float_or_none from allotropy.types import DictType @@ -62,8 +63,10 @@ def _get_sensorgram_datacube(sensorgram_data: pd.DataFrame) -> DataCube: structure_measures=[ DataCubeComponent(FieldComponentDatatype.double, "resonance", "RU") ], - dimensions=[sensorgram_data["Time (s)"].astype(float).to_list()], - measures=[sensorgram_data["Sensorgram (RU)"].astype(float).to_list()], + dimensions=[series_to_float_list(sensorgram_data["Time (s)"], "time")], + measures=[ + series_to_float_list(sensorgram_data["Sensorgram (RU)"], "sensorgram") + ], ) diff --git a/src/allotropy/parsers/luminex_xponent/luminex_xponent_reader.py b/src/allotropy/parsers/luminex_xponent/luminex_xponent_reader.py index b36e00b594..f498cebcd6 100644 --- a/src/allotropy/parsers/luminex_xponent/luminex_xponent_reader.py +++ b/src/allotropy/parsers/luminex_xponent/luminex_xponent_reader.py @@ -143,10 +143,13 @@ def build_section( if metric_token == "COUNT": # noqa: S105 analyte_cols = analyte_labels - # Convert each analyte column to numeric, coercing errors to NaN then filling with 0.0 + # Convert each analyte column to numeric, coercing errors to 0.0 + # Use try_float_or_none for locale support, then fill None with 0.0 converted = cast( pd.DataFrame, - out[analyte_cols].apply(pd.to_numeric, errors="coerce").fillna(0.0), + out[analyte_cols] + .apply(lambda col: col.apply(try_float_or_none)) + .fillna(0.0), ) out["Total Events"] = converted.sum(axis=1) else: diff --git a/src/allotropy/parsers/moldev_softmax_pro/softmax_pro_structure.py b/src/allotropy/parsers/moldev_softmax_pro/softmax_pro_structure.py index 3ca2e77723..5e40277be3 100644 --- a/src/allotropy/parsers/moldev_softmax_pro/softmax_pro_structure.py +++ b/src/allotropy/parsers/moldev_softmax_pro/softmax_pro_structure.py @@ -522,7 +522,9 @@ def create(reader: CsvReader, header: PlateHeader) -> PlateRawData: data = get_plate_dataframe(reader, header, columns=columns) # get temperature and elapsed time (kinetic) from the first column of the first row with value - first_row_idx = int(pd.to_numeric(data.first_valid_index())) + first_row_idx = int( + try_float(str(data.first_valid_index()), "first_valid_index") + ) temperature = try_non_nan_float_or_none(str(data.iloc[first_row_idx, 1])) elapsed_time = None if pd.notna(elapsed := data.iloc[first_row_idx, 0]): @@ -592,7 +594,9 @@ def create(reader: CsvReader, header: PlateHeader) -> SpectrumPlateRawData: wavelength_data: list[PlateWavelengthData] = [] for wavelength in header.wavelengths: data = get_plate_dataframe(reader, header, columns=columns) - first_row_idx = int(pd.to_numeric(data.first_valid_index())) + first_row_idx = int( + try_float(str(data.first_valid_index()), "first_valid_index") + ) temperature = try_non_nan_float_or_none(str(data.iloc[first_row_idx, 1])) wavelength_data.append( PlateWavelengthData.create( diff --git a/src/allotropy/parsers/roche_cedex_bioht/roche_cedex_bioht_reader.py b/src/allotropy/parsers/roche_cedex_bioht/roche_cedex_bioht_reader.py index 9f1819cba1..20839ca384 100644 --- a/src/allotropy/parsers/roche_cedex_bioht/roche_cedex_bioht_reader.py +++ b/src/allotropy/parsers/roche_cedex_bioht/roche_cedex_bioht_reader.py @@ -15,11 +15,14 @@ SAMPLE_ROLE_TYPES, ) from allotropy.parsers.utils.pandas import read_csv, SeriesData +from allotropy.parsers.utils.values import try_float_or_none from allotropy.types import IOType def to_num(data: pd.Series[Any]) -> pd.Series[Any]: - return pd.to_numeric(data, errors="coerce").replace(np.nan, None) + # try_float_or_none returns None for invalid values, but pandas converts None to np.nan + # So we need to explicitly replace np.nan with None, just like the original pd.to_numeric did + return data.apply(try_float_or_none).replace(np.nan, None) class RocheCedexBiohtReader: diff --git a/src/allotropy/parsers/thermo_fisher_genesys30/thermo_fisher_genesys30_structure.py b/src/allotropy/parsers/thermo_fisher_genesys30/thermo_fisher_genesys30_structure.py index 22cfc41005..aac9afe3de 100644 --- a/src/allotropy/parsers/thermo_fisher_genesys30/thermo_fisher_genesys30_structure.py +++ b/src/allotropy/parsers/thermo_fisher_genesys30/thermo_fisher_genesys30_structure.py @@ -22,7 +22,7 @@ from allotropy.exceptions import AllotropeConversionError from allotropy.parsers.constants import NOT_APPLICABLE from allotropy.parsers.thermo_fisher_genesys30 import constants -from allotropy.parsers.utils.pandas import SeriesData +from allotropy.parsers.utils.pandas import series_to_float_list, SeriesData from allotropy.parsers.utils.uuids import random_uuid_str @@ -72,7 +72,7 @@ def create_measurement_groups( ) ], dimensions=[data["wavelength(nm)"].tolist()], - measures=[data["ABS"].astype(float).tolist()], + measures=[series_to_float_list(data["ABS"], "absorbance")], ), device_control_custom_info={ "operating minimum": { diff --git a/src/allotropy/parsers/thermo_skanit/thermo_skanit_structure.py b/src/allotropy/parsers/thermo_skanit/thermo_skanit_structure.py index 01f6dbfc90..5cc92b6d95 100644 --- a/src/allotropy/parsers/thermo_skanit/thermo_skanit_structure.py +++ b/src/allotropy/parsers/thermo_skanit/thermo_skanit_structure.py @@ -26,7 +26,7 @@ ) from allotropy.parsers.utils.pandas import df_to_series_data, parse_header_row from allotropy.parsers.utils.uuids import random_uuid_str -from allotropy.parsers.utils.values import try_float_or_none +from allotropy.parsers.utils.values import try_float, try_float_or_none MEASUREMENT_TYPES = { MeasurementType.ULTRAVIOLET_ABSORBANCE: "Absorbance", @@ -402,7 +402,10 @@ def _set_headers(df: pd.DataFrame) -> pd.DataFrame: ) df = df[df.columns[valid_columns]] # Cast row numbers to int (float first to handle decimals, e.g. 1.0) - df.columns = df.columns.astype(float).astype(int) + # Use try_float for locale support + df.columns = pd.Index( # type: ignore[assignment] + [int(try_float(str(col), "column_name")) for col in df.columns] + ) return df @staticmethod @@ -442,7 +445,9 @@ def identify_data_and_sample_dfs( ] wavelength_data[current_wavelength] = data_df - current_wavelength = float(wavelength_match.group(1)) + current_wavelength = try_float( + wavelength_match.group(1), "wavelength" + ) current_data_section = [] reading_data = False reading_samples = False diff --git a/src/allotropy/parsers/utils/locale_context.py b/src/allotropy/parsers/utils/locale_context.py new file mode 100644 index 0000000000..1a8f20114e --- /dev/null +++ b/src/allotropy/parsers/utils/locale_context.py @@ -0,0 +1,40 @@ +"""Context management for locale-aware number parsing. + +This module provides utilities to set and manage the current locale context +for number parsing throughout the parsing pipeline. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager + +from allotropy.parsers.utils.values import _current_locale + + +@contextmanager +def set_locale_context(locale: str | None) -> Iterator[None]: + """Context manager to set the locale for number parsing. + + Args: + locale: The locale string (e.g., "en_US", "de_DE", "fr_FR") or None for default + + Example: + >>> with set_locale_context("de_DE"): + ... # All try_float() calls within this context will use German locale + ... value = try_float("1.234,56", "value") # Parses as 1234.56 + """ + token = _current_locale.set(locale) + try: + yield + finally: + _current_locale.reset(token) + + +def get_current_locale() -> str | None: + """Get the currently set locale for number parsing. + + Returns: + The current locale string or None if not set + """ + return _current_locale.get() diff --git a/src/allotropy/parsers/utils/locale_number_parser.py b/src/allotropy/parsers/utils/locale_number_parser.py new file mode 100644 index 0000000000..b6166488f8 --- /dev/null +++ b/src/allotropy/parsers/utils/locale_number_parser.py @@ -0,0 +1,282 @@ +"""Locale-aware number parsing utilities. + +This module provides locale-aware number parsing based on Babel's CLDR data. +It allows parsing numbers with locale-specific decimal and grouping separators. + +Example: + >>> parse_number_with_locale("1.234,56", "de_DE") + "1234.56" + >>> parse_number_with_locale("1,234.56", "en_US") + "1234.56" +""" + +from __future__ import annotations + +from dataclasses import dataclass +from decimal import Decimal +import logging +import math +import re + +from babel.core import Locale, UnknownLocaleError +from babel.numbers import ( + get_decimal_symbol, + get_group_symbol, + NumberFormatError, + parse_decimal, +) + +from allotropy.exceptions import AllotropeConversionError + +logger = logging.getLogger(__name__) + +# Default locale for number parsing (US English) +DEFAULT_LOCALE = "en_US" + + +@dataclass(frozen=True) +class NumberFormat: + """Format used to parse numbers for a locale. + + Attributes: + group_delimiter: The symbol used as the group separator (ex. "," in 1,234.56) + decimal_delimiter: The symbol used as the decimal point (ex. "." in 1,234.56) + """ + + group_delimiter: str + decimal_delimiter: str + + +# Alternative group delimiters that some locales support +GROUP_DELIMITER_ALTERNATIVES = { + "\xa0": " ", # Non-breaking space → regular space + "\u202f": " ", # Narrow no-break space → regular space + "'": "'", # Different apostrophe encodings +} + + +class _NumberFormatBuilder: + """Builder for locale-specific number formats.""" + + locale_str: str + locale: Locale + + def __init__(self, locale: str) -> None: + """Initialize the builder. + + Args: + locale: The locale string (e.g., "en_US", "de_DE", "fr_FR") + + Raises: + AllotropeConversionError: If locale is invalid + """ + locale = locale.replace("-", "_") + try: + self.locale = Locale.parse(locale) + except UnknownLocaleError: + # Try with just the language code (e.g., "es" from "es_LA") + locale = locale[: locale.index("_")] if "_" in locale else locale + try: + self.locale = Locale.parse(locale) + except UnknownLocaleError as e: + msg = f"Unsupported locale: {locale}" + raise AllotropeConversionError(msg) from e + self.locale_str = locale + + def build(self) -> NumberFormat: + """Build a NumberFormat for this locale.""" + return NumberFormat( + group_delimiter=get_group_symbol(self.locale_str), + decimal_delimiter=get_decimal_symbol(self.locale_str), + ) + + +# Cache for number formats to avoid repeated Babel lookups +_NUMBER_FORMAT_CACHE: dict[str, NumberFormat] = {} + + +def get_number_format(locale: str = DEFAULT_LOCALE) -> NumberFormat: + """Get the number format for a locale. + + Args: + locale: The locale string (e.g., "en_US", "de_DE") + + Returns: + NumberFormat for the locale + """ + if locale not in _NUMBER_FORMAT_CACHE: + try: + builder = _NumberFormatBuilder(locale) + _NUMBER_FORMAT_CACHE[locale] = builder.build() + except AllotropeConversionError: + logger.warning( + f"Unsupported locale {locale}, falling back to {DEFAULT_LOCALE}" + ) + if DEFAULT_LOCALE not in _NUMBER_FORMAT_CACHE: + _NUMBER_FORMAT_CACHE[DEFAULT_LOCALE] = _NumberFormatBuilder( + DEFAULT_LOCALE + ).build() + return _NUMBER_FORMAT_CACHE[DEFAULT_LOCALE] + return _NUMBER_FORMAT_CACHE[locale] + + +def parse_number_with_locale(number_str: str, locale: str = DEFAULT_LOCALE) -> str: + """Parse a locale-formatted number string and return a standard format. + + Converts a number with locale-specific separators to a standard format + (period decimal separator, no group separators). + + Args: + number_str: The number string to parse (e.g., "1.234,56" in de_DE) + locale: The locale to use for parsing (e.g., "de_DE") + + Returns: + The number in standard format (e.g., "1234.56") + + Raises: + AllotropeConversionError: If the number cannot be parsed + """ + if not number_str or number_str.strip() == "": + msg = "Cannot parse empty string as number" + raise AllotropeConversionError(msg) + + number_str = number_str.strip() + + # Handle special values (NaN, infinity) + special_value = _is_nan_or_inf(number_str) + if special_value is not False: + # Normalize to Python's standard string representation + return str(special_value).lower() + + try: + number_format = get_number_format(locale) + sanitized = _sanitize_number_to_parse(number_str, number_format) + + # Parse with Babel + if "e" in sanitized.lower(): + # Scientific notation + result = _parse_scientific_notation(sanitized, locale, number_format) + else: + # Standard notation + result = _parse_standard_notation(sanitized, locale, number_format) + + # Convert Decimal to string, preserving precision + return str(result) + except (NumberFormatError, ValueError, AllotropeConversionError) as e: + msg = f"Cannot parse '{number_str}' as number for locale {locale}: {e}" + raise AllotropeConversionError(msg) from e + + +def _sanitize_number_to_parse(number_str: str, number_format: NumberFormat) -> str: + """Sanitize number string for parsing.""" + # Convert alternative group delimiters + sanitized = _convert_group_delimiters(number_str, number_format) + + # Remove underscores (Python numeric literal support) + if "__" in sanitized: + msg = f"Invalid number format: {number_str}" + raise AllotropeConversionError(msg) + sanitized = sanitized.replace("_", "") + + return sanitized.lower() + + +def _convert_group_delimiters(number_str: str, number_format: NumberFormat) -> str: + """Convert alternative group delimiters to standard ones.""" + if number_format.group_delimiter in GROUP_DELIMITER_ALTERNATIVES: + alternative = GROUP_DELIMITER_ALTERNATIVES[number_format.group_delimiter] + return number_str.replace(alternative, number_format.group_delimiter) + return number_str + + +def _is_nan_or_inf(number_str: str) -> bool | float: + """Check if string represents NaN or infinity. + + Returns: + False if not NaN/inf, otherwise returns the normalized float value + """ + try: + value = float(number_str) + if math.isnan(value) or math.isinf(value): + return value + return False + except ValueError: + return False + + +def _parse_scientific_notation( + number_str: str, locale: str, number_format: NumberFormat +) -> Decimal: + """Parse number in scientific notation.""" + before_e, after_e = number_str.split("e", 1) + + # Validate exponent + if not re.match(r"^[+-]?\d+$", after_e): + msg = f"Invalid scientific notation: {number_str}" + raise AllotropeConversionError(msg) + + # Parse mantissa + parse_decimal(string=before_e, locale=locale, strict=True) + + # Remove group delimiter for Babel (it doesn't support groups in scientific notation) + sanitized = before_e.replace(number_format.group_delimiter, "") + "e" + after_e + return parse_decimal(string=sanitized, locale=locale, strict=True) + + +def _parse_standard_notation( + number_str: str, locale: str, number_format: NumberFormat +) -> Decimal: + """Parse number in standard notation.""" + # Extract parts + maybe_sign, before_decimal, after_decimal = _try_match_number( + number_str, number_format, locale + ) + + # Remove leading/trailing zeros + before_decimal = before_decimal.lstrip("0") if before_decimal else "" + if after_decimal: + after_decimal = after_decimal.rstrip("0").rstrip( + number_format.decimal_delimiter + ) + + # Reconstruct number + number = before_decimal + after_decimal + + # Add leading zero if starts with decimal + if number and number[0] == number_format.decimal_delimiter: + number = "0" + number + + # Handle empty string (was all zeros) + if not number: + number = "0" + + # Add sign + if maybe_sign == "-": + number = "-" + number + + return parse_decimal(string=number, locale=locale, strict=True) + + +def _try_match_number( + number_str: str, number_format: NumberFormat, locale: str +) -> tuple[str, str, str]: + """Match number string and extract parts.""" + # Match optional sign + sign_group = "([-+]?)" + + # Match digits and group delimiters before decimal + before_decimal_group = f"([\\d{number_format.group_delimiter}]*)?" + + # Match decimal and digits after + after_decimal_group = ( + f"([{number_format.decimal_delimiter}{number_format.group_delimiter}\\d]*)?" + ) + + pattern = f"^{sign_group}{before_decimal_group}{after_decimal_group}$" + match = re.match(pattern, number_str) + + if not match: + msg = f"Invalid number format for locale {locale}: {number_str}" + raise AllotropeConversionError(msg) + + return match.group(1), match.group(2) or "", match.group(3) or "" diff --git a/src/allotropy/parsers/utils/pandas.py b/src/allotropy/parsers/utils/pandas.py index 399d90e991..371889520e 100644 --- a/src/allotropy/parsers/utils/pandas.py +++ b/src/allotropy/parsers/utils/pandas.py @@ -23,6 +23,7 @@ assert_is_type, assert_not_none, str_to_bool, + try_float, try_float_or_none, ) from allotropy.types import IOType @@ -40,6 +41,26 @@ def run_with_data(series: pd.Series[str]) -> MapType: return list(data_frame.apply(run_with_data, axis="columns")) # type: ignore[call-overload] +def series_to_float_list(series: pd.Series[Any], value_name: str) -> list[float]: + """Convert pandas Series to list of floats using locale-aware parsing. + + This is a convenience helper for the common pattern of converting a pandas Series + to a list of floats. It uses try_float for locale-aware number parsing. + + Args: + series: The pandas Series to convert + value_name: A descriptive name for the values (used in error messages) + + Returns: + A list of floats parsed from the series values + + Example: + >>> series_to_float_list(df["Temperature"], "temperature") + [25.5, 26.0, 27.3] + """ + return [try_float(str(v), value_name) for v in series] + + def rm_df_columns(data: pd.DataFrame, pattern: str) -> pd.DataFrame: return data.drop( columns=[column for column in data.columns if re.match(pattern, column)] diff --git a/src/allotropy/parsers/utils/values.py b/src/allotropy/parsers/utils/values.py index 4552825e03..8dbac749aa 100644 --- a/src/allotropy/parsers/utils/values.py +++ b/src/allotropy/parsers/utils/values.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Callable +from contextvars import ContextVar import math import re from typing import Any, TypeVar @@ -13,10 +14,14 @@ ) from allotropy.exceptions import AllotropeConversionError, AllotropyParserError from allotropy.parsers.constants import NEGATIVE_ZERO +from allotropy.parsers.utils.locale_number_parser import parse_number_with_locale from allotropy.parsers.utils.units import get_quantity_class PrimitiveValue = str | int | float +# Context variable to store the current locale for number parsing +_current_locale: ContextVar[str | None] = ContextVar("current_locale", default=None) + def str_to_bool(value: str) -> bool: return value.lower() in ("yes", "y", "true", "t", "1") @@ -49,6 +54,17 @@ def try_int_or_none(value: str | None) -> int | None: def _try_float(value: str | float | None) -> float: if isinstance(value, float): return value + + # Check if locale-aware parsing is enabled + locale = _current_locale.get() + if locale: + try: + return float(parse_number_with_locale(str(value), locale)) + except AllotropeConversionError: + # Fall back to default parsing if locale parsing fails + pass + + # Default parsing: treat comma as decimal separator (European format) # NOTE: this will convert a string with commas for thousands into a decimal, potentially introducing # an unexpected error, e.g. one thousand represented as 1,000 would get converted to 1.0 # However, numbers are not usually represented like this in scientific output (we have no example of it) diff --git a/src/allotropy/to_allotrope.py b/src/allotropy/to_allotrope.py index 374a70bf54..b2f5d4b9c8 100644 --- a/src/allotropy/to_allotrope.py +++ b/src/allotropy/to_allotrope.py @@ -7,6 +7,7 @@ from allotropy.exceptions import AllotropeConversionError from allotropy.named_file_contents import NamedFileContents from allotropy.parser_factory import Vendor +from allotropy.parsers.utils.locale_context import set_locale_context from allotropy.types import IOType VendorType = Vendor | str @@ -18,9 +19,10 @@ def allotrope_from_io( vendor_type: VendorType, default_timezone: tzinfo | None = None, encoding: str | None = None, + locale: str | None = None, ) -> dict[str, Any]: model = allotrope_model_from_io( - contents, filepath, vendor_type, default_timezone, encoding + contents, filepath, vendor_type, default_timezone, encoding, locale ) return serialize_and_validate_allotrope(model) @@ -31,6 +33,7 @@ def allotrope_model_from_io( vendor_type: VendorType, default_timezone: tzinfo | None = None, encoding: str | None = None, + locale: str | None = None, ) -> Any: try: vendor = Vendor(vendor_type) @@ -42,7 +45,13 @@ def allotrope_model_from_io( msg = f"Unsupported file extension '{named_file_contents.extension}' for parser '{vendor.display_name}', expected one of '{vendor.supported_extensions}'." raise AllotropeConversionError(msg) parser = vendor.get_parser(default_timezone=default_timezone) - return parser.to_allotrope(named_file_contents) + + # Set locale context for parsing + if locale: + with set_locale_context(locale): + return parser.to_allotrope(named_file_contents) + else: + return parser.to_allotrope(named_file_contents) def allotrope_from_file( @@ -50,8 +59,11 @@ def allotrope_from_file( vendor_type: VendorType, default_timezone: tzinfo | None = None, encoding: str | None = None, + locale: str | None = None, ) -> dict[str, Any]: - model = allotrope_model_from_file(filepath, vendor_type, default_timezone, encoding) + model = allotrope_model_from_file( + filepath, vendor_type, default_timezone, encoding, locale + ) return serialize_and_validate_allotrope(model) @@ -60,6 +72,7 @@ def allotrope_model_from_file( vendor_type: VendorType, default_timezone: tzinfo | None = None, encoding: str | None = None, + locale: str | None = None, ) -> Any: try: if not os.path.isdir(filepath): @@ -70,6 +83,7 @@ def allotrope_model_from_file( vendor_type, default_timezone=default_timezone, encoding=encoding, + locale=locale, ) else: return allotrope_model_from_io( @@ -78,6 +92,7 @@ def allotrope_model_from_file( vendor_type, default_timezone=default_timezone, encoding=encoding, + locale=locale, ) except FileNotFoundError as e: diff --git a/tests/parsers/utils/locale_context_test.py b/tests/parsers/utils/locale_context_test.py new file mode 100644 index 0000000000..d70393dff9 --- /dev/null +++ b/tests/parsers/utils/locale_context_test.py @@ -0,0 +1,73 @@ +"""Tests for locale context management.""" + +import pytest + +from allotropy.parsers.utils.locale_context import ( + get_current_locale, + set_locale_context, +) +from allotropy.parsers.utils.values import try_float + + +class TestLocaleContext: + def test_default_locale_is_none(self) -> None: + assert get_current_locale() is None + + def test_set_locale_context(self) -> None: + assert get_current_locale() is None + + with set_locale_context("de_DE"): + assert get_current_locale() == "de_DE" + + assert get_current_locale() is None + + def test_nested_contexts(self) -> None: + with set_locale_context("en_US"): + assert get_current_locale() == "en_US" + + with set_locale_context("de_DE"): + assert get_current_locale() == "de_DE" + + assert get_current_locale() == "en_US" + + assert get_current_locale() is None + + def test_context_exception_handling(self) -> None: + """Ensure locale is reset even if exception occurs.""" + assert get_current_locale() is None + + try: + with set_locale_context("de_DE"): + assert get_current_locale() == "de_DE" + msg = "test error" + raise ValueError(msg) + except ValueError: + pass + + # Locale should be reset despite exception + assert get_current_locale() is None + + +class TestTryFloatWithLocale: + def test_try_float_with_us_locale(self) -> None: + with set_locale_context("en_US"): + result = try_float("1,234.56", "test") + assert result == pytest.approx(1234.56) + + def test_try_float_with_de_locale(self) -> None: + with set_locale_context("de_DE"): + result = try_float("1.234,56", "test") + assert result == pytest.approx(1234.56) + + def test_try_float_without_locale_uses_default(self) -> None: + # Without locale, comma is treated as decimal separator + result = try_float("1,5", "test") + assert result == pytest.approx(1.5) + + def test_try_float_fallback_on_parse_error(self) -> None: + # If locale parsing fails, should fall back to default + with set_locale_context("en_US"): + # This is invalid for en_US but valid for default (comma as decimal) + result = try_float("1,5", "test") + # Should fall back to default parsing (comma as decimal) + assert result == pytest.approx(1.5) diff --git a/tests/parsers/utils/locale_number_parser_test.py b/tests/parsers/utils/locale_number_parser_test.py new file mode 100644 index 0000000000..2cb5df3c55 --- /dev/null +++ b/tests/parsers/utils/locale_number_parser_test.py @@ -0,0 +1,132 @@ +"""Tests for locale-aware number parsing.""" + +import pytest + +from allotropy.exceptions import AllotropeConversionError +from allotropy.parsers.utils.locale_number_parser import ( + get_number_format, + parse_number_with_locale, +) + + +class TestGetNumberFormat: + def test_en_us_format(self) -> None: + fmt = get_number_format("en_US") + assert fmt.group_delimiter == "," + assert fmt.decimal_delimiter == "." + + def test_de_de_format(self) -> None: + fmt = get_number_format("de_DE") + assert fmt.group_delimiter == "." + assert fmt.decimal_delimiter == "," + + def test_fr_fr_format(self) -> None: + fmt = get_number_format("fr_FR") + assert fmt.decimal_delimiter == "," + # French uses narrow no-break space for grouping + assert fmt.group_delimiter == "\u202f" + + def test_invalid_locale_falls_back_to_default(self) -> None: + fmt = get_number_format("invalid_LOCALE") + # Should fall back to en_US + assert fmt.group_delimiter == "," + assert fmt.decimal_delimiter == "." + + +class TestParseNumberWithLocale: + # US English tests + def test_parse_us_simple_integer(self) -> None: + assert parse_number_with_locale("123", "en_US") == "123" + + def test_parse_us_simple_decimal(self) -> None: + assert parse_number_with_locale("123.45", "en_US") == "123.45" + + def test_parse_us_with_thousands(self) -> None: + assert parse_number_with_locale("1,234.56", "en_US") == "1234.56" + + def test_parse_us_millions(self) -> None: + assert parse_number_with_locale("1,234,567.89", "en_US") == "1234567.89" + + # German tests + def test_parse_de_simple_integer(self) -> None: + assert parse_number_with_locale("123", "de_DE") == "123" + + def test_parse_de_simple_decimal(self) -> None: + assert parse_number_with_locale("123,45", "de_DE") == "123.45" + + def test_parse_de_with_thousands(self) -> None: + assert parse_number_with_locale("1.234,56", "de_DE") == "1234.56" + + def test_parse_de_millions(self) -> None: + assert parse_number_with_locale("1.234.567,89", "de_DE") == "1234567.89" + + # French tests + def test_parse_fr_simple_integer(self) -> None: + assert parse_number_with_locale("123", "fr_FR") == "123" + + def test_parse_fr_simple_decimal(self) -> None: + assert parse_number_with_locale("123,45", "fr_FR") == "123.45" + + # Scientific notation + def test_parse_us_scientific_notation(self) -> None: + result = parse_number_with_locale("1.23e5", "en_US") + assert float(result) == pytest.approx(123000.0) + + def test_parse_de_scientific_notation(self) -> None: + result = parse_number_with_locale("1,23e5", "de_DE") + assert float(result) == pytest.approx(123000.0) + + def test_parse_scientific_with_negative_exponent(self) -> None: + result = parse_number_with_locale("1.23e-5", "en_US") + assert float(result) == pytest.approx(0.0000123) + + # Special values + def test_parse_nan(self) -> None: + assert parse_number_with_locale("nan", "en_US") == "nan" + assert parse_number_with_locale("NaN", "en_US") == "nan" + + def test_parse_infinity(self) -> None: + assert parse_number_with_locale("inf", "en_US") == "inf" + assert parse_number_with_locale("Infinity", "en_US") == "inf" + + # Negative numbers + def test_parse_negative_us(self) -> None: + assert parse_number_with_locale("-123.45", "en_US") == "-123.45" + + def test_parse_negative_de(self) -> None: + assert parse_number_with_locale("-123,45", "de_DE") == "-123.45" + + # Edge cases + def test_parse_leading_zeros(self) -> None: + assert parse_number_with_locale("00123.45", "en_US") == "123.45" + + def test_parse_trailing_zeros(self) -> None: + assert parse_number_with_locale("123.4500", "en_US") == "123.45" + + def test_parse_decimal_only(self) -> None: + assert parse_number_with_locale(".5", "en_US") == "0.5" + + def test_parse_integer_ending_in_decimal(self) -> None: + # Some locales allow trailing decimal separator + result = parse_number_with_locale("123.", "en_US") + assert result in ("123", "123.0") + + # Error cases + def test_parse_empty_string_raises(self) -> None: + with pytest.raises(AllotropeConversionError, match="empty string"): + parse_number_with_locale("", "en_US") + + def test_parse_invalid_format_raises(self) -> None: + with pytest.raises(AllotropeConversionError): + parse_number_with_locale("abc", "en_US") + + def test_parse_wrong_locale_format_raises(self) -> None: + # German number with US locale should fail + with pytest.raises(AllotropeConversionError): + parse_number_with_locale("1.234,56", "en_US") + + # Alternative group delimiters + def test_parse_fr_with_regular_space(self) -> None: + # French often uses space instead of narrow no-break space + result = parse_number_with_locale("1 234,56", "fr_FR") + assert result == "1234.56" diff --git a/tests/to_allotrope_locale_test.py b/tests/to_allotrope_locale_test.py new file mode 100644 index 0000000000..cdd8673ceb --- /dev/null +++ b/tests/to_allotrope_locale_test.py @@ -0,0 +1,47 @@ +"""Integration tests for locale support in top-level API.""" + +from io import BytesIO + +import pytest + +from allotropy.exceptions import AllotropeConversionError +from allotropy.to_allotrope import allotrope_from_io + + +def test_locale_parameter_is_optional() -> None: + """Verify that locale parameter is optional and defaults to None.""" + # This should work without locale parameter + # Using a simple CSV that doesn't require locale parsing + csv_content = b"A,B\n1,2\n3,4" + # We expect this to fail because we don't have a real parser for raw CSV, + # but it should fail at parser creation, not at locale handling + with pytest.raises(AllotropeConversionError): + allotrope_from_io(BytesIO(csv_content), "test.csv", "UNKNOWN_VENDOR") + + +def test_locale_parameter_accepts_string() -> None: + """Verify that locale parameter accepts locale strings.""" + csv_content = b"A,B\n1,2\n3,4" + # We expect this to fail because we don't have a real parser for raw CSV, + # but it should fail at parser creation, not at locale handling + with pytest.raises(AllotropeConversionError): + allotrope_from_io( + BytesIO(csv_content), + "test.csv", + "UNKNOWN_VENDOR", + locale="de_DE", + ) + + +def test_locale_parameter_accepts_none() -> None: + """Verify that locale parameter accepts None explicitly.""" + csv_content = b"A,B\n1,2\n3,4" + # We expect this to fail because we don't have a real parser for raw CSV, + # but it should fail at parser creation, not at locale handling + with pytest.raises(AllotropeConversionError): + allotrope_from_io( + BytesIO(csv_content), + "test.csv", + "UNKNOWN_VENDOR", + locale=None, + )