Skip to content

Commit af67373

Browse files
feat: Add global locale support for number parsing
Add locale parameter to top-level API functions to support locale-specific number formats using Babel's CLDR data. - Add locale_number_parser module with Babel-based parsing - Add locale_context module for thread-safe locale management - Update try_float() to respect locale context - Add locale parameter to allotrope_from_io/file functions - Add babel >= 2.9.0 dependency - Add comprehensive tests for locale parsing and context This enables parsing numbers in different locales (e.g., German "1.234,56" vs US "1,234.56") by passing locale="de_DE" to the top-level API functions. Co-Authored-By: Claude Opus 4.1 <noreply@anthropic.com>
1 parent dd0ef64 commit af67373

9 files changed

Lines changed: 620 additions & 3 deletions

File tree

CHANGELOG.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,12 @@ All notable changes to this packages will be documented in this file.
55
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
66
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
77

8+
## [Unreleased]
9+
10+
### Added
11+
12+
- Global locale support for number parsing - Add `locale` parameter to top-level API functions to support locale-specific number formats (e.g., German "1.234,56" vs US "1,234.56")
13+
814
## [0.1.116] - 2026-04-01
915

1016
### Added

pyproject.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ classifiers = [
4747
"Typing :: Typed",
4848
]
4949
dependencies = [
50+
"babel >= 2.9.0",
5051
"cattrs >= 23.2.0",
5152
"chardet >= 5.2.0",
5253
"defusedxml >= 0.7.1",
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
"""Context management for locale-aware number parsing.
2+
3+
This module provides utilities to set and manage the current locale context
4+
for number parsing throughout the parsing pipeline.
5+
"""
6+
7+
from __future__ import annotations
8+
9+
from collections.abc import Iterator
10+
from contextlib import contextmanager
11+
12+
from allotropy.parsers.utils.values import _current_locale
13+
14+
15+
@contextmanager
16+
def set_locale_context(locale: str | None) -> Iterator[None]:
17+
"""Context manager to set the locale for number parsing.
18+
19+
Args:
20+
locale: The locale string (e.g., "en_US", "de_DE", "fr_FR") or None for default
21+
22+
Example:
23+
>>> with set_locale_context("de_DE"):
24+
... # All try_float() calls within this context will use German locale
25+
... value = try_float("1.234,56", "value") # Parses as 1234.56
26+
"""
27+
token = _current_locale.set(locale)
28+
try:
29+
yield
30+
finally:
31+
_current_locale.reset(token)
32+
33+
34+
def get_current_locale() -> str | None:
35+
"""Get the currently set locale for number parsing.
36+
37+
Returns:
38+
The current locale string or None if not set
39+
"""
40+
return _current_locale.get()
Lines changed: 282 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,282 @@
1+
"""Locale-aware number parsing utilities.
2+
3+
This module provides locale-aware number parsing based on Babel's CLDR data.
4+
It allows parsing numbers with locale-specific decimal and grouping separators.
5+
6+
Example:
7+
>>> parse_number_with_locale("1.234,56", "de_DE")
8+
"1234.56"
9+
>>> parse_number_with_locale("1,234.56", "en_US")
10+
"1234.56"
11+
"""
12+
13+
from __future__ import annotations
14+
15+
from dataclasses import dataclass
16+
from decimal import Decimal
17+
import logging
18+
import math
19+
import re
20+
21+
from babel.core import Locale, UnknownLocaleError
22+
from babel.numbers import (
23+
get_decimal_symbol,
24+
get_group_symbol,
25+
NumberFormatError,
26+
parse_decimal,
27+
)
28+
29+
from allotropy.exceptions import AllotropeConversionError
30+
31+
logger = logging.getLogger(__name__)
32+
33+
# Default locale for number parsing (US English)
34+
DEFAULT_LOCALE = "en_US"
35+
36+
37+
@dataclass(frozen=True)
38+
class NumberFormat:
39+
"""Format used to parse numbers for a locale.
40+
41+
Attributes:
42+
group_delimiter: The symbol used as the group separator (ex. "," in 1,234.56)
43+
decimal_delimiter: The symbol used as the decimal point (ex. "." in 1,234.56)
44+
"""
45+
46+
group_delimiter: str
47+
decimal_delimiter: str
48+
49+
50+
# Alternative group delimiters that some locales support
51+
GROUP_DELIMITER_ALTERNATIVES = {
52+
"\xa0": " ", # Non-breaking space → regular space
53+
"\u202f": " ", # Narrow no-break space → regular space
54+
"'": "'", # Different apostrophe encodings
55+
}
56+
57+
58+
class _NumberFormatBuilder:
59+
"""Builder for locale-specific number formats."""
60+
61+
locale_str: str
62+
locale: Locale
63+
64+
def __init__(self, locale: str) -> None:
65+
"""Initialize the builder.
66+
67+
Args:
68+
locale: The locale string (e.g., "en_US", "de_DE", "fr_FR")
69+
70+
Raises:
71+
AllotropeConversionError: If locale is invalid
72+
"""
73+
locale = locale.replace("-", "_")
74+
try:
75+
self.locale = Locale.parse(locale)
76+
except UnknownLocaleError:
77+
# Try with just the language code (e.g., "es" from "es_LA")
78+
locale = locale[: locale.index("_")] if "_" in locale else locale
79+
try:
80+
self.locale = Locale.parse(locale)
81+
except UnknownLocaleError as e:
82+
msg = f"Unsupported locale: {locale}"
83+
raise AllotropeConversionError(msg) from e
84+
self.locale_str = locale
85+
86+
def build(self) -> NumberFormat:
87+
"""Build a NumberFormat for this locale."""
88+
return NumberFormat(
89+
group_delimiter=get_group_symbol(self.locale_str),
90+
decimal_delimiter=get_decimal_symbol(self.locale_str),
91+
)
92+
93+
94+
# Cache for number formats to avoid repeated Babel lookups
95+
_NUMBER_FORMAT_CACHE: dict[str, NumberFormat] = {}
96+
97+
98+
def get_number_format(locale: str = DEFAULT_LOCALE) -> NumberFormat:
99+
"""Get the number format for a locale.
100+
101+
Args:
102+
locale: The locale string (e.g., "en_US", "de_DE")
103+
104+
Returns:
105+
NumberFormat for the locale
106+
"""
107+
if locale not in _NUMBER_FORMAT_CACHE:
108+
try:
109+
builder = _NumberFormatBuilder(locale)
110+
_NUMBER_FORMAT_CACHE[locale] = builder.build()
111+
except AllotropeConversionError:
112+
logger.warning(
113+
f"Unsupported locale {locale}, falling back to {DEFAULT_LOCALE}"
114+
)
115+
if DEFAULT_LOCALE not in _NUMBER_FORMAT_CACHE:
116+
_NUMBER_FORMAT_CACHE[DEFAULT_LOCALE] = _NumberFormatBuilder(
117+
DEFAULT_LOCALE
118+
).build()
119+
return _NUMBER_FORMAT_CACHE[DEFAULT_LOCALE]
120+
return _NUMBER_FORMAT_CACHE[locale]
121+
122+
123+
def parse_number_with_locale(number_str: str, locale: str = DEFAULT_LOCALE) -> str:
124+
"""Parse a locale-formatted number string and return a standard format.
125+
126+
Converts a number with locale-specific separators to a standard format
127+
(period decimal separator, no group separators).
128+
129+
Args:
130+
number_str: The number string to parse (e.g., "1.234,56" in de_DE)
131+
locale: The locale to use for parsing (e.g., "de_DE")
132+
133+
Returns:
134+
The number in standard format (e.g., "1234.56")
135+
136+
Raises:
137+
AllotropeConversionError: If the number cannot be parsed
138+
"""
139+
if not number_str or number_str.strip() == "":
140+
msg = "Cannot parse empty string as number"
141+
raise AllotropeConversionError(msg)
142+
143+
number_str = number_str.strip()
144+
145+
# Handle special values (NaN, infinity)
146+
special_value = _is_nan_or_inf(number_str)
147+
if special_value is not False:
148+
# Normalize to Python's standard string representation
149+
return str(special_value).lower()
150+
151+
try:
152+
number_format = get_number_format(locale)
153+
sanitized = _sanitize_number_to_parse(number_str, number_format)
154+
155+
# Parse with Babel
156+
if "e" in sanitized.lower():
157+
# Scientific notation
158+
result = _parse_scientific_notation(sanitized, locale, number_format)
159+
else:
160+
# Standard notation
161+
result = _parse_standard_notation(sanitized, locale, number_format)
162+
163+
# Convert Decimal to string, preserving precision
164+
return str(result)
165+
except (NumberFormatError, ValueError, AllotropeConversionError) as e:
166+
msg = f"Cannot parse '{number_str}' as number for locale {locale}: {e}"
167+
raise AllotropeConversionError(msg) from e
168+
169+
170+
def _sanitize_number_to_parse(number_str: str, number_format: NumberFormat) -> str:
171+
"""Sanitize number string for parsing."""
172+
# Convert alternative group delimiters
173+
sanitized = _convert_group_delimiters(number_str, number_format)
174+
175+
# Remove underscores (Python numeric literal support)
176+
if "__" in sanitized:
177+
msg = f"Invalid number format: {number_str}"
178+
raise AllotropeConversionError(msg)
179+
sanitized = sanitized.replace("_", "")
180+
181+
return sanitized.lower()
182+
183+
184+
def _convert_group_delimiters(number_str: str, number_format: NumberFormat) -> str:
185+
"""Convert alternative group delimiters to standard ones."""
186+
if number_format.group_delimiter in GROUP_DELIMITER_ALTERNATIVES:
187+
alternative = GROUP_DELIMITER_ALTERNATIVES[number_format.group_delimiter]
188+
return number_str.replace(alternative, number_format.group_delimiter)
189+
return number_str
190+
191+
192+
def _is_nan_or_inf(number_str: str) -> bool | float:
193+
"""Check if string represents NaN or infinity.
194+
195+
Returns:
196+
False if not NaN/inf, otherwise returns the normalized float value
197+
"""
198+
try:
199+
value = float(number_str)
200+
if math.isnan(value) or math.isinf(value):
201+
return value
202+
return False
203+
except ValueError:
204+
return False
205+
206+
207+
def _parse_scientific_notation(
208+
number_str: str, locale: str, number_format: NumberFormat
209+
) -> Decimal:
210+
"""Parse number in scientific notation."""
211+
before_e, after_e = number_str.split("e", 1)
212+
213+
# Validate exponent
214+
if not re.match(r"^[+-]?\d+$", after_e):
215+
msg = f"Invalid scientific notation: {number_str}"
216+
raise AllotropeConversionError(msg)
217+
218+
# Parse mantissa
219+
parse_decimal(string=before_e, locale=locale, strict=True)
220+
221+
# Remove group delimiter for Babel (it doesn't support groups in scientific notation)
222+
sanitized = before_e.replace(number_format.group_delimiter, "") + "e" + after_e
223+
return parse_decimal(string=sanitized, locale=locale, strict=True)
224+
225+
226+
def _parse_standard_notation(
227+
number_str: str, locale: str, number_format: NumberFormat
228+
) -> Decimal:
229+
"""Parse number in standard notation."""
230+
# Extract parts
231+
maybe_sign, before_decimal, after_decimal = _try_match_number(
232+
number_str, number_format, locale
233+
)
234+
235+
# Remove leading/trailing zeros
236+
before_decimal = before_decimal.lstrip("0") if before_decimal else ""
237+
if after_decimal:
238+
after_decimal = after_decimal.rstrip("0").rstrip(
239+
number_format.decimal_delimiter
240+
)
241+
242+
# Reconstruct number
243+
number = before_decimal + after_decimal
244+
245+
# Add leading zero if starts with decimal
246+
if number and number[0] == number_format.decimal_delimiter:
247+
number = "0" + number
248+
249+
# Handle empty string (was all zeros)
250+
if not number:
251+
number = "0"
252+
253+
# Add sign
254+
if maybe_sign == "-":
255+
number = "-" + number
256+
257+
return parse_decimal(string=number, locale=locale, strict=True)
258+
259+
260+
def _try_match_number(
261+
number_str: str, number_format: NumberFormat, locale: str
262+
) -> tuple[str, str, str]:
263+
"""Match number string and extract parts."""
264+
# Match optional sign
265+
sign_group = "([-+]?)"
266+
267+
# Match digits and group delimiters before decimal
268+
before_decimal_group = f"([\\d{number_format.group_delimiter}]*)?"
269+
270+
# Match decimal and digits after
271+
after_decimal_group = (
272+
f"([{number_format.decimal_delimiter}{number_format.group_delimiter}\\d]*)?"
273+
)
274+
275+
pattern = f"^{sign_group}{before_decimal_group}{after_decimal_group}$"
276+
match = re.match(pattern, number_str)
277+
278+
if not match:
279+
msg = f"Invalid number format for locale {locale}: {number_str}"
280+
raise AllotropeConversionError(msg)
281+
282+
return match.group(1), match.group(2) or "", match.group(3) or ""

src/allotropy/parsers/utils/values.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from __future__ import annotations
22

33
from collections.abc import Callable
4+
from contextvars import ContextVar
45
import math
56
import re
67
from typing import Any, TypeVar
@@ -17,6 +18,9 @@
1718

1819
PrimitiveValue = str | int | float
1920

21+
# Context variable to store the current locale for number parsing
22+
_current_locale: ContextVar[str | None] = ContextVar("current_locale", default=None)
23+
2024

2125
def str_to_bool(value: str) -> bool:
2226
return value.lower() in ("yes", "y", "true", "t", "1")
@@ -49,6 +53,21 @@ def try_int_or_none(value: str | None) -> int | None:
4953
def _try_float(value: str | float | None) -> float:
5054
if isinstance(value, float):
5155
return value
56+
57+
# Check if locale-aware parsing is enabled
58+
locale = _current_locale.get()
59+
if locale:
60+
from allotropy.parsers.utils.locale_number_parser import (
61+
parse_number_with_locale,
62+
)
63+
64+
try:
65+
return float(parse_number_with_locale(str(value), locale))
66+
except AllotropeConversionError:
67+
# Fall back to default parsing if locale parsing fails
68+
pass
69+
70+
# Default parsing: treat comma as decimal separator (European format)
5271
# NOTE: this will convert a string with commas for thousands into a decimal, potentially introducing
5372
# an unexpected error, e.g. one thousand represented as 1,000 would get converted to 1.0
5473
# However, numbers are not usually represented like this in scientific output (we have no example of it)

0 commit comments

Comments
 (0)