Skip to content

Commit 45aca33

Browse files
tolik0claude
andcommitted
fix: JsonItemsParser should yield floats, not Decimals
ijson.items() parses non-integer JSON numbers as decimal.Decimal by default, which the CDK cannot serialize downstream (orjson raises 'Decimal is not JSON serializable'). This broke any JsonItemsDecoder stream with decimal fields (e.g. Amazon Brand Analytics clickShare/conversionShare, Sales & Traffic, Vendor reports). Pass use_float=True so non-integer numbers are parsed as float, matching the json.loads/orjson behavior of the other JSON parsers. Adds a regression test. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 8ee6423 commit 45aca33

2 files changed

Lines changed: 19 additions & 1 deletion

File tree

airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,9 @@ def parse(self, data: BufferedIOBase) -> PARSER_OUTPUT_TYPE:
149149
# ijson auto-selects the best available backend (yajl2_c when present)
150150
# and reads from `data` lazily — it does not call `.read()` on the
151151
# whole stream up front.
152-
yield from ijson.items(data, f"{self.items_path}.item")
152+
# use_float=True yields floats for non-integer numbers instead of Decimal, matching
153+
# json.loads/orjson behavior so downstream JSON serialization doesn't choke on Decimal.
154+
yield from ijson.items(data, f"{self.items_path}.item", use_float=True)
153155

154156

155157
@dataclass

unit_tests/sources/declarative/decoders/test_composite_decoder.py

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -451,6 +451,22 @@ def test_json_items_parser_composes_with_gzip(requests_mock) -> None:
451451
assert list(decoder.decode(response)) == payload["dataByAsin"]
452452

453453

454+
def test_json_items_parser_yields_floats_not_decimals(requests_mock) -> None:
455+
"""Non-integer numbers must be parsed as float (not Decimal) so downstream JSON
456+
serialization (orjson) does not fail on Decimal values."""
457+
import orjson
458+
459+
payload = {"data": [{"ratio": 0.5, "rank": 3, "amount": 0.0000}]}
460+
requests_mock.register_uri("GET", "https://airbyte.io/", content=json.dumps(payload).encode("utf-8"))
461+
response = requests.get("https://airbyte.io/", stream=True)
462+
463+
records = list(CompositeRawDecoder(parser=JsonItemsParser(items_path="data")).decode(response))
464+
assert isinstance(records[0]["ratio"], float)
465+
assert isinstance(records[0]["rank"], int)
466+
# Must be serializable by orjson (which rejects Decimal).
467+
orjson.dumps(records[0])
468+
469+
454470
def test_json_items_parser_requires_items_path() -> None:
455471
parser = JsonItemsParser()
456472
with pytest.raises(ValueError, match="items_path"):

0 commit comments

Comments
 (0)