Skip to content

Commit e963fcf

Browse files
fix(parser): coerce non-finite usage tokens and tidy review nits
Guard _safe_int against NaN/Infinity (json.loads accepts these literals; int(nan)/int(inf) raise), broaden fuzz floats to cover them, and add an explicit non-finite regression test. Revert entry_counts to truthiness so empty-string types stay skipped, wrap service_tier in the message payload, and drop the no-op [tool.hypothesis] block from pyproject.toml.
1 parent 6e3ae48 commit e963fcf

3 files changed

Lines changed: 32 additions & 11 deletions

File tree

pyproject.toml

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,6 @@ select = ["E", "F", "W", "I"]
2727
[tool.ruff.lint.isort]
2828
combine-as-imports = true
2929

30-
[tool.hypothesis]
31-
max_examples = 200
32-
3330
[tool.ruff.lint.per-file-ignores]
3431
# CLI bootstrap: sys.path must be set before local imports.
3532
"scripts/export.py" = ["E402"]

tests/test_parser_fuzz.py

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,8 @@ def _write_jsonl(path: os.PathLike[str], lines: list[str]) -> str:
5353
st.none(),
5454
st.booleans(),
5555
st.integers(),
56-
st.floats(allow_nan=False, allow_infinity=False),
56+
# Allow NaN/Infinity: json.loads accepts these literals, so the parser must too.
57+
st.floats(allow_nan=True, allow_infinity=True),
5758
st.text(max_size=200),
5859
)
5960

@@ -263,3 +264,21 @@ def test_non_numeric_usage_tokens_do_not_crash(tmp_path: Path) -> None:
263264
assert session["metadata"]["total_input_tokens"] == 0
264265
assert session["metadata"]["total_output_tokens"] == 0
265266
assert session["metadata"]["total_ephemeral_5m_tokens"] == 0
267+
268+
269+
def test_non_finite_usage_tokens_do_not_crash(tmp_path: Path) -> None:
270+
"""json.loads accepts NaN/Infinity literals; int(nan)/int(inf) raise, so the
271+
parser must coerce them to 0 rather than propagate ValueError/OverflowError."""
272+
# Raw literals (not valid via json.dumps of finite floats) — written directly.
273+
line = (
274+
'{"type": "assistant", "message": {"usage": '
275+
'{"input_tokens": NaN, "output_tokens": Infinity, '
276+
'"cache_read_input_tokens": -Infinity, '
277+
'"cache_creation": {"ephemeral_5m_input_tokens": NaN}}}}'
278+
)
279+
path = _write_jsonl(tmp_path / "nonfinite.jsonl", [line])
280+
session = parse_session(path)
281+
assert session["metadata"]["total_input_tokens"] == 0
282+
assert session["metadata"]["total_output_tokens"] == 0
283+
assert session["metadata"]["total_cache_read_tokens"] == 0
284+
assert session["metadata"]["total_ephemeral_5m_tokens"] == 0

utils/jsonl_parser.py

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
actually work with -- messages, tool calls, token counts, file activity, etc."""
33

44
import json
5+
import math
56
import os
67
from datetime import datetime
78
from typing import Any
@@ -41,12 +42,15 @@
4142

4243

4344
def _safe_int(val: Any) -> int:
44-
"""Coerce a value to int for token accounting; non-numeric input becomes 0
45-
so fuzzed/malformed usage fields never raise during arithmetic."""
45+
"""Coerce a value to int for token accounting; non-numeric or non-finite
46+
input becomes 0 so fuzzed/malformed usage fields never raise during
47+
arithmetic. json.loads accepts NaN/Infinity literals, so guard against them."""
4648
if isinstance(val, bool):
4749
return 0
48-
if isinstance(val, (int, float)):
49-
return int(val)
50+
if isinstance(val, int):
51+
return val
52+
if isinstance(val, float):
53+
return int(val) if math.isfinite(val) else 0
5054
return 0
5155

5256

@@ -122,8 +126,9 @@ def parse_session(filepath: str) -> SessionDict:
122126
metadata["first_timestamp"] = ts
123127
metadata["last_timestamp"] = ts
124128

125-
# Count entry types (upstream may send non-str discriminants)
126-
if entry_type is not None:
129+
# Count entry types (upstream may send non-str/unhashable discriminants;
130+
# coerce to str. Falsy types like "" are skipped, matching prior behavior).
131+
if entry_type:
127132
type_key = entry_type if isinstance(entry_type, str) else str(entry_type)
128133
metadata["entry_counts"][type_key] = metadata["entry_counts"].get(type_key, 0) + 1
129134

@@ -306,7 +311,7 @@ def _process_assistant(
306311
"output_tokens": _safe_int(usage.get("output_tokens")),
307312
"cache_read": _safe_int(usage.get("cache_read_input_tokens")),
308313
"cache_creation": _safe_int(usage.get("cache_creation_input_tokens")),
309-
"service_tier": usage.get("service_tier"),
314+
"service_tier": tier if isinstance(tier, str) else None,
310315
},
311316
}
312317
)

0 commit comments

Comments
 (0)