fix: enforce the documented strict JSON-compatible contract for #3486 (#3657)

Author: siddiksawani

src/agents/util/_custom_data.py

@@ -28,7 +28,7 @@ def normalize_custom_data(value: Mapping[str, Any] | None) -> dict[str, Any] | N
 
     copied = copy.deepcopy(dict(value))
     try:
-        return cast(dict[str, Any], json.loads(json.dumps(copied)))
+        return cast(dict[str, Any], json.loads(json.dumps(copied, allow_nan=False)))
     except (TypeError, ValueError) as exc:
         raise UserError("custom_data_extractor must return JSON-compatible data.") from exc
 

tests/test_tool_custom_data.py

@@ -96,6 +96,25 @@ def get_data() -> str:
         await Runner.run(agent, input="user")
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("bad_value", [float("nan"), float("inf"), float("-inf")])
+async def test_function_tool_custom_data_rejects_non_finite_floats(
+    bad_value: float,
+) -> None:
+    @function_tool(custom_data_extractor=lambda _ctx: {"score": bad_value})
+    def get_data() -> str:
+        return "tool_result"
+
+    model = FakeModel()
+    model.add_multiple_turn_outputs(
+        [[get_text_message("call tool"), get_function_tool_call("get_data", "{}")]]
+    )
+    agent = Agent(name="test", model=model, tools=[get_data])
+
+    with pytest.raises(UserError, match="custom_data_extractor must return JSON-compatible data"):
+        await Runner.run(agent, input="user")
+
+
 @pytest.mark.asyncio
 async def test_mcp_custom_data_extractor_maps_result_meta_to_tool_output_item() -> None:
     def extract_custom_data(ctx: Any) -> dict[str, Any]: