Skip to content

Commit 48b0f04

Browse files
committed
Strip nulls from remote eval prompt parameter defaults
## Summary Fix Python remote eval prompt parameter defaults that include explicit `None` / JSON `null` values from `PromptData.as_dict()`. `PromptData.as_dict()` currently preserves optional prompt fields like `tools`, `name`, `function_call`, and `tool_calls` as `None`. When these values are used as a `type: "prompt"` parameter default, the remote eval manifest contains explicit `null`s. The playground schema expects these fields to be omitted when unset, so the remote eval can fail to appear or fail to parse. This strips `None` values recursively only when serializing prompt parameter defaults, leaving the generic `PromptData.as_dict()` behavior unchanged. ## Test Plan - Add/updated Python parameter serialization test covering a `PromptData` prompt default with unset optional fields. - Verify serialized remote eval parameter container omits `tools`, `name`, `function_call`, and `tool_calls` when unset. - Manually verify `bt eval <repro>.py --dev` lists the remote eval in the playground.
1 parent fae4723 commit 48b0f04

2 files changed

Lines changed: 92 additions & 2 deletions

File tree

py/src/braintrust/parameters.py

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -215,14 +215,32 @@ def is_eval_parameter_schema(schema: Any) -> bool:
215215
return True
216216

217217

218+
def _strip_none_values(value: Any) -> Any:
219+
"""Recursively drop dict entries whose value is ``None``.
220+
221+
Prompt parameter defaults are serialized from Python prompt dataclasses,
222+
which preserve explicit ``None`` values for optional fields (e.g. a message's
223+
``name``/``function_call``/``tool_calls`` or a chat block's ``tools``). The
224+
JS/UI evaluator manifest schema treats those fields as optional-when-absent
225+
rather than nullable, so an explicit ``null`` fails validation and the remote
226+
eval never appears in the playground. Stripping ``None`` keeps the emitted
227+
default aligned with what the UI schema expects.
228+
"""
229+
if isinstance(value, dict):
230+
return {key: _strip_none_values(item) for key, item in value.items() if item is not None}
231+
if isinstance(value, list):
232+
return [_strip_none_values(item) for item in value]
233+
return value
234+
235+
218236
def _prompt_data_to_dict(
219237
prompt_data: PromptDataDict | PromptData | None,
220238
) -> dict[str, Any] | None:
221239
if prompt_data is None:
222240
return None
223241
if isinstance(prompt_data, PromptData):
224-
return prompt_data.as_dict()
225-
return dict(prompt_data)
242+
return _strip_none_values(prompt_data.as_dict())
243+
return _strip_none_values(dict(prompt_data))
226244

227245

228246
def _create_prompt(name: str, prompt_data: dict[str, Any]) -> "Prompt":

py/src/braintrust/test_parameters.py

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,10 @@
55
RemoteEvalParameters,
66
parameters_to_json_schema,
77
serialize_eval_parameters,
8+
serialize_remote_eval_parameters_container,
89
validate_parameters,
910
)
11+
from braintrust.prompt import PromptChatBlock, PromptData, PromptMessage
1012

1113

1214
HAS_PYDANTIC = importlib.util.find_spec("pydantic") is not None
@@ -563,3 +565,73 @@ def test_parameters_to_json_schema_does_not_mark_passthrough_values_required():
563565
)
564566

565567
assert "required" not in schema
568+
569+
570+
def _assert_no_none_values(node):
571+
if isinstance(node, dict):
572+
for key, value in node.items():
573+
assert value is not None, f"unexpected None at key {key!r}"
574+
_assert_no_none_values(value)
575+
elif isinstance(node, list):
576+
for item in node:
577+
_assert_no_none_values(item)
578+
579+
580+
def test_prompt_parameter_defaults_omit_none_values():
581+
prompt_data = PromptData(
582+
prompt=PromptChatBlock(messages=[PromptMessage(role="user", content="{{input}}")]),
583+
options={"model": "gpt-5-mini"},
584+
)
585+
586+
serialized = serialize_remote_eval_parameters_container(
587+
{
588+
"grouping_prompt": {
589+
"type": "prompt",
590+
"description": "Grouping prompt",
591+
"default": prompt_data,
592+
}
593+
}
594+
)
595+
596+
default = serialized["schema"]["grouping_prompt"]["default"]
597+
assert "tools" not in default["prompt"]
598+
assert "name" not in default["prompt"]["messages"][0]
599+
assert "function_call" not in default["prompt"]["messages"][0]
600+
assert "tool_calls" not in default["prompt"]["messages"][0]
601+
_assert_no_none_values(default)
602+
603+
604+
def test_prompt_parameter_defaults_omit_none_values_from_dict():
605+
prompt_default = {
606+
"prompt": {
607+
"type": "chat",
608+
"messages": [
609+
{
610+
"role": "user",
611+
"content": "{{input}}",
612+
"name": None,
613+
"function_call": None,
614+
"tool_calls": None,
615+
}
616+
],
617+
"tools": None,
618+
},
619+
"options": {"model": "gpt-5-mini"},
620+
}
621+
622+
serialized = serialize_eval_parameters(
623+
{
624+
"grouping_prompt": {
625+
"type": "prompt",
626+
"description": "Grouping prompt",
627+
"default": prompt_default,
628+
}
629+
}
630+
)
631+
632+
default = serialized["grouping_prompt"]["default"]
633+
assert "tools" not in default["prompt"]
634+
assert "name" not in default["prompt"]["messages"][0]
635+
assert "function_call" not in default["prompt"]["messages"][0]
636+
assert "tool_calls" not in default["prompt"]["messages"][0]
637+
_assert_no_none_values(default)

0 commit comments

Comments
 (0)