Skip to content

Commit d4f80a9

Browse files
adnanrhussainadnanrhussain
committed
test(python-sdk): add protocol path coverage for execute_prompt_chain_step
Adds TestExecutePromptChainStepProtocolPath (13 tests) covering the llm_provider injection path in BaseEvaluator.execute_prompt_chain_step: - Raw string return when parser_output_type=None - Clean JSON parse - Markdown fence stripping - Trailing prose stripping (JSON followed by explanation text) - Leading prose stripping (prose before JSON) - json_dict_normalizer path - Non-dict JSON raises OutputValidationError on normalizer path - Malformed JSON raises OutputValidationError - Schema mismatch raises OutputValidationError - Token usage recorded in step extras and total_token_usage - Token usage absent when LLMResponse has None tokens - Provider RuntimeError wrapped as APIError - EvaluatorError from provider re-raised unchanged - KeyboardInterrupt from provider propagated
1 parent b0d3562 commit d4f80a9

1 file changed

Lines changed: 241 additions & 0 deletions

File tree

‎sdks/python/tests/evaluators/test_base.py‎

Lines changed: 241 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -857,3 +857,244 @@ def passthrough(d: dict) -> dict:
857857
assert "JSON object" in str(exc_info.value)
858858
assert exc_info.value.provider is LLMProvider.GOOGLE
859859
assert exc_info.value.model == "gemini-2.0-flash"
860+
861+
862+
# ---------------------------------------------------------------------------
863+
# execute_prompt_chain_step — protocol path (llm_provider injected)
864+
# ---------------------------------------------------------------------------
865+
866+
867+
from learning_commons_evaluators.schemas.llm_provider import LLMResponse # noqa: E402
868+
869+
870+
def _make_adapter(
871+
content: str,
872+
model: str = "test-model",
873+
input_tokens: int | None = 10,
874+
output_tokens: int | None = 5,
875+
) -> AsyncMock:
876+
"""Minimal mock that satisfies LLMGeneratorProtocol.generate()."""
877+
adapter = AsyncMock()
878+
adapter.generate = AsyncMock(
879+
return_value=LLMResponse(
880+
content=content, model=model, input_tokens=input_tokens, output_tokens=output_tokens
881+
)
882+
)
883+
return adapter
884+
885+
886+
_PROTO_SETTINGS = PromptSettings(
887+
provider_type=LLMProvider.ANTHROPIC,
888+
model="claude-opus-4-8",
889+
temperature=0.0,
890+
)
891+
892+
_PROTO_TEMPLATE = ChatPromptTemplate.from_messages(
893+
[("system", "You are a grader."), ("human", "{input}")]
894+
)
895+
896+
897+
class TestExecutePromptChainStepProtocolPath:
898+
"""Protocol path: llm_provider injected — LangChain provider is never called."""
899+
900+
def _ev(self, adapter: AsyncMock) -> _StubEvaluator:
901+
return _StubEvaluator(create_config_no_telemetry(), llm_provider=adapter)
902+
903+
async def test_returns_raw_string_when_parser_type_is_none(self, evaluation_metadata):
904+
ev = self._ev(_make_adapter("plain prose"))
905+
out = await ev.execute_prompt_chain_step(
906+
step_name="raw",
907+
prompt_settings=_PROTO_SETTINGS,
908+
evaluation_metadata=evaluation_metadata,
909+
template=_PROTO_TEMPLATE,
910+
chain_inputs={"input": "Hello"},
911+
parser_output_type=None,
912+
)
913+
assert out == "plain prose"
914+
915+
async def test_parses_clean_json(self, evaluation_metadata):
916+
ev = self._ev(_make_adapter(_CHAIN_JSON))
917+
result = await ev.execute_prompt_chain_step(
918+
step_name="main",
919+
prompt_settings=_PROTO_SETTINGS,
920+
evaluation_metadata=evaluation_metadata,
921+
template=_PROTO_TEMPLATE,
922+
chain_inputs={"input": "Hello"},
923+
parser_output_type=_ChainOutput,
924+
)
925+
assert isinstance(result, _ChainOutput)
926+
assert result.label == "ok"
927+
assert result.score == 7
928+
929+
async def test_strips_markdown_fences(self, evaluation_metadata):
930+
fenced = f"```json\n{_CHAIN_JSON}\n```"
931+
ev = self._ev(_make_adapter(fenced))
932+
result = await ev.execute_prompt_chain_step(
933+
step_name="main",
934+
prompt_settings=_PROTO_SETTINGS,
935+
evaluation_metadata=evaluation_metadata,
936+
template=_PROTO_TEMPLATE,
937+
chain_inputs={"input": "Hello"},
938+
parser_output_type=_ChainOutput,
939+
)
940+
assert result.label == "ok"
941+
942+
async def test_strips_trailing_prose(self, evaluation_metadata):
943+
with_prose = f"{_CHAIN_JSON}\n\nHere is my reasoning for this score."
944+
ev = self._ev(_make_adapter(with_prose))
945+
result = await ev.execute_prompt_chain_step(
946+
step_name="main",
947+
prompt_settings=_PROTO_SETTINGS,
948+
evaluation_metadata=evaluation_metadata,
949+
template=_PROTO_TEMPLATE,
950+
chain_inputs={"input": "Hello"},
951+
parser_output_type=_ChainOutput,
952+
)
953+
assert result.label == "ok"
954+
955+
async def test_strips_leading_prose(self, evaluation_metadata):
956+
with_prefix = f"Here is the result:\n{_CHAIN_JSON}"
957+
ev = self._ev(_make_adapter(with_prefix))
958+
result = await ev.execute_prompt_chain_step(
959+
step_name="main",
960+
prompt_settings=_PROTO_SETTINGS,
961+
evaluation_metadata=evaluation_metadata,
962+
template=_PROTO_TEMPLATE,
963+
chain_inputs={"input": "Hello"},
964+
parser_output_type=_ChainOutput,
965+
)
966+
assert result.label == "ok"
967+
968+
async def test_json_dict_normalizer_path(self, evaluation_metadata):
969+
class _Out(BaseModel):
970+
n: int
971+
doubled: int
972+
973+
ev = self._ev(_make_adapter('{"n": 3}'))
974+
result = await ev.execute_prompt_chain_step(
975+
step_name="main",
976+
prompt_settings=_PROTO_SETTINGS,
977+
evaluation_metadata=evaluation_metadata,
978+
template=_PROTO_TEMPLATE,
979+
chain_inputs={"input": "Hello"},
980+
parser_output_type=_Out,
981+
json_dict_normalizer=lambda d: {**d, "doubled": d["n"] * 2},
982+
)
983+
assert result.n == 3
984+
assert result.doubled == 6
985+
986+
async def test_non_dict_json_in_normalizer_path_raises_output_validation_error(
987+
self, evaluation_metadata
988+
):
989+
class _Out(BaseModel):
990+
n: int
991+
992+
ev = self._ev(_make_adapter('["not", "an", "object"]'))
993+
with pytest.raises(OutputValidationError) as exc_info:
994+
await ev.execute_prompt_chain_step(
995+
step_name="main",
996+
prompt_settings=_PROTO_SETTINGS,
997+
evaluation_metadata=evaluation_metadata,
998+
template=_PROTO_TEMPLATE,
999+
chain_inputs={"input": "Hello"},
1000+
parser_output_type=_Out,
1001+
json_dict_normalizer=lambda d: d,
1002+
)
1003+
assert "JSON object" in str(exc_info.value)
1004+
1005+
async def test_malformed_json_raises_output_validation_error(self, evaluation_metadata):
1006+
ev = self._ev(_make_adapter("not json at all"))
1007+
with pytest.raises(OutputValidationError):
1008+
await ev.execute_prompt_chain_step(
1009+
step_name="main",
1010+
prompt_settings=_PROTO_SETTINGS,
1011+
evaluation_metadata=evaluation_metadata,
1012+
template=_PROTO_TEMPLATE,
1013+
chain_inputs={"input": "Hello"},
1014+
parser_output_type=_ChainOutput,
1015+
)
1016+
1017+
async def test_schema_mismatch_raises_output_validation_error(self, evaluation_metadata):
1018+
ev = self._ev(_make_adapter('{"label": "only"}')) # missing required `score`
1019+
with pytest.raises(OutputValidationError) as exc_info:
1020+
await ev.execute_prompt_chain_step(
1021+
step_name="main",
1022+
prompt_settings=_PROTO_SETTINGS,
1023+
evaluation_metadata=evaluation_metadata,
1024+
template=_PROTO_TEMPLATE,
1025+
chain_inputs={"input": "Hello"},
1026+
parser_output_type=_ChainOutput,
1027+
)
1028+
assert isinstance(exc_info.value.__cause__, PydanticValidationError)
1029+
1030+
async def test_token_usage_recorded_in_step_extras_and_total(self, evaluation_metadata):
1031+
ev = self._ev(
1032+
_make_adapter(_CHAIN_JSON, model="claude-opus-4-8", input_tokens=42, output_tokens=17)
1033+
)
1034+
await ev.execute_prompt_chain_step(
1035+
step_name="main",
1036+
prompt_settings=_PROTO_SETTINGS,
1037+
evaluation_metadata=evaluation_metadata,
1038+
template=_PROTO_TEMPLATE,
1039+
chain_inputs={"input": "Hello"},
1040+
parser_output_type=_ChainOutput,
1041+
)
1042+
step = evaluation_metadata.step_details["main"]
1043+
assert step.extras[PROMPT_STEP_EXTRA_TOKEN_USAGE]["input_tokens"] == 42
1044+
assert step.extras[PROMPT_STEP_EXTRA_TOKEN_USAGE]["output_tokens"] == 17
1045+
assert evaluation_metadata.total_token_usage[LLMProvider.ANTHROPIC].input_tokens == 42
1046+
1047+
async def test_token_usage_absent_when_llm_response_has_none_tokens(self, evaluation_metadata):
1048+
ev = self._ev(_make_adapter(_CHAIN_JSON, input_tokens=None, output_tokens=None))
1049+
await ev.execute_prompt_chain_step(
1050+
step_name="main",
1051+
prompt_settings=_PROTO_SETTINGS,
1052+
evaluation_metadata=evaluation_metadata,
1053+
template=_PROTO_TEMPLATE,
1054+
chain_inputs={"input": "Hello"},
1055+
parser_output_type=_ChainOutput,
1056+
)
1057+
assert not evaluation_metadata.total_token_usage
1058+
1059+
async def test_provider_error_wrapped_as_api_error(self, evaluation_metadata):
1060+
adapter = AsyncMock()
1061+
adapter.generate = AsyncMock(side_effect=RuntimeError("network timeout"))
1062+
ev = self._ev(adapter)
1063+
with pytest.raises(APIError) as exc_info:
1064+
await ev.execute_prompt_chain_step(
1065+
step_name="main",
1066+
prompt_settings=_PROTO_SETTINGS,
1067+
evaluation_metadata=evaluation_metadata,
1068+
template=_PROTO_TEMPLATE,
1069+
chain_inputs={"input": "Hello"},
1070+
parser_output_type=_ChainOutput,
1071+
)
1072+
assert isinstance(exc_info.value.__cause__, RuntimeError)
1073+
1074+
async def test_evaluator_error_from_provider_reraises_unchanged(self, evaluation_metadata):
1075+
adapter = AsyncMock()
1076+
adapter.generate = AsyncMock(side_effect=EvaluatorError("already wrapped"))
1077+
ev = self._ev(adapter)
1078+
with pytest.raises(EvaluatorError, match="already wrapped"):
1079+
await ev.execute_prompt_chain_step(
1080+
step_name="main",
1081+
prompt_settings=_PROTO_SETTINGS,
1082+
evaluation_metadata=evaluation_metadata,
1083+
template=_PROTO_TEMPLATE,
1084+
chain_inputs={"input": "Hello"},
1085+
parser_output_type=_ChainOutput,
1086+
)
1087+
1088+
async def test_keyboard_interrupt_from_provider_propagates(self, evaluation_metadata):
1089+
adapter = AsyncMock()
1090+
adapter.generate = AsyncMock(side_effect=KeyboardInterrupt)
1091+
ev = self._ev(adapter)
1092+
with pytest.raises(KeyboardInterrupt):
1093+
await ev.execute_prompt_chain_step(
1094+
step_name="main",
1095+
prompt_settings=_PROTO_SETTINGS,
1096+
evaluation_metadata=evaluation_metadata,
1097+
template=_PROTO_TEMPLATE,
1098+
chain_inputs={"input": "Hello"},
1099+
parser_output_type=_ChainOutput,
1100+
)

0 commit comments

Comments
 (0)