1616
1717import logging
1818from typing import ClassVar
19- from typing import Optional
2019
2120from google .genai import types as genai_types
2221from pydantic import Field
@@ -72,7 +71,7 @@ class LlmBackedUserSimulatorConfig(BaseUserSimulatorConfig):
7271(Not recommended) If you don't want a limit, you can set the value to -1.""" ,
7372 )
7473
75- custom_instructions : Optional [ str ] = Field (
74+ custom_instructions : str | None = Field (
7675 default = None ,
7776 description = """Custom instructions for the LlmBackedUserSimulator. The
7877instructions must contain the following formatting placeholders following Jinja syntax:
@@ -88,7 +87,7 @@ class LlmBackedUserSimulatorConfig(BaseUserSimulatorConfig):
8887
8988 @field_validator ("custom_instructions" )
9089 @classmethod
91- def validate_custom_instructions (cls , value : Optional [ str ] ) -> Optional [ str ] :
90+ def validate_custom_instructions (cls , value : str | None ) -> str | None :
9291 if value is None :
9392 return value
9493 if not is_valid_user_simulator_template (
@@ -158,11 +157,11 @@ def _summarize_conversation(
158157 async def _get_llm_response (
159158 self ,
160159 rewritten_dialogue : str ,
161- ) -> str :
162- """Sends a user message generation request to the LLM and returns the full response."""
160+ ) -> tuple [ str , str | None ] :
161+ """Sends a user message generation request to the LLM and returns the full response and potential error reason ."""
163162 if self ._invocation_count == 0 :
164163 # first invocation - send the static starting prompt
165- return self ._conversation_scenario .starting_prompt
164+ return self ._conversation_scenario .starting_prompt , None
166165
167166 user_agent_instructions = get_llm_backed_user_simulator_prompt (
168167 conversation_plan = self ._conversation_scenario .conversation_plan ,
@@ -187,19 +186,44 @@ async def _get_llm_response(
187186 add_default_retry_options_if_not_present (llm_request )
188187
189188 response = ""
189+ error_reason = None
190+ has_thought_tokens = False
190191 async with Aclosing (self ._llm .generate_content_async (llm_request )) as agen :
191192 async for llm_response in agen :
193+ error_code = llm_response .error_code
194+ if error_code :
195+ logger .warning (
196+ "User simulator LLM returned error: code=%s, message=%s" ,
197+ error_code ,
198+ getattr (llm_response , "error_message" , "" ),
199+ )
200+ error_reason = f"safety filters or other error (code={ error_code } )"
201+ response = ""
202+ break
203+
192204 generated_content : genai_types .Content = llm_response .content
193205 if (
194206 not generated_content
195207 or not hasattr (generated_content , "parts" )
196208 or not generated_content .parts
197209 ):
198210 continue
211+
199212 for part in generated_content .parts :
200- if part .text and not part .thought :
213+ if part .thought :
214+ has_thought_tokens = True
215+ elif part .text :
201216 response += part .text
202- return response
217+
218+ if not response :
219+ if error_reason :
220+ pass # Keep the error reason from error_code
221+ elif has_thought_tokens :
222+ error_reason = "LLM returned only thinking tokens"
223+ else :
224+ error_reason = "LLM returned empty response"
225+
226+ return response , error_reason
203227
204228 @override
205229 async def get_next_user_message (
@@ -234,11 +258,11 @@ async def get_next_user_message(
234258 rewritten_dialogue = self ._summarize_conversation (events )
235259
236260 # query the LLM for the next user message
237- response = await self ._get_llm_response (rewritten_dialogue )
261+ response , error_reason = await self ._get_llm_response (rewritten_dialogue )
238262 self ._invocation_count += 1
239263
240264 # is the conversation over? (Has the user simulator output the stop signal?)
241- if _STOP_SIGNAL .lower () in response .lower ():
265+ if response and _STOP_SIGNAL .lower () in response .lower ():
242266 logger .info (
243267 "Stopping user message generation as the stop signal was detected."
244268 )
@@ -256,11 +280,11 @@ async def get_next_user_message(
256280
257281 # if we are here, the user agent failed to generate a message, which is not
258282 # a valid result for the LLM backed user simulator.
259- raise RuntimeError ("Failed to generate a user message" )
283+ raise RuntimeError (f "Failed to generate a user message: { error_reason } " )
260284
261285 @override
262286 def get_simulation_evaluator (
263287 self ,
264- ) -> Optional [ Evaluator ] :
288+ ) -> Evaluator | None :
265289 """Returns an Evaluator that evaluates if the simulation was successful or not."""
266290 raise NotImplementedError ()
0 commit comments