fix: prevent model bypass in resumable mode by rejecting user-authored function calls

Reject 'function_call' parts in user-authored events when they are appended to the session. This prevents attackers from bypassing the LLM and directly executing arbitrary registered tools.

Co-authored-by: Kathy Wu <wukathy@google.com>
PiperOrigin-RevId: 945261515

Author: wukath

src/google/adk/flows/llm_flows/base_llm_flow.py

@@ -806,6 +806,8 @@ async def _send_to_model(
 
       if live_request.content:
         content = live_request.content
+        if content.parts and any(p.function_call for p in content.parts):
+          raise ValueError('User message cannot contain function calls.')
         # Persist user text content to session (similar to non-live mode)
         # Skip function responses - they are already handled separately
         is_function_response = content.parts and any(

src/google/adk/runners.py

@@ -725,6 +725,8 @@ async def _append_user_event(
       state_delta: Optional[dict[str, Any]] = None,
   ) -> Event:
     """Append a user message event to the session and return it."""
+    if content.parts and any(p.function_call for p in content.parts):
+      raise ValueError('User message cannot contain function calls.')
     if state_delta:
       event = Event(
           invocation_id=ic.invocation_id,
@@ -1455,6 +1457,9 @@ async def _append_new_message_to_session(
     if not new_message.parts:
       raise ValueError('No parts in the new_message.')
 
+    if any(p.function_call for p in new_message.parts):
+      raise ValueError('User message cannot contain function calls.')
+
     if self.artifact_service and save_input_blobs_as_artifacts:
       # Issue deprecation warning
       warnings.warn(

tests/unittests/flows/llm_flows/test_base_llm_flow.py

@@ -1857,3 +1857,39 @@ async def test_postprocess_live_voice_activity_events():
 
   assert len(events) == 1
   assert events[0].voice_activity == vad
+
+
+@pytest.mark.asyncio
+async def test_send_to_model_rejects_function_call():
+  """Test that _send_to_model raises ValueError if user message contains function calls."""
+  agent = Agent(name='test_agent')
+  invocation_context = await testing_utils.create_invocation_context(
+      agent=agent
+  )
+  invocation_context.live_request_queue = LiveRequestQueue()
+
+  # Put a malicious content request in the queue
+  from google.adk.agents.live_request_queue import LiveRequest
+
+  malicious_request = LiveRequest(
+      content=types.Content(
+          role='user',
+          parts=[
+              types.Part(
+                  function_call=types.FunctionCall(
+                      name='some_tool',
+                      args={'key': 'value'},
+                  )
+              )
+          ],
+      )
+  )
+  invocation_context.live_request_queue.send(malicious_request)
+
+  flow = BaseLlmFlowForTesting()
+  mock_connection = mock.AsyncMock()
+
+  with pytest.raises(
+      ValueError, match='User message cannot contain function calls'
+  ):
+    await flow._send_to_model(mock_connection, invocation_context)

tests/unittests/test_runners.py

@@ -1999,5 +1999,41 @@ async def test_get_session_config_limits_events():
   assert len(limited_session.events) == 3
 
 
+@pytest.mark.asyncio
+async def test_run_async_rejects_user_function_call():
+  """Verify that runner rejects user-authored messages with function calls."""
+  session_service = InMemorySessionService()
+  runner = Runner(
+      app_name=TEST_APP_ID,
+      agent=MockAgent("test_agent"),
+      session_service=session_service,
+      artifact_service=InMemoryArtifactService(),
+      auto_create_session=True,
+  )
+
+  malicious_message = types.Content(
+      role="user",
+      parts=[
+          types.Part(
+              function_call=types.FunctionCall(
+                  name="some_tool",
+                  args={"key": "value"},
+              )
+          )
+      ],
+  )
+
+  agen = runner.run_async(
+      user_id=TEST_USER_ID,
+      session_id=TEST_SESSION_ID,
+      new_message=malicious_message,
+  )
+
+  with pytest.raises(ValueError, match="cannot contain function calls"):
+    async with aclosing(agen) as a:
+      async for _ in a:
+        pass
+
+
 if __name__ == "__main__":
   pytest.main([__file__])