Skip to content

Commit 4f71b45

Browse files
GWealecopybara-github
authored andcommitted
chore: Add LiteLLM streaming sample
Co-authored-by: George Weale <gweale@google.com> PiperOrigin-RevId: 869397634
1 parent 9562cd6 commit 4f71b45

3 files changed

Lines changed: 149 additions & 0 deletions

File tree

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from . import agent
Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""LiteLLM sample agent for SSE text streaming."""
16+
17+
from __future__ import annotations
18+
19+
from google.adk import Agent
20+
from google.adk.models.lite_llm import LiteLlm
21+
22+
root_agent = Agent(
23+
name='litellm_streaming_agent',
24+
model=LiteLlm(model='gemini/gemini-2.5-flash'),
25+
description='A LiteLLM agent used for streaming text responses.',
26+
instruction='You are a verbose assistant',
27+
)
Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Runs the LiteLLM streaming sample with SSE enabled."""
16+
17+
from __future__ import annotations
18+
19+
import asyncio
20+
21+
from dotenv import load_dotenv
22+
from google.adk.agents.run_config import RunConfig
23+
from google.adk.agents.run_config import StreamingMode
24+
from google.adk.cli.utils import logs
25+
from google.adk.runners import InMemoryRunner
26+
from google.genai import types
27+
28+
from . import agent
29+
30+
load_dotenv(override=True)
31+
logs.log_to_tmp_folder()
32+
33+
34+
async def _run_prompt(
35+
*,
36+
runner: InMemoryRunner,
37+
user_id: str,
38+
session_id: str,
39+
prompt: str,
40+
) -> None:
41+
"""Runs one prompt and prints partial chunks in real time."""
42+
content = types.Content(
43+
role='user',
44+
parts=[types.Part.from_text(text=prompt)],
45+
)
46+
47+
print(f'User: {prompt}')
48+
print('Agent: ', end='', flush=True)
49+
saw_text = False
50+
saw_partial_text = False
51+
52+
# For `adk web`, enable the `Streaming` toggle in the UI to get
53+
# partial SSE responses similar to this script.
54+
async for event in runner.run_async(
55+
user_id=user_id,
56+
session_id=session_id,
57+
new_message=content,
58+
run_config=RunConfig(streaming_mode=StreamingMode.SSE),
59+
):
60+
if not event.content:
61+
continue
62+
text = ''.join(part.text for part in event.content.parts if part.text)
63+
if not text:
64+
continue
65+
66+
if event.partial:
67+
print(text, end='', flush=True)
68+
saw_text = True
69+
saw_partial_text = True
70+
continue
71+
72+
# With SSE mode, ADK emits a final aggregated event after partial chunks.
73+
if not saw_partial_text:
74+
print(text, end='', flush=True)
75+
saw_text = True
76+
77+
if saw_text:
78+
print()
79+
else:
80+
print('(no text response)')
81+
print('------------------------------------')
82+
83+
84+
async def main() -> None:
85+
app_name = 'litellm_streaming_demo'
86+
user_id = 'user_1'
87+
runner = InMemoryRunner(agent=agent.root_agent, app_name=app_name)
88+
session = await runner.session_service.create_session(
89+
app_name=app_name,
90+
user_id=user_id,
91+
)
92+
prompts = [
93+
'Write an essay about the roman empire',
94+
'Now summarize the essay into one sentence.',
95+
]
96+
97+
for prompt in prompts:
98+
await _run_prompt(
99+
runner=runner,
100+
user_id=user_id,
101+
session_id=session.id,
102+
prompt=prompt,
103+
)
104+
105+
106+
if __name__ == '__main__':
107+
asyncio.run(main())

0 commit comments

Comments
 (0)