Skip to content

Commit 1f9d63e

Browse files
binary-huskybinary-husky
committed
lazy init url relay client
1 parent 6144985 commit 1f9d63e

7 files changed

Lines changed: 328 additions & 26 deletions

File tree

‎ajet/tuner.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ def __init__(
2727
self.llm_inference_fn = llm_inference_fn
2828
self.target2proxy_registry: dict[str, dict[str,TunerTypeUnion]] = {}
2929
if config.ajet.enable_experimental_reverse_proxy:
30-
self._enable_experimental_interchange_server(llm_inference_fn)
30+
self.proxy_client_started = False
3131

3232

3333
def as_agentscope_model(
@@ -102,6 +102,9 @@ def as_oai_baseurl_apikey(
102102
"""
103103

104104
assert self.config.ajet.enable_experimental_reverse_proxy, "Please enable `ajet.enable_experimental_reverse_proxy` in yaml config to use `as_oai_baseurl_apikey` feature."
105+
if self.proxy_client_started is False:
106+
self._enable_experimental_interchange_server(self.llm_inference_fn)
107+
self.proxy_client_started = True
105108
baseurl_apikey_model = OpenaiClientBaseUrlTuner(
106109
config=self.config,
107110
context_tracker=self.context_tracker,
@@ -178,4 +181,5 @@ def _enable_experimental_interchange_server(self, llm_inference_fn):
178181
def terminate_episode(self):
179182
# experimental reverse proxy cleanup
180183
if self.config.ajet.enable_experimental_reverse_proxy:
181-
self.interchange_client._should_terminate = True
184+
if (self.proxy_client_started is True) and hasattr(self, "interchange_client"):
185+
self.interchange_client._should_terminate = True

‎ajet/tuner_lib/weight_tuner/experimental/as_oai_model_client.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ async def _service_loop(self):
115115
assert port is not None, "AJET_DAT_INTERCHANGE_PORT env var must be set"
116116
uri = f"ws://127.0.0.1:{port}/hook/context_tracker_client_listen"
117117

118-
async with websockets.connect(uri) as websocket:
118+
async with websockets.connect(uri, ping_timeout=3600) as websocket:
119119
try:
120120
# Send initialization parameters
121121
# Sending as a list [agent_name, target_tag, episode_uuid] to match "input (a,b,c)" structure

‎ajet/tuner_lib/weight_tuner/experimental/as_oai_model_server.py‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
from pprint import pformat
2222
from loguru import logger
2323

24-
from pydantic import BaseModel
24+
from pydantic import BaseModel, ConfigDict, model_validator
2525
from fastapi import FastAPI, WebSocket, WebSocketDisconnect, Header, HTTPException, Request
2626
import uvicorn
2727

@@ -30,14 +30,21 @@
3030

3131

3232
# Global variables for tracking requests and responses
33+
# class ChatCompletionRequestDeferBuildOff(ChatCompletionRequest):
34+
# model_config = ConfigDict(
35+
# defer_build=False,
36+
# validate_default=True,
37+
# validate_assignment=True,
38+
# )
39+
# ChatCompletionRequest.model_validate_json(x)
40+
3341
class InterchangeCompletionRequest(BaseModel):
3442
completion_request: ChatCompletionRequest
3543
agent_name: str
3644
target_tag: str
3745
episode_uuid: str
3846
timeline_uuid: str
3947

40-
4148
ajet_remote_handler_received: Dict[str, Dict[str, InterchangeCompletionRequest]] = defaultdict(dict)
4249
ajet_remote_handler_in_progress: Dict[str, Dict[str, InterchangeCompletionRequest]] = defaultdict(dict)
4350
ajet_remote_handler_completed: Dict[str, Dict[str, ChatCompletion]] = defaultdict(dict)
@@ -221,19 +228,30 @@ async def chat_completions(request: Request, authorization: str = Header(None)):
221228

222229
# Parse request body
223230
body = await request.json()
224-
new_req = ChatCompletionRequest(**body)
231+
new_req = ChatCompletionRequest.model_validate(body)
232+
225233
# Create timeline UUID
226234
timeline_uuid = uuid.uuid4().hex
235+
227236
# Add to received queue
228237
# logger.warning(f"Received new chat completion request for agent: {agent_name}, target_tag: {target_tag}, episode_uuid: {episode_uuid}, timeline_uuid: {timeline_uuid}")
229-
ajet_remote_handler_received[key][timeline_uuid] = InterchangeCompletionRequest(
238+
int_req = InterchangeCompletionRequest(
230239
completion_request = new_req,
231240
agent_name = agent_name,
232241
target_tag = target_tag,
233242
episode_uuid = episode_uuid,
234243
timeline_uuid = timeline_uuid,
235244
)
236245

246+
# fix Pydantic validation issue for tool_calls field
247+
for msg in int_req.completion_request.messages:
248+
if isinstance(msg, dict) and 'tool_calls' in msg:
249+
tc = msg['tool_calls']
250+
if not isinstance(tc, list):
251+
msg['tool_calls'] = list(tc) if tc else []
252+
253+
ajet_remote_handler_received[key][timeline_uuid] = int_req
254+
237255
# Wait for response (with periodic checks for client disconnect)
238256
max_wait_time = 1800 # 30 minutes timeout
239257
check_interval = 0.25 # Check every 250ms

‎docs/en/agent_framework_support.md‎

Lines changed: 42 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,42 @@
1-
# Supported Agent Frameworks
2-
3-
4-
5-
6-
##
7-
8-
9-
10-
## AgentScope
11-
12-
13-
14-
15-
16-
17-
1+
# Agent Framework Support
2+
3+
4+
<div class="card-grid">
5+
<a href="en/tune_your_first_agent/" class="feature-card">
6+
<div class="card-header"><img src="https://api.iconify.design/lucide:rocket.svg"
7+
class="card-icon card-icon-agent" alt="">
8+
<h3>AgentScope</h3>
9+
</div>
10+
<p class="card-desc">
11+
Agent-Oriented Programming for Building LLM Applications.
12+
</p>
13+
</a>
14+
<a href="#example-library" class="feature-card">
15+
<div class="card-header"><img src="https://api.iconify.design/lucide:library.svg"
16+
class="card-icon card-icon-general" alt="">
17+
<h3>LangChain</h3>
18+
</div>
19+
<p class="card-desc">
20+
LangChain provides the engineering platform and open source frameworks developers use to build, test, and deploy reliable AI agents.
21+
</p>
22+
</a>
23+
<a href="https://benchmark.agent-matrix.com/" class="feature-card">
24+
<div class="card-header"><img src="https://api.iconify.design/lucide:shield-check.svg" class="card-icon card-icon-tool"
25+
alt="">
26+
<h3>OpenAI SDK</h3>
27+
</div>
28+
<p class="card-desc">
29+
The OpenAI Agents SDK allows you to build agentic AI applications in a lightweight and easy-to-use package with minimal abstractions. By the way, both vLLM and SGLang offer compatible services.
30+
</p>
31+
</a>
32+
<a href="https://benchmark.agent-matrix.com/" class="feature-card">
33+
<div class="card-header"><img src="https://api.iconify.design/lucide:shield-check.svg" class="card-icon card-icon-tool"
34+
alt="">
35+
<h3>Raw HTTP</h3>
36+
</div>
37+
<p class="card-desc">
38+
Why use the Agent SDKs and all these abstractions? If you want to take control of the foundation of LLM Agents,
39+
in this AI era, you can always start from scratch and build your own "high-scrapers".
40+
</p>
41+
</a>
42+
</div>

‎docs/en/debugging_guide.md‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
In this tutorial, we introduce the way to debug the workflow and the training algorithms.
22

3-
## Workflow Debugging
3+
4+
5+
## Workflow Debugging (--backbone=debug)
46

57
1. Install VSCode and connect to GPU server.
68

@@ -74,4 +76,32 @@ Then, the modified launch.json will be
7476

7577
7. Press `F5` to start debugging.
7678

77-
8. You can set breakpoint inside the workflow to observe program execution now.
79+
8. You can set breakpoint inside the workflow to observe program execution now.
80+
81+
82+
## General Debugging (Ray Distributed Debugger)
83+
84+
1. Install the Ray Distributed Debugger extension in VSCode.
85+
86+
2. In AgentJet project:
87+
88+
2-1. In the place your want to set a conditional breakpoint, write
89+
`from ajet import bp; bp("TAG_1")`
90+
91+
2-2. When launching the training process, add `--debug` as commandline argument
92+
`ajet --conf your_config.yaml --debug="TAG_1"`
93+
94+
2-3. Open Tab "Ray Distributed Debugger" in VSCode, and just wait until the breakpoint is hit.
95+
96+
97+
## Comparison
98+
99+
| Feature | Workflow Debugging | General Debugging (Ray) |
100+
| :--- | :--- | :--- |
101+
| **Backend** | `debug`, `tinker` | `verl`, `trinity` |
102+
| **Reboot Speed** | Very Fast | Slow |
103+
| **Debug Target** | Workflow | Everything |
104+
| **VSCode Extension** | Python | Python + Ray Distributed Debugger |
105+
| **Launch Mode** | `F5` standard launch (via `launch.json`) | Command line execution with `ajet ... --debug="TAG"` |
106+
| **Commandline** | `--backbone=debug` | `--debug="TAG1\|TAG2\|TAG3"` |
107+

0 commit comments

Comments
 (0)