File tree Expand file tree Collapse file tree
docs/source/dev_guide/dev_api Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -10,3 +10,7 @@ Router Logic
1010.. autoclass :: vllm_router.routers.routing_logic.RoundRobinRouter
1111 :members:
1212 :show-inheritance:
13+
14+ .. autoclass :: vllm_router.routers.routing_logic.KvawareRouter
15+ :members:
16+ :show-inheritance:
Original file line number Diff line number Diff line change @@ -202,6 +202,11 @@ def route_request(
202202
203203
204204class KvawareRouter (RoutingInterface ):
205+ """
206+ Route the request to the appropriate engine URL by where the KV cache
207+ of the longest prefix match is found.
208+ """
209+
205210 def __init__ (self , lmcache_controller_port : int ):
206211 self .lmcache_controller_port = lmcache_controller_port
207212 self .kv_manager = controller_manager .LMCacheControllerManager (
@@ -239,12 +244,13 @@ async def route_request(
239244 of the longest prefix match is found.
240245 If there is no session id in the request header, it will pick a server
241246 with round robin.
247+
242248 Args:
243249 endpoints (List[EndpointInfo]): The list of engine URLs
244250 engine_stats (Dict[str, EngineStats]): The engine stats indicating
245- the 'physical' load of each engine
251+ the 'physical' load of each engine
246252 request_stats (Dict[str, RequestStats]): The request stats
247- indicating the request-level performance of each engine
253+ indicating the request-level performance of each engine
248254 request (Request): The incoming request
249255 request_json (Dict): The request body (needed for finding the
250256 longest prefix match)
You can’t perform that action at this time.
0 commit comments