Skip to content

Commit 78ea024

Browse files
committed
router: add GMS placement indexer policy
Signed-off-by: Maksim Khadkevich <mkhadkevich@nvidia.com>
1 parent ad7d5bc commit 78ea024

18 files changed

Lines changed: 1763 additions & 110 deletions

File tree

.github/filters.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ core:
101101
- 'components/src/dynamo/mocker/**'
102102
- 'components/src/dynamo/frontend/**'
103103
- 'components/src/dynamo/common/**'
104+
- 'components/src/dynamo/gms_router_policy.py'
104105
- 'components/src/dynamo/gpu_memory_service/**'
105106
# TODO(will): Move TokenSpeed to a dedicated framework filter once CI has
106107
# TokenSpeed Docker images and backend-specific build/test jobs.
@@ -196,6 +197,7 @@ frontend:
196197
- 'components/src/dynamo/mocker/**'
197198
- 'components/src/dynamo/frontend/**'
198199
- 'components/src/dynamo/common/**'
200+
- 'components/src/dynamo/gms_router_policy.py'
199201
- 'deploy/inference-gateway/**'
200202
- 'container/templates/frontend.Dockerfile'
201203
- '!**/*.md'
Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
import pytest
5+
6+
from dynamo import gms_router_policy
7+
from dynamo.gms_router_policy import (
8+
DynamoGmsPlacementPublisher,
9+
maybe_fetch_gms_placement,
10+
)
11+
12+
pytestmark = [pytest.mark.pre_merge, pytest.mark.unit, pytest.mark.gpu_0]
13+
14+
15+
def test_router_policy_exposes_no_gms_control_endpoint():
16+
assert not hasattr(gms_router_policy, "GMS_CONTROL_ENDPOINT")
17+
assert not hasattr(gms_router_policy, "make_gms_control_handler")
18+
19+
20+
@pytest.mark.asyncio
21+
async def test_missing_local_gms_socket_leaves_request_on_vanilla_path():
22+
request = {
23+
"prompt": "hello",
24+
"gms_placement": {
25+
"source_nixl_agent_name": "source",
26+
"source_nixl_agent_metadata_hex": "00",
27+
"hashes": ["00"],
28+
"descriptors": [None],
29+
},
30+
}
31+
32+
result = await maybe_fetch_gms_placement(request, None)
33+
34+
assert result is None
35+
assert request["prompt"] == "hello"
36+
assert "gms_placement" in request
37+
38+
39+
def test_router_gms_decode_transfer_arg_defaults_off():
40+
from dynamo.router.args import parse_args
41+
42+
config = parse_args(["--endpoint", "ns.comp.generate"])
43+
44+
assert config.router_gms_decode_transfer is False
45+
assert config.kv_router_kwargs()["router_gms_decode_transfer"] is False
46+
47+
48+
def test_router_gms_decode_transfer_arg_can_opt_in():
49+
from dynamo.router.args import parse_args
50+
51+
config = parse_args(
52+
["--endpoint", "ns.comp.generate", "--router-gms-decode-transfer"]
53+
)
54+
55+
assert config.router_gms_decode_transfer is True
56+
assert config.kv_router_kwargs()["router_gms_decode_transfer"] is True
57+
58+
59+
class _FakeKvPublisher:
60+
def __init__(self):
61+
self.stored = []
62+
self.removed = []
63+
64+
def publish_gms_placement_stored(self, *args, **kwargs):
65+
self.stored.append((args, kwargs))
66+
67+
def publish_gms_placement_removed(self, *args, **kwargs):
68+
self.removed.append((args, kwargs))
69+
70+
71+
def test_dynamo_gms_placement_publisher_normalizes_descriptor():
72+
fake = _FakeKvPublisher()
73+
publisher = DynamoGmsPlacementPublisher(fake, dp_rank=2)
74+
75+
publisher.publish_stored(
76+
bytes.fromhex("aa" * 32),
77+
"host_pinned",
78+
64,
79+
metadata={
80+
"source_nixl_agent_name": "agent-a",
81+
"source_nixl_agent_metadata_hex": "abcd",
82+
"source_nixl_ip": "10.0.0.1",
83+
"source_nixl_listen_port": "5555",
84+
"gms_descriptor": {
85+
"ptr": "1234",
86+
"size": "64",
87+
"tier": "host",
88+
"generation": "7",
89+
"ranges": [
90+
{
91+
"ptr": "1234",
92+
"size": "64",
93+
"tier": "host",
94+
"layer": "3",
95+
"offset": "8",
96+
}
97+
],
98+
},
99+
},
100+
)
101+
102+
assert len(fake.stored) == 1
103+
args, kwargs = fake.stored[0]
104+
assert args[0] == "agent-a"
105+
assert args[1] == "abcd"
106+
assert args[2][0]["content_hash_hex"] == "aa" * 32
107+
descriptor = args[2][0]["descriptor"]
108+
assert descriptor["remote_ptr"] == 1234
109+
assert descriptor["generation"] == 7
110+
assert descriptor["ranges"][0]["layer"] == 3
111+
assert kwargs == {
112+
"source_nixl_ip": "10.0.0.1",
113+
"source_nixl_listen_port": 5555,
114+
"dp_rank": 2,
115+
}
116+
117+
118+
def test_dynamo_gms_placement_publisher_drops_missing_metadata():
119+
fake = _FakeKvPublisher()
120+
publisher = DynamoGmsPlacementPublisher(fake)
121+
122+
publisher.publish_stored(
123+
bytes.fromhex("bb" * 32),
124+
"host_pinned",
125+
64,
126+
metadata={"gms_descriptor": {"remote_ptr": 1, "size": 64}},
127+
)
128+
129+
assert fake.stored == []
130+
131+
132+
def test_dynamo_gms_placement_publisher_removes_hash():
133+
fake = _FakeKvPublisher()
134+
publisher = DynamoGmsPlacementPublisher(fake, dp_rank=1)
135+
136+
publisher.publish_removed(bytes.fromhex("cc" * 32), "host_pinned")
137+
138+
assert fake.removed == [((["cc" * 32],), {"dp_rank": 1})]

0 commit comments

Comments
 (0)