Skip to content

Commit 657d95e

Browse files
yizhaodevyizhaodev
authored
docs: upgrade llm-d to v0.7.0 and use standalone helm charts (llm-d#455)
Signed-off-by: Raymond Zhao <zhaoeryi@gmail.com>
1 parent df763ce commit 657d95e

3 files changed

Lines changed: 270 additions & 25 deletions

File tree

‎examples/deploy-demo/deploy-k8s.sh‎

Lines changed: 29 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,14 @@ BATCH_INTERNAL_GATEWAY_NAME="${BATCH_INTERNAL_GATEWAY_NAME:-batch-internal-gatew
3636
BATCH_INTERNAL_GATEWAY_NAMESPACE="${BATCH_INTERNAL_GATEWAY_NAMESPACE:-${GATEWAY_NAMESPACE}}"
3737
GATEWAY_LOCAL_PORT="${GATEWAY_LOCAL_PORT:-8080}"
3838

39-
LLMD_VERSION="${LLMD_VERSION:-v0.6.0}"
39+
LLMD_VERSION="${LLMD_VERSION:-v0.7.0}"
4040
LLMD_GIT_DIR="/tmp/llm-d-${LLMD_VERSION}"
4141
LLMD_RELEASE_POSTFIX="${LLMD_RELEASE_POSTFIX:-llmd}"
4242

43-
# Model name matches the simulated-accelerators helmfile default ("random")
43+
GAIE_CHART_VERSION="${GAIE_CHART_VERSION:-v1.5.0}"
44+
MODELSERVICE_CHART_VERSION="${MODELSERVICE_CHART_VERSION:-v0.4.12}"
45+
46+
# Model name matches the simulated model default ("random")
4447
MODEL_NAME="${MODEL_NAME:-random}"
4548
LLMD_POOL_NAME="gaie-${LLMD_RELEASE_POSTFIX}"
4649

@@ -429,20 +432,29 @@ install_llmd_deps() {
429432
deploy_llmd_model() {
430433
step "Deploying model with llm-d..."
431434

432-
local llmd_dir="${LLMD_GIT_DIR}"
433-
local sim_dir="${llmd_dir}/guides/simulated-accelerators"
434-
435-
# Reduce resource requests — defaults (4 CPU) exceed demo/CI cluster capacity.
436-
local istio_config="${llmd_dir}/guides/prereq/gateway-provider/common-configurations/istio.yaml"
437-
# Disable llm-d's own gateway — we use our own istio-gateway with AuthPolicy
438-
yq -i '.gateway.enabled = false' "${istio_config}"
439-
yq -i '.inferenceExtension.resources = {"requests": {"cpu": "500m", "memory": "512Mi"}, "limits": {"cpu": "2", "memory": "1Gi"}}' \
440-
"${sim_dir}/gaie-sim/values.yaml"
441-
RELEASE_NAME_POSTFIX="${LLMD_RELEASE_POSTFIX}" \
442-
helmfile apply -f "${sim_dir}/helmfile.yaml.gotmpl" -e istio -n "${LLM_NAMESPACE}"
435+
local sim_dir="${SCRIPT_DIR}/llmd-sim"
436+
local pool_name="${LLMD_POOL_NAME}"
437+
local epp_host="${pool_name}-epp.${LLM_NAMESPACE}.svc.cluster.local"
438+
439+
# Install InferencePool (GAIE EPP)
440+
step "Installing InferencePool chart ${GAIE_CHART_VERSION}..."
441+
helm upgrade --install "${pool_name}" \
442+
oci://registry.k8s.io/gateway-api-inference-extension/charts/inferencepool \
443+
--version "${GAIE_CHART_VERSION}" \
444+
--namespace "${LLM_NAMESPACE}" \
445+
-f "${sim_dir}/gaie-sim-values.yaml" \
446+
--set "provider.istio.destinationRule.host=${epp_host}"
447+
448+
# Install ModelService (vllm-sim)
449+
step "Installing ModelService chart ${MODELSERVICE_CHART_VERSION}..."
450+
helm repo add llm-d-modelservice https://llm-d-incubation.github.io/llm-d-modelservice/ --force-update
451+
helm upgrade --install "ms-${LLMD_RELEASE_POSTFIX}" llm-d-modelservice/llm-d-modelservice \
452+
--version "${MODELSERVICE_CHART_VERSION}" \
453+
--namespace "${LLM_NAMESPACE}" \
454+
-f "${sim_dir}/ms-sim-values.yaml"
443455

444456
# Wait for llm-d deployments
445-
wait_for_deployment "${LLMD_POOL_NAME}-epp" "${LLM_NAMESPACE}" 300s
457+
wait_for_deployment "${pool_name}-epp" "${LLM_NAMESPACE}" 300s
446458
wait_for_deployment "ms-${LLMD_RELEASE_POSTFIX}-llm-d-modelservice-decode" "${LLM_NAMESPACE}" 300s
447459

448460
log "llm-d model deployed."
@@ -451,16 +463,8 @@ deploy_llmd_model() {
451463
uninstall_llmd() {
452464
step "Removing llm-d stack (${LLM_NAMESPACE})..."
453465
timeout_delete 30s httproute --all -n "${LLM_NAMESPACE}" || true
454-
local sim_helmfile="${LLMD_GIT_DIR}/guides/simulated-accelerators/helmfile.yaml.gotmpl"
455-
if [ -f "${sim_helmfile}" ]; then
456-
RELEASE_NAME_POSTFIX="${LLMD_RELEASE_POSTFIX}" \
457-
helmfile destroy -f "${sim_helmfile}" -e istio -n "${LLM_NAMESPACE}" 2>/dev/null \
458-
|| warn "helmfile destroy failed"
459-
else
460-
helm uninstall "ms-${LLMD_RELEASE_POSTFIX}" -n "${LLM_NAMESPACE}" --timeout 60s 2>/dev/null || true
461-
helm uninstall "${LLMD_POOL_NAME}" -n "${LLM_NAMESPACE}" --timeout 60s 2>/dev/null || true
462-
helm uninstall "infra-${LLMD_RELEASE_POSTFIX}" -n "${LLM_NAMESPACE}" --timeout 60s 2>/dev/null || true
463-
fi
466+
helm uninstall "ms-${LLMD_RELEASE_POSTFIX}" -n "${LLM_NAMESPACE}" --timeout 60s 2>/dev/null || true
467+
helm uninstall "${LLMD_POOL_NAME}" -n "${LLM_NAMESPACE}" --timeout 60s 2>/dev/null || true
464468
timeout_delete 30s inferencepool --all -n "${LLM_NAMESPACE}" || true
465469
}
466470

@@ -875,7 +879,7 @@ usage() {
875879
echo ""
876880
echo "Examples:"
877881
echo " $0 install"
878-
echo " MODEL_NAME=my-model LLMD_VERSION=v0.5.0 $0 install"
882+
echo " MODEL_NAME=my-model LLMD_VERSION=v0.7.0 $0 install"
879883
exit "${1:-0}"
880884
}
881885

‎examples/deploy-demo/llmd-sim/gaie-sim-values.yaml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
# InferencePool chart values for simulated-accelerators demo.
2+
# Used with: oci://registry.k8s.io/gateway-api-inference-extension/charts/inferencepool v1.5.0
3+
#
4+
# Equivalent to the former guides/simulated-accelerators/gaie-sim/values.yaml
5+
# in the llm-d repo, which was removed after v0.6.0.
6+
7+
inferenceExtension:
8+
replicas: 1
9+
image:
10+
registry: ghcr.io
11+
repository: llm-d/llm-d-inference-scheduler
12+
tag: v0.8.0
13+
pullPolicy: Always
14+
extProcPort: 9002
15+
pluginsConfigFile: "default-plugins.yaml"
16+
tracing:
17+
enabled: false
18+
monitoring:
19+
interval: "10s"
20+
prometheus:
21+
enabled: true
22+
auth:
23+
enabled: true
24+
secretName: sim-gateway-sa-metrics-reader-secret
25+
# Reduced for demo — defaults (4 CPU) exceed demo/CI cluster capacity.
26+
resources:
27+
requests:
28+
cpu: "500m"
29+
memory: 512Mi
30+
limits:
31+
cpu: "2"
32+
memory: 1Gi
33+
34+
inferencePool:
35+
targetPorts:
36+
- number: 8000
37+
modelServerType: vllm
38+
modelServers:
39+
matchLabels:
40+
llm-d.ai/inference-serving: "true"
41+
llm-d.ai/guide: "simulated-accelerators"
42+
llm-d.ai/accelerator-variant: "cpu"
43+
llm-d.ai/model: "random"
44+
45+
provider:
46+
name: istio
47+
istio:
48+
destinationRule:
49+
trafficPolicy:
50+
connectionPool:
51+
http:
52+
http1MaxPendingRequests: 256000
53+
maxRequestsPerConnection: 256000
54+
http2MaxRequests: 256000
55+
idleTimeout: "900s"
56+
tcp:
57+
maxConnections: 256000
58+
maxConnectionDuration: "1800s"
59+
connectTimeout: "900s"

‎examples/deploy-demo/llmd-sim/ms-sim-values.yaml‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
# ModelService chart values for simulated-accelerators demo.
2+
# Used with: llm-d-modelservice/llm-d-modelservice v0.4.12
3+
#
4+
# Equivalent to the former guides/simulated-accelerators/ms-sim/values.yaml
5+
# in the llm-d repo, which was removed after v0.6.0.
6+
7+
multinode: false
8+
9+
modelArtifacts:
10+
uri: "hf://random"
11+
name: random
12+
size: 5Mi
13+
labels:
14+
llm-d.ai/inference-serving: "true"
15+
llm-d.ai/guide: "simulated-accelerators"
16+
llm-d.ai/accelerator-variant: "cpu"
17+
llm-d.ai/model: "random"
18+
19+
routing:
20+
servicePort: 8000
21+
proxy:
22+
image: ghcr.io/llm-d/llm-d-routing-sidecar:v0.8.0
23+
connector: nixlv2
24+
secure: false
25+
26+
decode:
27+
create: true
28+
replicas: 3
29+
monitoring:
30+
podmonitor:
31+
enabled: true
32+
portName: "vllm"
33+
path: "/metrics"
34+
interval: "30s"
35+
initContainers:
36+
- name: uds-tokenizer
37+
image: ghcr.io/llm-d/llm-d-uds-tokenizer:v0.8.0
38+
imagePullPolicy: IfNotPresent
39+
restartPolicy: Always
40+
env:
41+
- name: LOG_LEVEL
42+
value: "DEBUG"
43+
- name: PROBE_PORT
44+
value: "8082"
45+
ports:
46+
- name: health
47+
containerPort: 8082
48+
livenessProbe:
49+
httpGet:
50+
path: /healthz
51+
port: 8082
52+
initialDelaySeconds: 10
53+
periodSeconds: 10
54+
readinessProbe:
55+
httpGet:
56+
path: /healthz
57+
port: 8082
58+
initialDelaySeconds: 5
59+
periodSeconds: 5
60+
volumeMounts:
61+
- name: uds-socket
62+
mountPath: /tmp/tokenizer
63+
containers:
64+
- name: "vllm"
65+
image: "ghcr.io/llm-d/llm-d-inference-sim:v0.8.2"
66+
modelCommand: imageDefault
67+
ports:
68+
- containerPort: 8200
69+
name: vllm
70+
protocol: TCP
71+
mountModelVolume: true
72+
volumeMounts:
73+
- name: metrics-volume
74+
mountPath: /.config
75+
- name: uds-socket
76+
mountPath: /tmp/tokenizer
77+
startupProbe:
78+
httpGet:
79+
path: /v1/models
80+
port: vllm
81+
initialDelaySeconds: 15
82+
periodSeconds: 30
83+
timeoutSeconds: 5
84+
failureThreshold: 60
85+
livenessProbe:
86+
httpGet:
87+
path: /health
88+
port: vllm
89+
periodSeconds: 10
90+
timeoutSeconds: 5
91+
failureThreshold: 3
92+
readinessProbe:
93+
httpGet:
94+
path: /v1/models
95+
port: vllm
96+
periodSeconds: 5
97+
timeoutSeconds: 2
98+
failureThreshold: 3
99+
volumes:
100+
- name: metrics-volume
101+
emptyDir: {}
102+
- name: uds-socket
103+
emptyDir: {}
104+
105+
prefill:
106+
create: true
107+
replicas: 1
108+
monitoring:
109+
podmonitor:
110+
enabled: true
111+
portName: "vllm"
112+
path: "/metrics"
113+
interval: "30s"
114+
initContainers:
115+
- name: uds-tokenizer
116+
image: ghcr.io/llm-d/llm-d-uds-tokenizer:v0.8.0
117+
imagePullPolicy: IfNotPresent
118+
restartPolicy: Always
119+
env:
120+
- name: LOG_LEVEL
121+
value: "DEBUG"
122+
- name: PROBE_PORT
123+
value: "8082"
124+
ports:
125+
- name: health
126+
containerPort: 8082
127+
livenessProbe:
128+
httpGet:
129+
path: /healthz
130+
port: 8082
131+
initialDelaySeconds: 10
132+
periodSeconds: 10
133+
readinessProbe:
134+
httpGet:
135+
path: /healthz
136+
port: 8082
137+
initialDelaySeconds: 5
138+
periodSeconds: 5
139+
volumeMounts:
140+
- name: uds-socket
141+
mountPath: /tmp/tokenizer
142+
containers:
143+
- name: "vllm"
144+
image: "ghcr.io/llm-d/llm-d-inference-sim:v0.8.2"
145+
modelCommand: imageDefault
146+
ports:
147+
- containerPort: 8000
148+
name: vllm
149+
protocol: TCP
150+
mountModelVolume: true
151+
volumeMounts:
152+
- name: metrics-volume
153+
mountPath: /.config
154+
- name: uds-socket
155+
mountPath: /tmp/tokenizer
156+
startupProbe:
157+
httpGet:
158+
path: /v1/models
159+
port: vllm
160+
initialDelaySeconds: 15
161+
periodSeconds: 30
162+
timeoutSeconds: 5
163+
failureThreshold: 60
164+
livenessProbe:
165+
httpGet:
166+
path: /health
167+
port: vllm
168+
periodSeconds: 10
169+
timeoutSeconds: 5
170+
failureThreshold: 3
171+
readinessProbe:
172+
httpGet:
173+
path: /v1/models
174+
port: vllm
175+
periodSeconds: 5
176+
timeoutSeconds: 2
177+
failureThreshold: 3
178+
volumes:
179+
- name: metrics-volume
180+
emptyDir: {}
181+
- name: uds-socket
182+
emptyDir: {}

0 commit comments

Comments
 (0)