Skip to content

Commit db612e7

Browse files
author
Yuriy Bezsonov
committed
fix(perf-platform): apply NLB services sequentially to prevent race condition
1 parent 51a1ef5 commit db612e7

1 file changed

Lines changed: 29 additions & 12 deletions

File tree

infra/scripts/setup/perf-platform.sh

Lines changed: 29 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,11 @@ log_success "RBAC applied"
209209
# =============================================================================
210210

211211
log_info "Provisioning internal NLB for ECS reachability..."
212+
# Apply sequentially: AWS Load Balancer Controller shares one NLB across two
213+
# Services via `aws-load-balancer-name` only if the second Service sees the
214+
# NLB already exists. A single concurrent `kubectl apply` makes both reconcile
215+
# loops race — both try to CreateLoadBalancer and the second hits
216+
# DuplicateLoadBalancerName and gets stuck.
212217
kubectl apply -f - <<EOF
213218
apiVersion: v1
214219
kind: Service
@@ -229,7 +234,26 @@ spec:
229234
port: 4040
230235
targetPort: 4040
231236
protocol: TCP
232-
---
237+
EOF
238+
239+
log_info "Waiting for pyroscope-nlb to provision the shared NLB..."
240+
NLB_DNS=""
241+
for i in {1..60}; do
242+
NLB_DNS=$(kubectl get svc pyroscope-nlb -n "${NAMESPACE}" \
243+
-o jsonpath='{.status.loadBalancer.ingress[0].hostname}' 2>/dev/null || echo "")
244+
if [[ -n "${NLB_DNS}" ]]; then
245+
break
246+
fi
247+
sleep 10
248+
done
249+
250+
if [[ -z "${NLB_DNS}" ]]; then
251+
log_error "NLB DNS was not assigned within 10 minutes"
252+
exit 1
253+
fi
254+
255+
log_info "Attaching perf-analyzer listener to the same NLB..."
256+
kubectl apply -f - <<EOF
233257
apiVersion: v1
234258
kind: Service
235259
metadata:
@@ -251,22 +275,15 @@ spec:
251275
protocol: TCP
252276
EOF
253277

254-
log_info "Waiting for NLB DNS to be assigned..."
255-
NLB_DNS=""
256-
for i in {1..60}; do
257-
NLB_DNS=$(kubectl get svc pyroscope-nlb -n "${NAMESPACE}" \
278+
for i in {1..30}; do
279+
ANALYZER_NLB_DNS=$(kubectl get svc perf-analyzer-nlb -n "${NAMESPACE}" \
258280
-o jsonpath='{.status.loadBalancer.ingress[0].hostname}' 2>/dev/null || echo "")
259-
if [[ -n "${NLB_DNS}" ]]; then
281+
if [[ -n "${ANALYZER_NLB_DNS}" ]]; then
260282
break
261283
fi
262-
sleep 10
284+
sleep 5
263285
done
264286

265-
if [[ -z "${NLB_DNS}" ]]; then
266-
log_error "NLB DNS was not assigned within 10 minutes"
267-
exit 1
268-
fi
269-
270287
aws ssm put-parameter \
271288
--name "perf-platform-internal-nlb" \
272289
--value "${NLB_DNS}" \

0 commit comments

Comments
 (0)