Skip to content

Commit e8ff5cc

Browse files
yoavkatzyoavkatz
authored
Merge pull request #45 from yoavkatz/fix/authbridge-otel-skip-hosts
Route OTEL collector around authbridge sidecar via skip_hosts
2 parents 9921f52 + 0ef173c commit e8ff5cc

2 files changed

Lines changed: 160 additions & 34 deletions

File tree

‎exgentic_a2a_runner/authbridge/wait-for-reload.sh‎

Lines changed: 68 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,9 @@
2626
# - `authbridge-proxy` post-binary-split (proxy-sidecar mode);
2727
# the operator picks this name when injecting
2828
# the proxy-sidecar binary
29-
# This script auto-detects which one is in the pod.
29+
# This script auto-detects which one is in the pod. During a rollout it
30+
# probes every sidecar-bearing pod each poll and succeeds as soon as any
31+
# reports the target SHA, so it doesn't matter which pod is "first".
3032

3133
set -euo pipefail
3234

@@ -35,57 +37,89 @@ AGENT_NAME=${2:?agent name required}
3537
WANT_SHA=${3:?want-sha required}
3638
TIMEOUT=${4:-180}
3739

38-
# Auto-detect the authbridge container name. Sidecars are injected by
39-
# the AuthBridge webhook at pod-admission, so the Deployment spec only
40-
# lists the agent — we have to look at a running pod. Try the legacy
41-
# name first (`envoy-proxy`) since most clusters today still run the
42-
# older operator layout; fall back to the post-#411 combined name
43-
# (`authbridge`).
44-
POD_CONTAINERS=$(kubectl -n "$NAMESPACE" get pods \
45-
-l app.kubernetes.io/name="$AGENT_NAME" \
46-
-o jsonpath='{.items[0].spec.containers[*].name}' 2>/dev/null || true)
40+
# The operator-injected sidecar container is one of these names,
41+
# depending on operator version (see header comment).
4742
AUTHBRIDGE_CANDIDATES=(authbridge-proxy authbridge envoy-proxy)
48-
AUTHBRIDGE_CONTAINER=""
49-
for candidate in "${AUTHBRIDGE_CANDIDATES[@]}"; do
50-
if echo "$POD_CONTAINERS" | tr ' ' '\n' | grep -qx "$candidate"; then
51-
AUTHBRIDGE_CONTAINER="$candidate"
52-
break
53-
fi
54-
done
55-
if [[ -z "$AUTHBRIDGE_CONTAINER" ]]; then
56-
echo "ERROR: could not find an authbridge sidecar container in pods of $AGENT_NAME." >&2
57-
echo " Looked for: ${AUTHBRIDGE_CANDIDATES[*]}." >&2
58-
echo " Containers found: ${POD_CONTAINERS:-<none>}" >&2
59-
exit 1
60-
fi
61-
echo "[*] Authbridge container detected: $AUTHBRIDGE_CONTAINER"
43+
44+
# Enumerate the agent's pods and, for each, the authbridge sidecar
45+
# container it carries (if any). Emits one "pod<TAB>container" line per
46+
# sidecar-bearing pod. We re-run this every poll iteration rather than
47+
# picking a single pod up front:
48+
#
49+
# During a rollout the pod list contains, in arbitrary order, the old
50+
# pod (agent-only if it predates AuthBridge, or a sidecar still serving
51+
# the stale config) alongside the new pod (mounting the patched
52+
# ConfigMap). The old `.items[0]` assumption picked whichever came first
53+
# — often the old/agent-only pod — and then either failed to find a
54+
# sidecar at all or waited forever on a pod whose config will never
55+
# advance (it's being terminated). Because success is defined purely by
56+
# "some sidecar reports the target SHA", we just probe every sidecar pod
57+
# each round; the converged pod wins regardless of list order, and
58+
# old/terminating pods that never match are harmless.
59+
list_sidecar_pods() {
60+
kubectl -n "$NAMESPACE" get pods \
61+
-l app.kubernetes.io/name="$AGENT_NAME" \
62+
-o jsonpath='{range .items[*]}{.metadata.name}{"|"}{range .spec.containers[*]}{.name}{","}{end}{"\n"}{end}' \
63+
2>/dev/null | while IFS='|' read -r pod containers; do
64+
[[ -z "$pod" ]] && continue
65+
for candidate in "${AUTHBRIDGE_CANDIDATES[@]}"; do
66+
if echo "$containers" | tr ',' '\n' | grep -qx "$candidate"; then
67+
printf '%s\t%s\n' "$pod" "$candidate"
68+
break
69+
fi
70+
done
71+
done
72+
}
6273

6374
DEADLINE=$(( $(date +%s) + TIMEOUT ))
6475

6576
echo "[*] Waiting for authbridge to load the patched config (timeout ${TIMEOUT}s)"
6677
echo " target SHA: $WANT_SHA"
6778

68-
ACTIVE_SHA=""
79+
SAW_SIDECAR=false # did we ever find a sidecar-bearing pod?
80+
LAST_POD="" # for the failure diagnostics
81+
LAST_CONTAINER=""
82+
LAST_SHA=""
6983
while [[ $(date +%s) -lt $DEADLINE ]]; do
70-
ACTIVE_SHA=$(kubectl -n "$NAMESPACE" exec deploy/"$AGENT_NAME" -c "$AUTHBRIDGE_CONTAINER" -- \
71-
wget -q -O - http://localhost:9093/reload/status 2>/dev/null | \
72-
python3 -c 'import json, sys
84+
SIDECAR_PODS=$(list_sidecar_pods || true)
85+
if [[ -n "$SIDECAR_PODS" ]]; then
86+
SAW_SIDECAR=true
87+
while IFS=$'\t' read -r pod container; do
88+
[[ -z "$pod" ]] && continue
89+
LAST_POD="$pod"; LAST_CONTAINER="$container"
90+
active=$(kubectl -n "$NAMESPACE" exec "$pod" -c "$container" -- \
91+
wget -q -O - http://localhost:9093/reload/status 2>/dev/null | \
92+
python3 -c 'import json, sys
7393
try:
7494
print(json.load(sys.stdin).get("active_config_sha256", ""))
7595
except Exception:
7696
pass' 2>/dev/null || true)
77-
if [[ "$ACTIVE_SHA" == "$WANT_SHA" ]]; then
78-
echo "[*] Active config SHA matches — patch is live."
79-
exit 0
97+
[[ -n "$active" ]] && LAST_SHA="$active"
98+
if [[ "$active" == "$WANT_SHA" ]]; then
99+
echo "[*] Active config SHA matches on pod $pod ($container) — patch is live."
100+
exit 0
101+
fi
102+
done <<< "$SIDECAR_PODS"
80103
fi
81104
sleep 3
82105
done
83106

107+
if ! $SAW_SIDECAR; then
108+
echo "ERROR: could not find an authbridge sidecar container in pods of $AGENT_NAME." >&2
109+
echo " Looked for: ${AUTHBRIDGE_CANDIDATES[*]}." >&2
110+
echo " Pods and their containers:" >&2
111+
kubectl -n "$NAMESPACE" get pods -l app.kubernetes.io/name="$AGENT_NAME" \
112+
-o jsonpath='{range .items[*]}{" "}{.metadata.name}{": "}{range .spec.containers[*]}{.name}{" "}{end}{"\n"}{end}' >&2 2>/dev/null || true
113+
exit 1
114+
fi
115+
84116
echo "ERROR: authbridge active config did not match patched SHA within ${TIMEOUT}s." >&2
85117
echo " want: $WANT_SHA" >&2
86-
echo " last active: ${ACTIVE_SHA:-<none>}" >&2
87-
echo " Last 20 lines of the $AUTHBRIDGE_CONTAINER container:" >&2
88-
kubectl -n "$NAMESPACE" logs deploy/"$AGENT_NAME" -c "$AUTHBRIDGE_CONTAINER" --tail=20 >&2 || true
118+
echo " last active: ${LAST_SHA:-<none>}" >&2
119+
if [[ -n "$LAST_POD" ]]; then
120+
echo " Last 20 lines of the $LAST_CONTAINER container (pod $LAST_POD):" >&2
121+
kubectl -n "$NAMESPACE" logs "$LAST_POD" -c "$LAST_CONTAINER" --tail=20 >&2 || true
122+
fi
89123
echo >&2
90124
echo " Likely causes:" >&2
91125
echo " - ConfigMap parse error (look for 'reload failed' above)" >&2

‎exgentic_a2a_runner/deploy-agent.sh‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -629,6 +629,98 @@ PYEOF
629629
) || exit 1
630630
fi
631631

632+
# Step 7.5: Bake OTEL-collector skip_hosts into the NAMESPACE authbridge base
633+
# BEFORE the agent is created, so the sidecar has it from first pod boot — no
634+
# post-deploy patch of the running agent, no reload race.
635+
#
636+
# Why the namespace base and not the per-agent ConfigMap: the rossoctl operator
637+
# webhook force-overwrites `authbridge-config-<agent>` (server-side apply,
638+
# ForceOwnership) on every pod admission, so a per-agent pre-seed can't survive.
639+
# The one thing it does honor is the namespace-level `authbridge-runtime-config`
640+
# `config.yaml`, which it reads as the base and MERGES into each per-agent CM —
641+
# it only injects reverse_proxy_addr / reverse_proxy_backend / forward_proxy_addr
642+
# into `listener`, so a `listener.skip_hosts` we add here rides through untouched.
643+
#
644+
# NOTE: `skip_hosts` is an AuthBridge-*binary* listener key; it is not defined in
645+
# rossoctl/operator source and has NOT been verified against the sidecar's config
646+
# schema. If the sidecar rejects the merged config, look for a "reload failed" /
647+
# parse error in the sidecar container logs
648+
# (kubectl -n "$NAMESPACE" logs deploy/<agent> -c authbridge # or authbridge-proxy / envoy-proxy)
649+
# and correct the key name/shape below. This edit is namespace-wide and persists
650+
# for every agent in $NAMESPACE.
651+
if [ "$AUTHBRIDGE_ENABLED" = "true" ] && [ "$CLUSTER_MODE" != "in-cluster" ]; then
652+
echo "Step 7.5: Adding OTEL-collector skip_hosts to namespace authbridge base..."
653+
NS_AB_CM="authbridge-runtime-config"
654+
655+
if ! command -v python3 >/dev/null 2>&1 || ! python3 -c 'import yaml' 2>/dev/null; then
656+
echo "Error: python3 with PyYAML is required to merge skip_hosts into $NS_AB_CM." >&2
657+
echo " Install with one of:" >&2
658+
echo " pip3 install --user pyyaml" >&2
659+
echo " brew install libyaml && pip3 install pyyaml # macOS" >&2
660+
echo " sudo apt install python3-yaml # Debian/Ubuntu" >&2
661+
exit 1
662+
elif ! kubectl -n "$NAMESPACE" get configmap "$NS_AB_CM" >/dev/null 2>&1; then
663+
echo "Error: ConfigMap $NAMESPACE/$NS_AB_CM not found." >&2
664+
echo " The operator/Helm owns this base; it must exist before deploying an" >&2
665+
echo " AuthBridge-enabled agent. We do not fabricate one here (that would drop" >&2
666+
echo " the pipeline: block and break auth). Ensure the namespace is provisioned." >&2
667+
exit 1
668+
else
669+
CURRENT_NS_YAML=$(
670+
kubectl -n "$NAMESPACE" get configmap "$NS_AB_CM" \
671+
-o jsonpath='{.data.config\.yaml}'
672+
)
673+
MERGED_NS_YAML=$(printf '%s' "$CURRENT_NS_YAML" | python3 - <<'PYEOF'
674+
import sys
675+
import yaml
676+
677+
# Hosts the sidecar must NOT intercept (OTEL collector, rossoctl-system).
678+
SKIP_HOSTS = [
679+
"otel-collector.rossoctl-system.svc.cluster.local",
680+
"otel-collector",
681+
]
682+
683+
cfg = yaml.safe_load(sys.stdin.read()) or {}
684+
listener = cfg.get("listener")
685+
if not isinstance(listener, dict):
686+
listener = {}
687+
existing = listener.get("skip_hosts")
688+
if not isinstance(existing, list):
689+
existing = []
690+
# Union + sort + de-dupe so re-runs are byte-stable and we don't clobber
691+
# hosts someone else added.
692+
listener["skip_hosts"] = sorted(set(existing) | set(SKIP_HOSTS))
693+
cfg["listener"] = listener
694+
# sort_keys=True keeps output deterministic across runs for the no-op check.
695+
sys.stdout.write(yaml.safe_dump(cfg, default_flow_style=False, sort_keys=True))
696+
PYEOF
697+
) || { echo "Error: skip_hosts merge failed for $NAMESPACE/$NS_AB_CM (bad YAML in config.yaml?)" >&2; exit 1; }
698+
699+
if [ -z "$MERGED_NS_YAML" ]; then
700+
echo "Error: skip_hosts merge produced empty output for $NAMESPACE/$NS_AB_CM" >&2
701+
exit 1
702+
elif [ "$CURRENT_NS_YAML" = "$MERGED_NS_YAML" ]; then
703+
echo "✓ Namespace authbridge base already has skip_hosts — nothing to patch"
704+
else
705+
TMP_NS_CONFIG=$(mktemp)
706+
printf '%s' "$MERGED_NS_YAML" >"$TMP_NS_CONFIG"
707+
# Conflict-free create --dry-run | apply (same pattern as apply-pipeline.sh).
708+
if kubectl -n "$NAMESPACE" create configmap "$NS_AB_CM" \
709+
--from-file=config.yaml="$TMP_NS_CONFIG" \
710+
--dry-run=client -o yaml \
711+
| kubectl -n "$NAMESPACE" apply -f - >/dev/null; then
712+
echo "✓ skip_hosts added to $NAMESPACE/$NS_AB_CM"
713+
rm -f "$TMP_NS_CONFIG"
714+
else
715+
echo "Error: could not apply skip_hosts to $NAMESPACE/$NS_AB_CM" >&2
716+
rm -f "$TMP_NS_CONFIG"
717+
exit 1
718+
fi
719+
fi
720+
fi
721+
echo ""
722+
fi
723+
632724
AGENT_JSON=$(cat <<EOF
633725
{
634726
"name": "$AGENT_NAME",

0 commit comments

Comments
 (0)