Skip to content

Commit 3f42e1f

Browse files
committed
Add unified k3d local development with Helm mode (DEV_MODE=helm)
Enable all AMP services to run inside the same k3d cluster as OpenChoreo, eliminating the need for Docker Compose network bridging and host.docker.internal hacks. Docker Compose remains the default mode. New files: - deployments/dev-cluster-config.yaml: k3d config with AMP port mappings - deployments/scripts/build-and-import.sh: build images and import into k3d - deployments/scripts/helm-deploy-amp.sh: helm install/upgrade/uninstall Modified files: - Makefile: DEV_MODE dispatch + helm-* targets (sync, build, logs, status) - deployments/scripts/env.sh: AMP_NAMESPACE, AMP_RELEASE_NAME, AMP_IMAGE_TAG - deployments/scripts/setup-k3d.sh: use dev config when DEV_MODE=helm - deployments/values/values-local.yaml: local image repos + in-cluster URLs - CLAUDE.md: document Helm mode commands
1 parent 7231904 commit 3f42e1f

7 files changed

Lines changed: 596 additions & 58 deletions

File tree

Makefile

Lines changed: 231 additions & 54 deletions
Large diffs are not rendered by default.
Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
# k3d cluster config for Helm dev mode (DEV_MODE=helm)
2+
# Based on single-cluster-config.yaml with additional AMP service port mappings.
3+
# This allows all services (OpenChoreo + AMP) to run in a single k3d cluster.
4+
apiVersion: k3d.io/v1alpha5
5+
kind: Simple
6+
metadata:
7+
name: openchoreo-local-v0.14.0
8+
image: rancher/k3s:v1.32.9-k3s1
9+
servers: 1
10+
agents: 0
11+
kubeAPI:
12+
hostPort: "6550"
13+
ports:
14+
# === OpenChoreo Ports (same as single-cluster-config.yaml) ===
15+
# Control Plane uses port range 8xxx
16+
# HTTP traffic to OpenChoreo UI and API (Kgateway LoadBalancer on port 80)
17+
- port: 8080:80
18+
nodeFilters:
19+
- loadbalancer
20+
# HTTPS traffic to OpenChoreo UI and API (Kgateway LoadBalancer on port 443)
21+
- port: 8443:443
22+
nodeFilters:
23+
- loadbalancer
24+
# Data Plane uses port range 19xxx
25+
# HTTP traffic to workloads via Gateway
26+
- port: 19080:19080
27+
nodeFilters:
28+
- loadbalancer
29+
# HTTPS traffic to workloads via Gateway
30+
- port: 19443:19443
31+
nodeFilters:
32+
- loadbalancer
33+
# Build Plane uses port range 10xxx
34+
# Argo Workflows UI for development testing
35+
- port: 10081:2746
36+
nodeFilters:
37+
- loadbalancer
38+
# Container Registry for storing built images
39+
- port: 10082:5000
40+
nodeFilters:
41+
- loadbalancer
42+
# Observability Plane uses port range 11xxx
43+
# Observer API
44+
- port: 11080:8080
45+
nodeFilters:
46+
- loadbalancer
47+
# OpenSearch Dashboard
48+
- port: 11081:5601
49+
nodeFilters:
50+
- loadbalancer
51+
# OpenSearch API for Fluent Bit data pushing
52+
- port: 11082:9200
53+
nodeFilters:
54+
- loadbalancer
55+
56+
# === AMP Service Ports ===
57+
# Console (React frontend)
58+
- port: 3000:3000
59+
nodeFilters:
60+
- loadbalancer
61+
# Agent Manager API
62+
- port: 9000:9000
63+
nodeFilters:
64+
- loadbalancer
65+
# Internal API / Gateway Management
66+
- port: 9243:9243
67+
nodeFilters:
68+
- loadbalancer
69+
# Traces Observer Service
70+
- port: 9098:9098
71+
nodeFilters:
72+
- loadbalancer
73+
74+
# === OTel / Observability Ports ===
75+
# Data Prepper HTTP source
76+
- port: 21893:21893
77+
nodeFilters:
78+
- loadbalancer
79+
# OTel gRPC
80+
- port: 22893:22893
81+
nodeFilters:
82+
- loadbalancer
83+
# OTel HTTP
84+
- port: 22894:22894
85+
nodeFilters:
86+
- loadbalancer
87+
options:
88+
k3s:
89+
extraArgs:
90+
# Add host.k3d.internal to API server TLS certificate SANs.
91+
# This allows consistent DataPlane configuration across single and multi-cluster setups
92+
# where Control Plane pods can access the API server via host.k3d.internal:6550
93+
- arg: "--tls-san=host.k3d.internal"
94+
nodeFilters:
95+
- server:*
96+
# Configure kubelet eviction thresholds to prevent resource exhaustion
97+
- arg: "--kubelet-arg=eviction-hard=imagefs.available<1%,nodefs.available<1%"
98+
nodeFilters:
99+
- server:*
100+
- arg: "--kubelet-arg=eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%"
101+
nodeFilters:
102+
- server:*
103+
# Disable Traefik to avoid conflicts with OpenChoreo Gateway controller
104+
- arg: "--disable=traefik"
105+
nodeFilters:
106+
- server:*
107+
# Configure insecure registries for HTTP access
108+
# Allows kubelet to pull images from Build Plane registry via HTTP
109+
registries:
110+
config: |
111+
mirrors:
112+
"host.k3d.internal:10082":
113+
endpoint:
114+
- http://host.k3d.internal:10082
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# Build Docker images from production Dockerfiles and import them into k3d.
5+
# Usage:
6+
# ./build-and-import.sh # Build and import all components
7+
# ./build-and-import.sh api # Build and import API only
8+
# ./build-and-import.sh console api # Build and import Console and API
9+
#
10+
# Supported components: api, console, traces-observer, evaluation-job
11+
12+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
13+
ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
14+
15+
source "$SCRIPT_DIR/env.sh"
16+
17+
# Components and their build contexts (relative to ROOT_DIR)
18+
declare -A COMPONENT_CONTEXT=(
19+
[api]="agent-manager-service"
20+
[console]="console"
21+
[traces-observer]="traces-observer-service"
22+
[evaluation-job]="evaluation-job"
23+
)
24+
25+
declare -A COMPONENT_IMAGE=(
26+
[api]="amp-api"
27+
[console]="amp-console"
28+
[traces-observer]="amp-traces-observer"
29+
[evaluation-job]="amp-evaluation-job"
30+
)
31+
32+
ALL_COMPONENTS="api console traces-observer evaluation-job"
33+
34+
# Determine which components to build
35+
if [ $# -eq 0 ]; then
36+
COMPONENTS="$ALL_COMPONENTS"
37+
else
38+
COMPONENTS="$*"
39+
fi
40+
41+
# Validate component names
42+
for comp in $COMPONENTS; do
43+
if [ -z "${COMPONENT_CONTEXT[$comp]}" ]; then
44+
echo "Unknown component: $comp"
45+
echo "Valid components: $ALL_COMPONENTS"
46+
exit 1
47+
fi
48+
done
49+
50+
echo "=== Building and importing images into k3d ==="
51+
echo ""
52+
53+
# Verify k3d cluster exists
54+
if ! k3d cluster list 2>/dev/null | grep -q "${CLUSTER_NAME}"; then
55+
echo "k3d cluster '${CLUSTER_NAME}' not found. Run 'make setup-k3d' first."
56+
exit 1
57+
fi
58+
59+
FAILED=""
60+
61+
for comp in $COMPONENTS; do
62+
IMAGE="${COMPONENT_IMAGE[$comp]}:${AMP_IMAGE_TAG}"
63+
CONTEXT="${ROOT_DIR}/${COMPONENT_CONTEXT[$comp]}"
64+
65+
echo "Building ${comp} -> ${IMAGE}..."
66+
if docker build -t "$IMAGE" "$CONTEXT" --quiet; then
67+
echo "Importing ${IMAGE} into k3d cluster..."
68+
if k3d image import "$IMAGE" -c "${CLUSTER_NAME}"; then
69+
echo "${comp} ready."
70+
else
71+
echo "Failed to import ${comp}."
72+
FAILED="$FAILED $comp"
73+
fi
74+
else
75+
echo "Failed to build ${comp}."
76+
FAILED="$FAILED $comp"
77+
fi
78+
echo ""
79+
done
80+
81+
if [ -n "$FAILED" ]; then
82+
echo "Failed components:${FAILED}"
83+
exit 1
84+
fi
85+
86+
echo "All images built and imported successfully."

deployments/scripts/env.sh

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,8 @@ OPENCHOREO_VERSION="0.14.0"
33
OPENCHOREO_PATCH_VERSION="0.0.0-b53c6dc3"
44
CLUSTER_NAME="openchoreo-local-v${OPENCHOREO_VERSION}"
55
CLUSTER_CONTEXT="k3d-${CLUSTER_NAME}"
6+
7+
# AMP (Agent Management Platform) variables
8+
AMP_NAMESPACE="wso2-amp"
9+
AMP_RELEASE_NAME="amp"
10+
AMP_IMAGE_TAG="0.0.0-dev"
Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# Deploy AMP to the k3d cluster using Helm.
5+
# Usage:
6+
# ./helm-deploy-amp.sh # Install or upgrade
7+
# ./helm-deploy-amp.sh --uninstall # Uninstall (preserves cluster)
8+
9+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
10+
ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
11+
CHART_DIR="$ROOT_DIR/deployments/helm-charts/wso2-agent-manager"
12+
VALUES_FILE="$ROOT_DIR/deployments/values/values-local.yaml"
13+
14+
source "$SCRIPT_DIR/env.sh"
15+
16+
if [ "$1" = "--uninstall" ]; then
17+
echo "=== Uninstalling AMP from k3d ==="
18+
if helm status "$AMP_RELEASE_NAME" -n "$AMP_NAMESPACE" --kube-context "${CLUSTER_CONTEXT}" &>/dev/null; then
19+
helm uninstall "$AMP_RELEASE_NAME" -n "$AMP_NAMESPACE" --kube-context "${CLUSTER_CONTEXT}"
20+
echo "AMP uninstalled. Cluster and namespace preserved."
21+
else
22+
echo "AMP release '${AMP_RELEASE_NAME}' not found in namespace '${AMP_NAMESPACE}'."
23+
fi
24+
exit 0
25+
fi
26+
27+
echo "=== Deploying AMP to k3d cluster ==="
28+
echo ""
29+
30+
# Verify cluster is accessible
31+
if ! kubectl cluster-info --context "${CLUSTER_CONTEXT}" &>/dev/null; then
32+
echo "k3d cluster '${CLUSTER_NAME}' is not accessible."
33+
echo "Run 'make setup-k3d' or 'k3d cluster start ${CLUSTER_NAME}' first."
34+
exit 1
35+
fi
36+
37+
# Create namespace if it doesn't exist
38+
kubectl create namespace "$AMP_NAMESPACE" --context "${CLUSTER_CONTEXT}" --dry-run=client -o yaml | \
39+
kubectl apply --context "${CLUSTER_CONTEXT}" -f -
40+
41+
# Update Helm dependencies
42+
echo "Updating Helm chart dependencies..."
43+
helm dependency update "$CHART_DIR"
44+
echo ""
45+
46+
# Install or upgrade
47+
echo "Running helm upgrade --install..."
48+
helm upgrade --install "$AMP_RELEASE_NAME" "$CHART_DIR" \
49+
--namespace "$AMP_NAMESPACE" \
50+
--kube-context "${CLUSTER_CONTEXT}" \
51+
--values "$VALUES_FILE" \
52+
--wait \
53+
--timeout 5m
54+
55+
echo ""
56+
echo "Waiting for deployments to be ready..."
57+
kubectl wait --for=condition=Available deployment --all \
58+
-n "$AMP_NAMESPACE" \
59+
--context "${CLUSTER_CONTEXT}" \
60+
--timeout=300s 2>/dev/null || true
61+
62+
echo ""
63+
echo "AMP deployed successfully!"
64+
echo ""
65+
echo "Services:"
66+
echo " Console: http://localhost:3000"
67+
echo " API: http://localhost:9000"
68+
echo ""
69+
echo "Status:"
70+
kubectl get pods -n "$AMP_NAMESPACE" --context "${CLUSTER_CONTEXT}"

deployments/scripts/setup-k3d.sh

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,14 @@ cd "$SCRIPT_DIR"
99

1010
source "$SCRIPT_DIR/env.sh"
1111

12-
echo "=== Setting up k3d Cluster for OpenChoreo ==="
12+
# Select k3d config based on DEV_MODE
13+
if [ "${DEV_MODE}" = "helm" ]; then
14+
K3D_CONFIG="../dev-cluster-config.yaml"
15+
echo "=== Setting up k3d Cluster for OpenChoreo + AMP (Helm mode) ==="
16+
else
17+
K3D_CONFIG="../single-cluster-config.yaml"
18+
echo "=== Setting up k3d Cluster for OpenChoreo ==="
19+
fi
1320

1421
# Check prerequisites
1522
if ! command -v k3d &> /dev/null; then
@@ -52,6 +59,27 @@ if k3d cluster list 2>/dev/null | grep -q "${CLUSTER_NAME}"; then
5259
done
5360
fi
5461

62+
# When using Helm mode, verify AMP ports are mapped
63+
if [ "${DEV_MODE}" = "helm" ]; then
64+
echo ""
65+
echo "🔍 Checking AMP port mappings..."
66+
MISSING_PORTS=""
67+
for PORT in 3000 9000; do
68+
if ! docker port "k3d-${CLUSTER_NAME}-serverlb" "${PORT}/tcp" &>/dev/null; then
69+
MISSING_PORTS="${MISSING_PORTS} ${PORT}"
70+
fi
71+
done
72+
if [ -n "$MISSING_PORTS" ]; then
73+
echo "⚠️ AMP ports not mapped:${MISSING_PORTS}"
74+
echo " The cluster was created without AMP port mappings."
75+
echo " To fix, delete and recreate the cluster:"
76+
echo " k3d cluster delete ${CLUSTER_NAME}"
77+
echo " DEV_MODE=helm make setup-k3d"
78+
else
79+
echo "✅ AMP ports are mapped correctly"
80+
fi
81+
fi
82+
5583
echo ""
5684
echo "Cluster info:"
5785
kubectl cluster-info --context ${CLUSTER_CONTEXT}
@@ -62,9 +90,9 @@ else
6290
echo "📁 Creating shared directory for OpenChoreo..."
6391
mkdir -p /tmp/k3d-shared
6492

65-
# Create k3d cluster with OpenChoreo configuration
66-
echo "🚀 Creating k3d cluster with OpenChoreo configuration..."
67-
k3d cluster create --config ../single-cluster-config.yaml
93+
# Create k3d cluster with appropriate configuration
94+
echo "🚀 Creating k3d cluster with config: ${K3D_CONFIG}..."
95+
k3d cluster create --config "${K3D_CONFIG}"
6896

6997
echo ""
7098
echo "✅ k3d cluster created successfully!"
Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
# Local development overrides for helm-based deployment (DEV_MODE=helm)
2+
# Used by: DEV_MODE=helm make dev-up, make helm-install, make helm-upgrade
3+
#
4+
# All services run inside the k3d cluster. Images are built locally and
5+
# imported via `k3d image import` (pullPolicy: Never).
6+
7+
agentManagerService:
8+
image:
9+
repository: amp-api
10+
tag: "0.0.0-dev"
11+
pullPolicy: Never
12+
service:
13+
type: LoadBalancer
14+
port: 9000
15+
config:
16+
logLevel: "DEBUG"
17+
corsAllowedOrigin: "*"
18+
# In-cluster OpenChoreo URL (no host.docker.internal needed)
19+
openChoreo:
20+
baseURL: "http://openchoreo-api.openchoreo-control-plane:8080/api/v1"
21+
# In-cluster Thunder IDP
22+
keyManager:
23+
jwksUrl: "http://amp-thunder-extension-service.amp-thunder.svc.cluster.local:8090/oauth2/jwks"
24+
oidc:
25+
tokenUrl: "http://amp-thunder-extension-service.amp-thunder.svc.cluster.local:8090/oauth2/token"
26+
# In-cluster OpenBao
27+
openbao:
28+
url: "http://amp-secrets-openbao.amp-secrets.svc.cluster.local:8200"
29+
# In-cluster OTel endpoint
30+
otel:
31+
exporterEndpoint: "http://obs-gateway-gateway-router.openchoreo-data-plane.svc.cluster.local:22893/otel"
32+
# In-cluster observer
33+
observerURL: "http://observer.openchoreo-observability-plane.svc.cluster.local:8080"
34+
traceObserverURL: "http://amp-traces-observer.openchoreo-observability-plane.svc.cluster.local:9098"
35+
36+
console:
37+
image:
38+
repository: amp-console
39+
tag: "0.0.0-dev"
40+
pullPolicy: Never
41+
service:
42+
type: LoadBalancer
43+
config:
44+
disableAuth: "true"
45+
apiBaseUrl: "http://localhost:9000"
46+
instrumentationUrl: "http://localhost:21893"
47+
48+
dbMigration:
49+
image:
50+
repository: amp-api
51+
tag: "0.0.0-dev"
52+
pullPolicy: Never
53+
54+
postgresql:
55+
primary:
56+
persistence:
57+
enabled: true
58+
size: 5Gi

0 commit comments

Comments
 (0)