Support local and remote deployment

Signed-off-by: Andrea Terzolo <andreaterzolo3@gmail.com>

Author: Andreagit97

k8s-deployment/Makefile

@@ -1,64 +1,136 @@
 
-NAMESPACE := default
-DST_DIR   := /suse
+SETUP_TIMEOUT ?= 120s
+SETUP_AGENT_HELM_CHART_DIR ?= ~/personal/helm-charts/stable/suse-observability-agent
+
+# SETUP_TYPE can be 'local' or 'remote' (default: local)
+SETUP_TYPE ?= local
+SETUP_API_KEY ?= null
+SETUP_CLUSTER_NAME ?= minikube-local-setup
+SETUP_RECEIVER_ENDPOINT ?= http://test-server-service:7077/stsAgent
+# To be set only in remote mode
+SETUP_OTEL_ENDPOINT ?= null
+
+.PHONY: create-cluster
+create-cluster:
+	@echo "🚀 Creating Kubernetes cluster..."
+	minikube start --driver=kvm2 --cpus=6 --memory=12g --disk-size='30g' 
+
+.PHONY: delete-cluster
+delete-cluster:
+	@echo "🧹 Deleting Kubernetes cluster..."
+	minikube delete
+
+.PHONY: load-agent-image
+load-agent-image:
+	@echo "🚀 Loading agent image into the cluster..."
+# we should also build again the ebpf probes but at the moment we skip it since we suppose we are in the middle of the development
+	rake local_build
+	docker build --tag registry/agent:latest -f ../Dockerfile ..
+	minikube image rm registry/agent:latest 2>/dev/null || true
+	minikube image load registry/agent:latest
+
+.PHONY: load-test-server-image
+load-test-server-image:
+	@echo "🚀 Loading test server image into the cluster..."
+	go build -o ../test-server ../test-server
+	docker build --tag test-server:latest -f ../test-server/Dockerfile ../test-server
+	minikube image rm test-server:latest 2>/dev/null || true
+	minikube image load test-server:latest
+
+.PHONY: deploy-prometheus
+deploy-prometheus:
+	@echo "🚀 Deploying Prometheus..."
+	helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+	helm upgrade --install --wait --timeout $(SETUP_TIMEOUT) prometheus prometheus-community/prometheus --version 27.32.0 \
+  		--values ./yaml/prometheus-config.yaml \
+  		--namespace open-telemetry --create-namespace
+
+.PHONY: deploy-test-server
+deploy-test-server: load-test-server-image
+	@echo "🚀 Deploying the test server in monitoring namespace..."
+	kubectl create namespace monitoring || true
+	kubectl apply -f ./yaml/test-server.yaml -n monitoring
+	@echo "🕐 Waiting for test-server..."
+	kubectl wait --for=condition=Ready -n monitoring pod -l app=test-server --timeout=$(SETUP_TIMEOUT)
+
+.PHONY: deploy-otel-collector
+deploy-otel-collector:
+	@echo "🚀 Deploying OTEL collector ($(SETUP_TYPE))..."
+# We need to create the secret and configMap for the remote setup so that the OTEL collector configuration can use them
+	if [ "$(SETUP_TYPE)" = "remote" ]; then \
+		kubectl create namespace open-telemetry || true; \
+		kubectl create secret generic open-telemetry-collector --namespace open-telemetry --from-literal=SETUP_API_KEY="$(SETUP_API_KEY)"; \
+		kubectl create configmap open-telemetry-collector-config --namespace open-telemetry \
+			--from-literal=SETUP_CLUSTER_NAME="$(SETUP_CLUSTER_NAME)" \
+			--from-literal=SETUP_OTEL_ENDPOINT="$(SETUP_OTEL_ENDPOINT)"; \
+	fi
+	helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
+	helm upgrade --install --wait --timeout $(SETUP_TIMEOUT) opentelemetry-collector open-telemetry/opentelemetry-collector --version 0.132.0 \
+		--values ./yaml/otel-collector-config-$(SETUP_TYPE).yaml \
+		--namespace open-telemetry --create-namespace
+
+.PHONY: deploy-agent
+deploy-agent: load-agent-image
+	@echo "🚀 Deploying the process agent ($(SETUP_TYPE))..."
+	helm upgrade --install --devel\
+		suse-observability-agent \
+		$(SETUP_AGENT_HELM_CHART_DIR) \
+		--set-string 'stackstate.apiKey'='$(SETUP_API_KEY)' \
+		--set-string 'stackstate.cluster.name'='$(SETUP_CLUSTER_NAME)' \
+		--set-string 'stackstate.url'='$(SETUP_RECEIVER_ENDPOINT)' \
+		--set logsAgent.enabled=false \
+		--set clusterAgent.enabled=false \
+		--set checksAgent.enabled=false \
+		--set kubernetes-rbac-agent.enabled=false \
+		--set processAgent.podCorrelation.enabled=true \
+		--set processAgent.podCorrelation.protocolMetrics=true \
+		--set nodeAgent.httpTracing.enabled=false \
+		--set-string processAgent.podCorrelation.exporter.type=otlp \
+		--set-string processAgent.podCorrelation.exporter.endpoint=opentelemetry-collector.open-telemetry.svc.cluster.local:4317 \
+		--namespace monitoring \
+		--create-namespace \
+		--set-string 'nodeAgent.skipKubeletTLSVerify'='true' \
+		--set-string 'nodeAgent.containers.processAgent.image.pullPolicy'='Never' \
+		--set-string 'nodeAgent.containers.processAgent.image.tag'='latest' \
+		--set-string 'nodeAgent.containers.processAgent.image.registry'='registry' \
+		--set-string 'nodeAgent.containers.processAgent.image.repository'='agent'
 
 .PHONY: deploy
-deploy:
-	@echo "🚀 Deploying the process-agent and basic components in namespace '$(NAMESPACE)'..."
-	kubectl apply -f ./yaml/deployment.yaml -n $(NAMESPACE)
-	@echo "🚀 Deploying the Postgres service in namespace '$(NAMESPACE)'..."
-	kubectl apply -f ./yaml/postgres.yaml -n $(NAMESPACE)
-	@echo "🕐 Waiting for necessary components to be up..."
-	kubectl wait --for=condition=Ready pod -l app=loader-agent --timeout=180s
-	kubectl wait --for=condition=Ready pod -l app=opentelemetry --timeout=180s
-	kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=prometheus --timeout=180s
-	@echo "🕐 Waiting for postgres service to be up..."
-	kubectl wait --for=condition=Ready pod -l app=postgres-client --timeout=180s
-	kubectl wait --for=condition=Ready pod -l app=postgres-server --timeout=180s
-
-.PHONY: update
-# Substitute all the binaries and config files inside the dockers
-# NOTE: kubectl cp will overwrite the previous files.
-update:
-	@echo "🔄 Updating binaries inside pods..."
-	for loader_pod in $$(kubectl get pods -n ${NAMESPACE} -l app=loader-agent -o custom-columns=:metadata.name --no-headers); do \
-			echo "=> Considering pod: $${loader_pod}"; \
-			echo "  [1/6] Copying process-agent binary..."; \
-			kubectl cp "../process-agent" ${NAMESPACE}/$${loader_pod}:"${DST_DIR}/process-agent"; \
-			echo "  [2/6] Copying process-agent config..."; \
-			kubectl cp "./process-agent-config.yaml" ${NAMESPACE}/$${loader_pod}:"${DST_DIR}/config.yaml"; \
-			echo "  [3/6] Creating directory for 'ebpf-object-files'..."; \
-			kubectl exec -n ${NAMESPACE} $${loader_pod} -- mkdir -p "${DST_DIR}/ebpf-object-files/"; \
-			echo "  [4/6] Copying 'ebpf-object-files/x86_64'..."; \
-			kubectl cp "../ebpf-object-files/x86_64" ${NAMESPACE}/$${loader_pod}:"${DST_DIR}/ebpf-object-files/"; \
-			echo "  [5/6] Copying test-server binary..."; \
-			kubectl cp "../test-server/test-server" ${NAMESPACE}/$${loader_pod}:"${DST_DIR}/test-server"; \
-			echo "  [5/6] Copying test-server config..."; \
-			kubectl cp "./test-server-config.json" ${NAMESPACE}/$${loader_pod}:"${DST_DIR}/config.json"; \
-	done; \
-	echo "🧹 Deleting pods..."; \
-	kubectl delete pods -n ${NAMESPACE} -l app=process-agent --force
-	kubectl delete pods -n ${NAMESPACE} -l app=test-server --force
-	@echo "🕐 Waiting for process-agent to be up..."
-	kubectl wait --for=condition=Ready pod -l app=process-agent --timeout=180s
-	@echo "🕐 Waiting for test-server to be up..."
-	kubectl wait --for=condition=Ready pod -l app=test-server --timeout=180s
-
-
-.PHONY: e2e-tests
-e2e-tests:
-	@echo "🚀 Running end-to-end tests..."
-	go test -tags 'docker kubelet kubeapiserver linux cri containerd linux_bpf k8s_e2e' ./tests -v
+deploy: deploy-prometheus deploy-otel-collector deploy-agent
+	@if [ "$(SETUP_TYPE)" = "local" ]; then $(MAKE) deploy-test-server; fi
+	@echo "✅ Completed deploy (mode=$(SETUP_TYPE))"
+
+.PHONY: redeploy-agent
+redeploy-agent:
+	@echo "🚀 Removing old agent chart..."
+	helm uninstall suse-observability-agent -n monitoring || true
+	@echo "🚀 Redeploying the process-agent ($(SETUP_TYPE))..."
+	$(MAKE) deploy-agent
+	@echo "✅ Completed re-deploy (mode=$(SETUP_TYPE))"
 
 # Obtain a local copy of the json output from the test server pod
 # useful only with legacy metrics
 .PHONY: take-output
 take-output:
-	@echo "📄 Dump output file..."; \
-	kubectl cp $$(kubectl get pods -l app=test-server -o custom-columns=:metadata.name --no-headers):output.json ./output.json \
-
-.PHONY: clean
-clean:
-	echo "🧹 Cleaning up..."; \
-	kubectl delete -f ./yaml/deployment.yaml -n $(NAMESPACE)
-	kubectl delete -f ./yaml/postgres.yaml -n $(NAMESPACE)
+	if [ "$(SETUP_TYPE)" = "remote" ]; then return; fi
+	@echo "📄 Dump output file..."
+	kubectl cp -n monitoring $$(kubectl get -n monitoring pods -l app=test-server -o custom-columns=:metadata.name --no-headers):output.json ./output.json
+
+############################
+# End-to-end tests targets #
+############################
+.PHONY: deploy-e2e-tests
+deploy-e2e-tests:
+	@echo "🚀 Deploying Postgres service..."
+	kubectl apply -f ./yaml/postgres.yaml
+	kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=postgres-server --timeout=$(SETUP_TIMEOUT)
+	kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=postgres-client --timeout=$(SETUP_TIMEOUT)
+
+.PHONY: run-e2e-tests
+run-e2e-tests:
+	@echo "🚀 Running end-to-end tests..."
+	go test -count=1 -tags 'docker kubelet kubeapiserver linux cri containerd linux_bpf k8s_e2e' ./tests -v
+
+.PHONY: clean-e2e-tests
+clean-e2e-tests:
+	kubectl delete -f ./yaml/postgres.yaml

k8s-deployment/README.md

@@ -2,30 +2,45 @@
 
 ## Goals of this folder
 
-1. Quickly iterate on the development of the process agent. No need of rebuilding docker images for every change, it is enough to restart existing pods with the new agent binaries.
-2. Run integration tests against the process agent. At the moment we have integration tests only for the agent exposing OTEL metrics.
+1. Quickly iterate on the development of the process agent deploying it in a local k8s cluster.
+2. Run integration tests against the process agent. At the moment we have integration tests only for OTEL metrics.
 
 ## 1. Deploy local agent binary into a k8s cluster
 
 These are the steps required to have a process agent up and running in a local Kubernetes cluster.
 
 ```bash
-# 1. Create a k8s cluster (kind, minikube, k3s, ecc...)
-minikube start --driver=kvm2
+# 1. Create a minikube k8s cluster
+make create-cluster
+
+# 2. Deploy the setup
+# Here we have 2 options:
+# 1. Option "local": 
+# - the agent will send legacy metrics to a local test server.
+# - the agent will send OTEL metrics to a local OTEL collector and then prometheus will scrape the OTEL collector
+# In this case, environment variables are not strictly necessary since they have default values in the Makefile, you can define them just to be sure you have a clean setup
+export SETUP_AGENT_HELM_CHART_DIR=<path_to_your_local_agent_helm_chart>
+export SETUP_TYPE=local
+export SETUP_RECEIVER_ENDPOINT=http://test-server-service:7077/stsAgent
+make deploy
 
-# 2. Deploy initial docker images (process-agent, loader-agent, test-server, OTEL collector, prometheus)
+# 2. Option "remote": 
+# - the agent will send legacy metrics to the platform receiver.
+# - the agent will send OTEL metrics to a local OTEL collector and then this collector will forward them to the platform OTEL endpoint.
+export SETUP_AGENT_HELM_CHART_DIR=<path_to_your_local_agent_helm_chart>
+export SETUP_TYPE=remote
+export SETUP_API_KEY=<your_stackstate_platform_api_key>
+export SETUP_CLUSTER_NAME=<your_cluster_name>
+export SETUP_RECEIVER_ENDPOINT=<your_stackstate_platform_endpoint>
 make deploy
 
-# 3. Build locally the process-agent and the test-server (or update configs)
-rake local_build
-# see how to build the process agent and ebpf artifacts in the project README.md
+# 3. If you need to push new changes to the agent binary, you can run redeploy-agent
+make redeploy-agent
 
-# 4. Push the new binaries/configs inside pods
-make update
+# 4. delete the cluster
+make delete-cluster
 ```
 
-Everytime you do some local changes to the process-agent and test-server code, or you change a config, you should rerun `make update`
-
 ### Legacy Metrics
 
 If you want to obtain a local copy of the json output from the test server pod you can run:
@@ -39,15 +54,16 @@ make take-output
 You can inspect OTEL metrics using Prometheus. The Prometheus UI can be accessed by through the NodePort service:
 
 ```bash
-PROMETHEUS_PORT=$(kubectl get services -A -o json | jq -r '.items[] | select(.metadata.name == "prometheus-service") | .spec.ports[] | select(.name == "web") | .nodePort')
-PROMETHEUS_ADDR=$(kubectl get nodes -A -o json | jq -r '.items[] | select(.metadata.name == "minikube") | .status.addresses[] | select(.type == "InternalIP") | .address')
-echo "Prometheus is available at: http://${PROMETHEUS_ADDR}:${PROMETHEUS_PORT}"
+PROMETHEUS_PORT=$(kubectl get --namespace open-telemetry -o jsonpath="{.spec.ports[0].nodePort}" services prometheus-server)
+PROMETHEUS_ADDR=$(kubectl get nodes --namespace open-telemetry -o jsonpath="{.items[0].status.addresses[0].address}")
+echo http://$PROMETHEUS_ADDR:$PROMETHEUS_PORT
 ```
 
 ## 2. Test OTEL metrics
 
 In order to run e2e tests you need the the cluster up and running with the agent already deployed
 
 ```bash
+kubectl apply -f ./yaml/postgres.yaml
 make e2e-tests
 ```

k8s-deployment/yaml/otel-collector-config-local.yaml

@@ -0,0 +1,61 @@
+mode: deployment
+image:
+  # we need the contrib image for the prometheus exporter
+  repository: "ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib"
+ports:
+  # These are internal metrics of the collector
+  metrics:
+    enabled: true
+    servicePort: 8888
+  # This is the port for the prometheus exporter we configure in the collector
+  prom-metrics:
+    enabled: true
+    containerPort: 9464
+    servicePort: 9464
+    protocol: TCP
+
+presets:
+  kubernetesAttributes:
+    enabled: true
+    extractAllPodLabels: true
+
+# This is the config file for the collector:
+config:
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+          endpoint: 0.0.0.0:4317
+        http:
+          endpoint: 0.0.0.0:4318
+  processors:
+    batch: {}
+    resource:
+      attributes:
+      - key: service.instance.id
+        from_attribute: k8s.pod.uid
+        action: insert
+        # Use the k8s namespace also as the open telemetry namespace
+      - key: service.namespace
+        from_attribute: k8s.namespace.name
+        action: insert
+  exporters:
+    debug:
+      verbosity: normal
+    prometheus:
+      endpoint: 0.0.0.0:9464
+    nop: {}
+  service:
+    pipelines:
+      traces:
+        receivers: [otlp]
+        processors: []
+        exporters: [nop]
+      metrics:
+        receivers: [otlp]
+        processors: [resource, batch]
+        exporters: [debug, prometheus]
+      logs:
+        receivers: [otlp]
+        processors: []
+        exporters: [nop]