From 7e04cb6fc472fbc5c620112e8731f083daf74706 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Mon, 16 Mar 2026 14:29:56 -0700 Subject: [PATCH 01/64] feat: scaffold helm umbrella chart for observability-stack Adds charts/observability-stack/ as an umbrella Helm chart using upstream dependencies: - opensearch 3.5.0 - opensearch-dashboards 3.5.0 - data-prepper 0.3.1 - opentelemetry-collector 0.147.0 - prometheus 28.13.0 values.yaml mirrors the existing docker-compose configuration. All 5 dependencies resolve and helm template renders successfully. Signed-off-by: Kyle Hounslow --- .gitignore | 2 + charts/observability-stack/.helmignore | 5 + charts/observability-stack/Chart.yaml | 47 ++++ .../observability-stack/templates/NOTES.txt | 22 ++ .../templates/_helpers.tpl | 31 +++ charts/observability-stack/values.yaml | 231 ++++++++++++++++++ 6 files changed, 338 insertions(+) create mode 100644 charts/observability-stack/.helmignore create mode 100644 charts/observability-stack/Chart.yaml create mode 100644 charts/observability-stack/templates/NOTES.txt create mode 100644 charts/observability-stack/templates/_helpers.tpl create mode 100644 charts/observability-stack/values.yaml diff --git a/.gitignore b/.gitignore index 2ec88dbe..24473f48 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ node_modules dist build .DS_Store +charts/observability-stack/charts/ +charts/observability-stack/Chart.lock diff --git a/charts/observability-stack/.helmignore b/charts/observability-stack/.helmignore new file mode 100644 index 00000000..a8aea9f3 --- /dev/null +++ b/charts/observability-stack/.helmignore @@ -0,0 +1,5 @@ +.DS_Store +.git +*.swp +*.bak +*.tmp diff --git a/charts/observability-stack/Chart.yaml b/charts/observability-stack/Chart.yaml new file mode 100644 index 00000000..4be4cf42 --- /dev/null +++ b/charts/observability-stack/Chart.yaml @@ -0,0 +1,47 @@ +apiVersion: v2 +name: observability-stack +description: OpenTelemetry-native observability platform for microservices, web apps, and AI agents +type: application +version: 0.1.0 +appVersion: "3.6.0" + +home: https://github.com/opensearch-project/observability-stack +sources: + - https://github.com/opensearch-project/observability-stack + +maintainers: + - name: kylehounslow + url: https://github.com/kylehounslow + +keywords: + - opensearch + - observability + - opentelemetry + - data-prepper + - agent-tracing + +dependencies: + - name: opensearch + version: "3.5.0" + repository: "https://opensearch-project.github.io/helm-charts/" + condition: opensearch.enabled + + - name: opensearch-dashboards + version: "3.5.0" + repository: "https://opensearch-project.github.io/helm-charts/" + condition: opensearch-dashboards.enabled + + - name: data-prepper + version: "0.3.1" + repository: "https://opensearch-project.github.io/helm-charts/" + condition: data-prepper.enabled + + - name: opentelemetry-collector + version: "0.147.0" + repository: "https://open-telemetry.github.io/opentelemetry-helm-charts" + condition: opentelemetry-collector.enabled + + - name: prometheus + version: "28.13.0" + repository: "https://prometheus-community.github.io/helm-charts" + condition: prometheus.enabled diff --git a/charts/observability-stack/templates/NOTES.txt b/charts/observability-stack/templates/NOTES.txt new file mode 100644 index 00000000..5747b3dd --- /dev/null +++ b/charts/observability-stack/templates/NOTES.txt @@ -0,0 +1,22 @@ +🚀 Observability Stack deployed! + +Components: +{{- if index .Values "opensearch" "enabled" }} + - OpenSearch: https://{{ .Release.Name }}-opensearch:9200 +{{- end }} +{{- if index .Values "opensearch-dashboards" "enabled" }} + - OpenSearch Dashboards: http://{{ .Release.Name }}-opensearch-dashboards:5601 +{{- end }} +{{- if index .Values "data-prepper" "enabled" }} + - Data Prepper: {{ .Release.Name }}-data-prepper:21890 +{{- end }} +{{- if index .Values "opentelemetry-collector" "enabled" }} + - OTel Collector (gRPC): {{ .Release.Name }}-opentelemetry-collector:4317 + - OTel Collector (HTTP): {{ .Release.Name }}-opentelemetry-collector:4318 +{{- end }} +{{- if index .Values "prometheus" "enabled" }} + - Prometheus: http://{{ .Release.Name }}-prometheus-server:9090 +{{- end }} + +Send telemetry to: + OTEL_EXPORTER_OTLP_ENDPOINT={{ .Release.Name }}-opentelemetry-collector:4317 diff --git a/charts/observability-stack/templates/_helpers.tpl b/charts/observability-stack/templates/_helpers.tpl new file mode 100644 index 00000000..f4bfe253 --- /dev/null +++ b/charts/observability-stack/templates/_helpers.tpl @@ -0,0 +1,31 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "observability-stack.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "observability-stack.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "observability-stack.labels" -}} +helm.sh/chart: {{ include "observability-stack.name" . }}-{{ .Chart.Version }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/part-of: observability-stack +{{- end }} diff --git a/charts/observability-stack/values.yaml b/charts/observability-stack/values.yaml new file mode 100644 index 00000000..d77cfc7c --- /dev/null +++ b/charts/observability-stack/values.yaml @@ -0,0 +1,231 @@ +# Default values for observability-stack umbrella chart +# Mirrors the docker-compose setup for Kubernetes deployment + +# -- OpenSearch +opensearch: + enabled: true + singleNode: true + replicas: 1 + resources: + requests: + memory: "2Gi" + cpu: "500m" + config: + opensearch.yml: | + plugins.query.datasources.encryption.masterkey: "BTqK4Ytdz67La1kShIKV3Pu9" + +# -- OpenSearch Dashboards +opensearch-dashboards: + enabled: true + replicas: 1 + opensearchHosts: "https://{{ .Release.Name }}-opensearch:9200" + config: + opensearch_dashboards.yml: | + server.host: "0.0.0.0" + opensearch.hosts: ["https://{{ .Release.Name }}-opensearch:9200"] + opensearch.ssl.verificationMode: none + opensearch.requestHeadersAllowlist: ["securitytenant", "Authorization"] + opensearch_security.multitenancy.enabled: false + opensearch_security.readonly_mode.roles: ["kibana_read_only"] + data_source.enabled: true + workspace.enabled: true + savedObjects.maxImportPayloadBytes: 10485760 + server.maxPayloadBytes: 10485760 + +# -- Data Prepper +data-prepper: + enabled: true + pipelineConfig: + enabled: true + config: + # Main OTLP pipeline - receives all telemetry and routes by type + otlp-pipeline: + delay: 10 + source: + otlp: + port: 21890 + ssl: false + route: + - logs: 'getEventType() == "LOG"' + - traces: 'getEventType() == "TRACE"' + sink: + - pipeline: + name: "otel-logs-pipeline" + routes: ["logs"] + - pipeline: + name: "otel-traces-pipeline" + routes: ["traces"] + + otel-logs-pipeline: + workers: 5 + delay: 10 + source: + pipeline: + name: "otlp-pipeline" + buffer: + bounded_blocking: {} + processor: + - copy_values: + entries: + - from_key: "time" + to_key: "@timestamp" + sink: + - opensearch: + hosts: ["https://{{ .Release.Name }}-opensearch:9200"] + username: "admin" + password: "{{ .Values.opensearchPassword }}" + insecure: true + index_type: log-analytics-plain + + otel-traces-pipeline: + delay: 100 + source: + pipeline: + name: "otlp-pipeline" + sink: + - pipeline: + name: "traces-raw-pipeline" + - pipeline: + name: "service-map-pipeline" + + traces-raw-pipeline: + source: + pipeline: + name: "otel-traces-pipeline" + processor: + - otel_traces: {} + sink: + - opensearch: + hosts: ["https://{{ .Release.Name }}-opensearch:9200"] + username: "admin" + password: "{{ .Values.opensearchPassword }}" + insecure: true + index_type: trace-analytics-plain-raw + + service-map-pipeline: + delay: 100 + source: + pipeline: + name: "otel-traces-pipeline" + processor: + - otel_apm_service_map: + group_by_attributes: [telemetry.sdk.language] + window_duration: 10s + route: + - otel_apm_service_map_route: 'getEventType() == "SERVICE_MAP"' + - service_processed_metrics: 'getEventType() == "METRIC"' + sink: + - opensearch: + hosts: ["https://{{ .Release.Name }}-opensearch:9200"] + username: "admin" + password: "{{ .Values.opensearchPassword }}" + index_type: otel-v2-apm-service-map + routes: [otel_apm_service_map_route] + insecure: true + - prometheus: + url: "http://{{ .Release.Name }}-prometheus-server:9090/api/v1/write" + threshold: + max_events: 500 + flush_interval: 5s + routes: [service_processed_metrics] + +# -- OpenTelemetry Collector +opentelemetry-collector: + enabled: true + mode: deployment + image: + repository: "otel/opentelemetry-collector-contrib" + config: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + cors: + allowed_origins: + - "http://*" + - "https://*" + processors: + memory_limiter: + check_interval: 5s + limit_percentage: 80 + spike_limit_percentage: 25 + batch: + timeout: 10s + send_batch_size: 1024 + resourcedetection: + detectors: [env] + transform: + error_mode: ignore + trace_statements: + - context: span + statements: + - replace_pattern(name, "\\?.*", "") + # Workaround for https://github.com/opensearch-project/data-prepper/issues/5616 + - set(attributes["error_type"], attributes["error.type"]) where attributes["error.type"] != nil + - delete_key(attributes, "error.type") + log_statements: + - context: log + statements: + - flatten(body) where IsMap(body) + - set(body, ToKeyValueString(body)) where IsMap(body) + exporters: + debug: + verbosity: detailed + otlp/opensearch: + endpoint: "{{ .Release.Name }}-data-prepper:21890" + tls: + insecure: true + otlphttp/prometheus: + endpoint: "http://{{ .Release.Name }}-prometheus-server:9090/api/v1/otlp" + tls: + insecure: true + service: + pipelines: + traces: + receivers: [otlp] + processors: [resourcedetection, memory_limiter, transform, batch] + exporters: [otlp/opensearch, debug] + metrics: + receivers: [otlp] + processors: [resourcedetection, memory_limiter, batch] + exporters: [otlphttp/prometheus, debug] + logs: + receivers: [otlp] + processors: [resourcedetection, memory_limiter, transform, batch] + exporters: [otlp/opensearch, debug] + + ports: + otlp: + enabled: true + containerPort: 4317 + servicePort: 4317 + protocol: TCP + otlp-http: + enabled: true + containerPort: 4318 + servicePort: 4318 + protocol: TCP + +# -- Prometheus +prometheus: + enabled: true + server: + persistentVolume: + enabled: false + extraFlags: + - "web.enable-remote-write-receiver" + - "web.enable-otlp-receiver" + alertmanager: + enabled: false + kube-state-metrics: + enabled: false + prometheus-node-exporter: + enabled: false + prometheus-pushgateway: + enabled: false + +# -- Global settings +opensearchPassword: "My_password_123!@#" From 65baa718719cd453eea17fdf1dd25265b2542de7 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Mon, 16 Mar 2026 15:36:34 -0700 Subject: [PATCH 02/64] fix: working helm values for kind deployment - Add OPENSEARCH_INITIAL_ADMIN_PASSWORD env var for OpenSearch 3.5+ - Override Data Prepper image to latest (chart default 2.8.0 lacks otlp source) - Explicitly disable SSL on Data Prepper server and peer_forwarder - Fix service name references for inter-component connectivity - Use otel/opentelemetry-collector-contrib image (required by chart) Validated: all 5 pods running on kind + finch (macOS arm64) Signed-off-by: Kyle Hounslow --- charts/observability-stack/values.yaml | 44 ++++++++++++++++---------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/charts/observability-stack/values.yaml b/charts/observability-stack/values.yaml index d77cfc7c..cd8724d6 100644 --- a/charts/observability-stack/values.yaml +++ b/charts/observability-stack/values.yaml @@ -10,6 +10,9 @@ opensearch: requests: memory: "2Gi" cpu: "500m" + extraEnvs: + - name: OPENSEARCH_INITIAL_ADMIN_PASSWORD + value: "My_password_123!@#" config: opensearch.yml: | plugins.query.datasources.encryption.masterkey: "BTqK4Ytdz67La1kShIKV3Pu9" @@ -18,11 +21,11 @@ opensearch: opensearch-dashboards: enabled: true replicas: 1 - opensearchHosts: "https://{{ .Release.Name }}-opensearch:9200" + opensearchHosts: "https://opensearch-cluster-master:9200" config: opensearch_dashboards.yml: | server.host: "0.0.0.0" - opensearch.hosts: ["https://{{ .Release.Name }}-opensearch:9200"] + opensearch.hosts: ["https://opensearch-cluster-master:9200"] opensearch.ssl.verificationMode: none opensearch.requestHeadersAllowlist: ["securitytenant", "Authorization"] opensearch_security.multitenancy.enabled: false @@ -35,6 +38,21 @@ opensearch-dashboards: # -- Data Prepper data-prepper: enabled: true + image: + tag: "latest" + config: + data-prepper-config.yaml: | + ssl: false + peer_forwarder: + ssl: false + livenessProbe: + initialDelaySeconds: 30 + periodSeconds: 10 + failureThreshold: 10 + readinessProbe: + initialDelaySeconds: 30 + periodSeconds: 10 + failureThreshold: 10 pipelineConfig: enabled: true config: @@ -71,9 +89,9 @@ data-prepper: to_key: "@timestamp" sink: - opensearch: - hosts: ["https://{{ .Release.Name }}-opensearch:9200"] + hosts: ["https://opensearch-cluster-master:9200"] username: "admin" - password: "{{ .Values.opensearchPassword }}" + password: "My_password_123!@#" insecure: true index_type: log-analytics-plain @@ -96,9 +114,9 @@ data-prepper: - otel_traces: {} sink: - opensearch: - hosts: ["https://{{ .Release.Name }}-opensearch:9200"] + hosts: ["https://opensearch-cluster-master:9200"] username: "admin" - password: "{{ .Values.opensearchPassword }}" + password: "My_password_123!@#" insecure: true index_type: trace-analytics-plain-raw @@ -116,18 +134,12 @@ data-prepper: - service_processed_metrics: 'getEventType() == "METRIC"' sink: - opensearch: - hosts: ["https://{{ .Release.Name }}-opensearch:9200"] + hosts: ["https://opensearch-cluster-master:9200"] username: "admin" - password: "{{ .Values.opensearchPassword }}" + password: "My_password_123!@#" index_type: otel-v2-apm-service-map routes: [otel_apm_service_map_route] insecure: true - - prometheus: - url: "http://{{ .Release.Name }}-prometheus-server:9090/api/v1/write" - threshold: - max_events: 500 - flush_interval: 5s - routes: [service_processed_metrics] # -- OpenTelemetry Collector opentelemetry-collector: @@ -175,11 +187,11 @@ opentelemetry-collector: debug: verbosity: detailed otlp/opensearch: - endpoint: "{{ .Release.Name }}-data-prepper:21890" + endpoint: "obs-stack-data-prepper:21890" tls: insecure: true otlphttp/prometheus: - endpoint: "http://{{ .Release.Name }}-prometheus-server:9090/api/v1/otlp" + endpoint: "http://obs-stack-prometheus-server:80/api/v1/otlp" tls: insecure: true service: From f9c6718c68db17b6bcafc66c838ca6ff03a89720 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Mon, 16 Mar 2026 15:48:39 -0700 Subject: [PATCH 03/64] fix: add OpenSearch credentials to OSD config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OSD was failing to authenticate to OpenSearch (401 No Authorization header). Added opensearch.username and opensearch.password to opensearch_dashboards.yml. End-to-end pipeline validated: curl → OTel Collector → Data Prepper → OpenSearch ✅ OSD UI accessible on port-forward ✅ TODO: centralize credentials via .env-style values (not hardcoded) Signed-off-by: Kyle Hounslow --- charts/observability-stack/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/observability-stack/values.yaml b/charts/observability-stack/values.yaml index cd8724d6..160dfa81 100644 --- a/charts/observability-stack/values.yaml +++ b/charts/observability-stack/values.yaml @@ -27,6 +27,8 @@ opensearch-dashboards: server.host: "0.0.0.0" opensearch.hosts: ["https://opensearch-cluster-master:9200"] opensearch.ssl.verificationMode: none + opensearch.username: "admin" + opensearch.password: "My_password_123!@#" opensearch.requestHeadersAllowlist: ["securitytenant", "Authorization"] opensearch_security.multitenancy.enabled: false opensearch_security.readonly_mode.roles: ["kibana_read_only"] From 26e49170ded77d2d9aab1cdfa709e7bd142e6f10 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Mon, 16 Mar 2026 15:57:36 -0700 Subject: [PATCH 04/64] feat: add init job for dashboards, index patterns, and saved objects Helm post-install/post-upgrade hook that runs the existing init-opensearch-dashboards.py script as a K8s Job. Creates: workspace, index patterns (logs/traces/service-map), trace-to-logs correlation, APM config, agent observability dashboard, overview dashboard, and saved queries. Script patched to read BASE_URL from env var for K8s service names. Validated: job completes in ~30s, all saved objects created. Signed-off-by: Kyle Hounslow --- .../files/init-opensearch-dashboards.py | 1202 +++++++++++++++++ .../templates/init-dashboards-configmap.yaml | 15 + .../templates/init-dashboards-job.yaml | 46 + 3 files changed, 1263 insertions(+) create mode 100644 charts/observability-stack/files/init-opensearch-dashboards.py create mode 100644 charts/observability-stack/templates/init-dashboards-configmap.yaml create mode 100644 charts/observability-stack/templates/init-dashboards-job.yaml diff --git a/charts/observability-stack/files/init-opensearch-dashboards.py b/charts/observability-stack/files/init-opensearch-dashboards.py new file mode 100644 index 00000000..4623f0f2 --- /dev/null +++ b/charts/observability-stack/files/init-opensearch-dashboards.py @@ -0,0 +1,1202 @@ +#!/usr/bin/env python3 + +import os +import time +import requests +import yaml + +BASE_URL = os.getenv("BASE_URL", "http://opensearch-dashboards:5601") +USERNAME = os.getenv("OPENSEARCH_USER", "admin") +PASSWORD = os.getenv("OPENSEARCH_PASSWORD", "My_password_123!@#") +PROMETHEUS_HOST = os.getenv("PROMETHEUS_HOST", "prometheus") +PROMETHEUS_PORT = os.getenv("PROMETHEUS_PORT", "9090") + +def wait_for_dashboards(): + """Wait for OpenSearch Dashboards to be ready""" + print("🔄 Initializing OpenSearch workspace...") + + while True: + try: + response = requests.get( + f"{BASE_URL}/api/status", auth=(USERNAME, PASSWORD), timeout=5 + ) + if response.status_code == 200: + break + except requests.exceptions.RequestException: + pass + + print("⏳ Waiting for OpenSearch Dashboards...") + time.sleep(5) + +def get_existing_workspace(): + """Check if Observability Stack workspace already exists""" + try: + response = requests.post( + f"{BASE_URL}/api/workspaces/_list", + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + json={}, + verify=False, + timeout=10, + ) + print(f"Workspace list response: {response.status_code}") + if response.status_code == 200: + result = response.json() + if result.get("success"): + workspaces = result.get("result", {}).get("workspaces", []) + for workspace in workspaces: + if workspace.get("name") == "Observability Stack": + return workspace.get("id") + elif response.status_code == 404: + print("⚠️ Workspace API not available - workspaces may not be supported in this version") + return None + except requests.exceptions.RequestException as e: + print(f"⚠️ Error checking workspaces: {e}") + return None + +def create_workspace(): + """Create new Observability Stack workspace""" + print("🏗️ Creating Observability Stack workspace...") + + payload = { + "attributes": { + "name": "Observability Stack", + "description": "AI Agent observability workspace with logs, traces, and metrics", + "features": ["use-case-observability"] + } + } + + try: + response = requests.post( + f"{BASE_URL}/api/workspaces", + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + json=payload, + verify=False, + timeout=10, + ) + + print(f"Create workspace response: {response.status_code}") + if response.status_code == 200: + result = response.json() + if result.get("success"): + workspace_id = result.get("result", {}).get("id") + if workspace_id: + print(f"✅ Created workspace: {workspace_id}") + return workspace_id + elif response.status_code == 404: + print("⚠️ Workspace API not available - using default dashboard") + return "default" + else: + print(f"⚠️ Workspace creation failed: {response.text}") + return "default" + except requests.exceptions.RequestException as e: + print(f"⚠️ Error creating workspace: {e}") + return "default" + + +def get_existing_index_pattern(workspace_id, title): + """Check if an index pattern with the given title already exists""" + try: + # Use workspace-specific URL if workspace exists + if workspace_id and workspace_id != "default": + url = f"{BASE_URL}/w/{workspace_id}/api/saved_objects/_find?type=index-pattern&search_fields=title&search={title}" + else: + url = f"{BASE_URL}/api/saved_objects/_find?type=index-pattern&search_fields=title&search={title}" + + response = requests.get( + url, + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + verify=False, + timeout=10, + ) + + if response.status_code == 200: + result = response.json() + saved_objects = result.get("saved_objects", []) + for obj in saved_objects: + attributes = obj.get("attributes", {}) + # Exact match on title + if attributes.get("title") == title: + return obj.get("id") + return None + except requests.exceptions.RequestException as e: + print(f"⚠️ Error checking existing index pattern {title}: {e}") + return None + + +def create_index_pattern( + workspace_id, title, time_field=None, signal_type=None, schema_mappings=None, + display_name=None +): + """Create index pattern in workspace and return its ID""" + # Check if index pattern already exists + existing_id = get_existing_index_pattern(workspace_id, title) + if existing_id: + print(f"✅ Index pattern already exists: {title}") + return existing_id + + payload = { + "attributes": { + "title": title + } + } + + if time_field: + payload["attributes"]["timeFieldName"] = time_field + if signal_type: + payload["attributes"]["signalType"] = signal_type + if schema_mappings: + payload["attributes"]["schemaMappings"] = schema_mappings + if display_name: + payload["attributes"]["displayName"] = display_name + + # Use workspace-specific URL if workspace exists, otherwise use default + if workspace_id and workspace_id != "default": + url = f"{BASE_URL}/w/{workspace_id}/api/saved_objects/index-pattern" + else: + url = f"{BASE_URL}/api/saved_objects/index-pattern" + + try: + response = requests.post( + url, + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + json=payload, + verify=False, + timeout=10, + ) + print(f"Index pattern {title} creation: {response.status_code}") + + if response.status_code == 200: + result = response.json() + pattern_id = result.get("id") + if pattern_id: + print(f"✅ Created index pattern: {title}") + return pattern_id + return None + except requests.exceptions.RequestException as e: + print(f"⚠️ Error creating index pattern {title}: {e}") + return None + + +def get_existing_prometheus_datasource(datasource_name): + """Check if a Prometheus datasource with the given name already exists""" + try: + response = requests.get( + f"{BASE_URL}/api/saved_objects/_find?per_page=10000&type=data-connection", + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + verify=False, + timeout=10, + ) + + if response.status_code == 200: + result = response.json() + saved_objects = result.get("saved_objects", []) + for obj in saved_objects: + attributes = obj.get("attributes", {}) + if attributes.get("connectionId") == datasource_name: + return obj.get("id") + elif response.status_code == 404: + # List endpoint not available + return None + return None + except requests.exceptions.RequestException as e: + print(f"⚠️ Error checking existing Prometheus datasources: {e}") + return None + + +def create_prometheus_datasource(workspace_id): + """Create Prometheus datasource using direct query API""" + datasource_name = "ObservabilityStack_Prometheus" + + # Check if datasource already exists + existing_id = get_existing_prometheus_datasource(datasource_name) + if existing_id: + print(f"✅ Prometheus datasource already exists: {existing_id}") + # Associate with workspace if provided + if workspace_id and workspace_id != "default": + associate_prometheus_with_workspace(workspace_id, existing_id) + return existing_id + + print("🔧 Creating Prometheus datasource...") + + prometheus_endpoint = f"http://{PROMETHEUS_HOST}:{PROMETHEUS_PORT}" + + payload = { + "name": datasource_name, + "allowedRoles": [], + "connector": "prometheus", + "properties": { + "prometheus.uri": prometheus_endpoint, + "prometheus.auth.type": "basicauth", + "prometheus.auth.username": "", + "prometheus.auth.password": "", + }, + } + + try: + response = requests.post( + f"{BASE_URL}/api/directquery/dataconnections", + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + json=payload, + verify=False, + timeout=10, + ) + + print(f"Prometheus datasource creation: {response.status_code}") + + if response.status_code == 200: + print(f"✅ Created Prometheus datasource: {datasource_name}") + + # Fetch the datasource ID from saved objects + datasource_id = get_existing_prometheus_datasource(datasource_name) + if datasource_id and workspace_id and workspace_id != "default": + associate_prometheus_with_workspace(workspace_id, datasource_id) + + return datasource_name + elif response.status_code == 400: + # Check if error is due to duplicate + error_text = response.text + if "already exists with name" in error_text: + print(f"✅ Prometheus datasource already exists: {datasource_name}") + # Fetch the datasource ID and associate + datasource_id = get_existing_prometheus_datasource(datasource_name) + if datasource_id and workspace_id and workspace_id != "default": + associate_prometheus_with_workspace(workspace_id, datasource_id) + return datasource_name + else: + print(f"⚠️ Prometheus datasource creation failed: {error_text}") + return None + else: + print(f"⚠️ Prometheus datasource creation failed: {response.text}") + return None + except requests.exceptions.RequestException as e: + print(f"⚠️ Error creating Prometheus datasource: {e}") + return None + + +def associate_prometheus_with_workspace(workspace_id, datasource_id): + """Associate Prometheus datasource with workspace""" + print(f"🔗 Associating Prometheus datasource with workspace {workspace_id}...") + + payload = { + "workspaceId": workspace_id, + "savedObjects": [{"type": "data-connection", "id": datasource_id}], + } + + try: + response = requests.post( + f"{BASE_URL}/api/workspaces/_associate", + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + json=payload, + verify=False, + timeout=10, + ) + + print(f"Prometheus datasource association: {response.status_code}") + + if response.status_code == 200: + print("✅ Prometheus datasource associated with workspace") + else: + print(f"⚠️ Association failed: {response.text}") + except requests.exceptions.RequestException as e: + print(f"⚠️ Error associating Prometheus datasource: {e}") + + +def associate_datasource_with_workspace(workspace_id, datasource_id): + """Associate datasource with workspace""" + print(f"🔗 Associating datasource {datasource_id} with workspace {workspace_id}...") + + payload = { + "workspaceId": workspace_id, + "savedObjects": [{"type": "data-source", "id": datasource_id}], + } + + try: + response = requests.post( + f"{BASE_URL}/api/workspaces/_associate", + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + json=payload, + verify=False, + timeout=10, + ) + + print(f"Datasource association: {response.status_code}") + + if response.status_code == 200: + print("✅ Datasource associated with workspace") + else: + print(f"⚠️ Association failed: {response.text}") + except requests.exceptions.RequestException as e: + print(f"⚠️ Error associating datasource: {e}") + + +def get_existing_opensearch_datasource(datasource_title): + """Check if OpenSearch datasource already exists""" + try: + response = requests.get( + f"{BASE_URL}/api/saved_objects/_find?per_page=10000&type=data-source", + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + verify=False, + timeout=10, + ) + + if response.status_code == 200: + result = response.json() + saved_objects = result.get("saved_objects", []) + for obj in saved_objects: + attributes = obj.get("attributes", {}) + if attributes.get("title") == datasource_title: + return obj.get("id") + return None + except requests.exceptions.RequestException as e: + print(f"⚠️ Error checking existing OpenSearch datasources: {e}") + return None + + +def create_opensearch_datasource(workspace_id): + """Create OpenSearch datasource from local cluster""" + datasource_title = "local_cluster" + + # Check if datasource already exists + existing_id = get_existing_opensearch_datasource(datasource_title) + if existing_id: + print(f"✅ OpenSearch datasource already exists: {existing_id}") + # Associate with workspace if provided + if workspace_id and workspace_id != "default": + associate_datasource_with_workspace(workspace_id, existing_id) + return existing_id + + print("🔧 Creating OpenSearch datasource...") + + opensearch_endpoint = "https://opensearch:9200" + + payload = { + "attributes": { + "title": datasource_title, + "description": "Local OpenSearch cluster", + "endpoint": opensearch_endpoint, + "auth": { + "type": "username_password", + "credentials": {"username": USERNAME, "password": PASSWORD}, + }, + "dataSourceVersion": "3.5.0", + "dataSourceEngineType": "OpenSearch", + } + } + + try: + response = requests.post( + f"{BASE_URL}/api/saved_objects/data-source", + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + json=payload, + verify=False, + timeout=10, + ) + + print(f"OpenSearch datasource creation: {response.status_code}") + + if response.status_code == 200: + result = response.json() + datasource_id = result.get("id") + if datasource_id: + print(f"✅ Created OpenSearch datasource: {datasource_id}") + + # Associate with workspace if provided + if workspace_id and workspace_id != "default": + associate_datasource_with_workspace(workspace_id, datasource_id) + return datasource_id + else: + print(f"⚠️ OpenSearch datasource creation failed: {response.text}") + return None + except requests.exceptions.RequestException as e: + print(f"⚠️ Error creating OpenSearch datasource: {e}") + return None + + +def set_default_index_pattern(workspace_id, pattern_id): + """Set the default index pattern""" + print(f"⭐ Setting default index pattern: {pattern_id}") + + # Use workspace-specific URL if workspace exists, otherwise use default + if workspace_id and workspace_id != "default": + url = f"{BASE_URL}/w/{workspace_id}/api/opensearch-dashboards/settings/defaultIndex" + else: + url = f"{BASE_URL}/api/opensearch-dashboards/settings/defaultIndex" + + payload = {"value": pattern_id} + + try: + response = requests.post( + url, + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + json=payload, + verify=False, + timeout=10, + ) + + print(f"Set default index pattern: {response.status_code}") + + if response.status_code == 200: + print("✅ Default index pattern set to logs-otel-v1-*") + else: + print(f"⚠️ Setting default failed: {response.text}") + except requests.exceptions.RequestException as e: + print(f"⚠️ Error setting default index pattern: {e}") + + +def get_existing_correlation(workspace_id, correlation_type_prefix): + """Check if a correlation with the given type prefix already exists""" + try: + if workspace_id and workspace_id != "default": + url = f"{BASE_URL}/w/{workspace_id}/api/saved_objects/_find?type=correlations" + else: + url = f"{BASE_URL}/api/saved_objects/_find?type=correlations" + + response = requests.get( + url, + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + verify=False, + timeout=10, + ) + + if response.status_code == 200: + result = response.json() + saved_objects = result.get("saved_objects", []) + for obj in saved_objects: + attributes = obj.get("attributes", {}) + ct = attributes.get("correlationType", "") + if ct.startswith(correlation_type_prefix): + return obj.get("id") + return None + except requests.exceptions.RequestException as e: + print(f"⚠️ Error checking existing correlation: {e}") + return None + + +def create_correlation(workspace_id, correlation_type, title, entities, references): + """Create a correlation saved object (idempotent)""" + # Determine prefix for existence check (APM-Config- or trace-to-logs-) + prefix = correlation_type.split("-")[0] + "-" + correlation_type.split("-")[1] if "-" in correlation_type else correlation_type + existing_id = get_existing_correlation(workspace_id, prefix) + if existing_id: + print(f"✅ Correlation already exists ({prefix}*): {existing_id}") + return existing_id + + print(f"🔗 Creating correlation: {title}...") + + payload = { + "attributes": { + "correlationType": correlation_type, + "title": title, + "version": "1.0.0", + "entities": entities, + }, + "references": references, + } + + if workspace_id and workspace_id != "default": + payload["workspaces"] = [workspace_id] + url = f"{BASE_URL}/w/{workspace_id}/api/saved_objects/correlations" + else: + url = f"{BASE_URL}/api/saved_objects/correlations" + + try: + response = requests.post( + url, + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + json=payload, + verify=False, + timeout=10, + ) + + if response.status_code == 200: + result = response.json() + correlation_id = result.get("id") + print(f"✅ Created correlation: {title} ({correlation_id})") + return correlation_id + else: + print(f"⚠️ Correlation creation failed: {response.text}") + return None + except requests.exceptions.RequestException as e: + print(f"⚠️ Error creating correlation: {e}") + return None + + +def create_trace_to_logs_correlation(workspace_id, traces_pattern_id, logs_pattern_id): + """Create trace-to-logs correlation for cross-signal navigation""" + return create_correlation( + workspace_id, + correlation_type=f"trace-to-logs-otel-v1-apm-span*", + title="trace-to-logs_otel-v1-apm-span*", + entities=[ + {"tracesDataset": {"id": "references[0].id"}}, + {"logsDataset": {"id": "references[1].id"}}, + ], + references=[ + {"name": "entities[0].index", "type": "index-pattern", "id": traces_pattern_id}, + {"name": "entities[1].index", "type": "index-pattern", "id": logs_pattern_id}, + ], + ) + + +def create_apm_config_correlation(workspace_id, traces_pattern_id, service_map_pattern_id, prometheus_datasource_id): + """Create APM config correlation that ties traces, service map, and Prometheus together""" + if not prometheus_datasource_id: + print("⚠️ Skipping APM config - no Prometheus datasource ID") + return None + + return create_correlation( + workspace_id, + correlation_type=f"APM-Config-{workspace_id}", + title="apm-config", + entities=[ + {"tracesDataset": {"id": "references[0].id"}}, + {"serviceMapDataset": {"id": "references[1].id"}}, + {"prometheusDataSource": {"id": "references[2].id"}}, + ], + references=[ + {"name": "entities[0].index", "type": "index-pattern", "id": traces_pattern_id}, + {"name": "entities[1].index", "type": "index-pattern", "id": service_map_pattern_id}, + {"name": "entities[2].dataConnection", "type": "data-connection", "id": prometheus_datasource_id}, + ], + ) + + +def create_or_update_saved_query( + workspace_id, query_id, title, description, query_string, language="PPL" +): + """Create or update a saved query in the workspace""" + print(f"💾 Creating/updating saved query: {title}...") + + # Base attributes for both create and update + base_attributes = { + "title": title, + "description": description, + "query": {"query": query_string, "language": language}, + } + + # Set URL based on workspace + if workspace_id and workspace_id != "default": + url = f"{BASE_URL}/w/{workspace_id}/api/saved_objects/query/{query_id}" + else: + url = f"{BASE_URL}/api/saved_objects/query/{query_id}" + + try: + # Try POST first (create) - includes workspaces field + create_payload = {"attributes": base_attributes} + if workspace_id and workspace_id != "default": + create_payload["workspaces"] = [workspace_id] + + response = requests.post( + url, + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + json=create_payload, + verify=False, + timeout=10, + ) + + if response.status_code == 200: + print(f"✅ Created saved query: {title}") + return query_id + elif response.status_code == 409: + # Query exists, update it with PUT - only attributes, no workspaces field + print(f"🔄 Query exists, updating: {title}") + update_payload = {"attributes": base_attributes} + + response = requests.put( + url, + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + json=update_payload, + verify=False, + timeout=10, + ) + + if response.status_code == 200: + print(f"✅ Updated saved query: {title}") + return query_id + else: + print(f"⚠️ Saved query update failed: {response.text}") + return None + else: + print(f"⚠️ Saved query creation failed: {response.text}") + return None + except requests.exceptions.RequestException as e: + print(f"⚠️ Error creating/updating saved query: {e}") + return None + + +def create_default_saved_queries(workspace_id): + """Create a collection of useful saved queries for agent observability""" + print("📝 Creating saved queries...") + + import glob + + # Load all saved-queries-*.yaml files + queries_files = glob.glob("/config/saved-queries-*.yaml") + + if not queries_files: + print("⚠️ No saved-queries-*.yaml files found") + return 0 + + total_created = 0 + for queries_file in sorted(queries_files): + print(f"📄 Loading {os.path.basename(queries_file)}...") + try: + with open(queries_file, "r") as f: + config = yaml.safe_load(f) + queries = config.get("queries", []) + except yaml.YAMLError as e: + print(f"⚠️ Error parsing {queries_file}: {e}") + continue + + if not queries: + print(f"⚠️ No queries found in {queries_file}") + continue + + for query_def in queries: + result = create_or_update_saved_query( + workspace_id, + query_def.get("id"), + query_def.get("title"), + query_def.get("description"), + query_def.get("query"), + query_def.get("language", "PPL"), + ) + if result: + total_created += 1 + + print(f"✅ Processed {total_created} saved queries from {len(queries_files)} file(s)") + return total_created + + +def get_existing_dashboard(workspace_id, dashboard_id): + """Check if dashboard already exists""" + try: + if workspace_id and workspace_id != "default": + url = f"{BASE_URL}/w/{workspace_id}/api/saved_objects/dashboard/{dashboard_id}" + else: + url = f"{BASE_URL}/api/saved_objects/dashboard/{dashboard_id}" + + response = requests.get( + url, + auth=(USERNAME, PASSWORD), + headers={"osd-xsrf": "true"}, + verify=False, + timeout=10, + ) + return response.status_code == 200 + except requests.exceptions.RequestException: + return False + + +def set_default_dashboard(workspace_id, dashboard_id): + """Set the default dashboard for the observability overview page""" + print(f"⭐ Setting default dashboard: {dashboard_id}") + + if workspace_id and workspace_id != "default": + url = f"{BASE_URL}/w/{workspace_id}/api/opensearch-dashboards/settings" + else: + url = f"{BASE_URL}/api/opensearch-dashboards/settings" + + payload = {"changes": {"observability:defaultDashboard": dashboard_id}} + + try: + response = requests.post( + url, + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + json=payload, + verify=False, + timeout=10, + ) + + if response.status_code == 200: + print("✅ Default dashboard set") + else: + print(f"⚠️ Setting default dashboard failed: {response.text}") + except requests.exceptions.RequestException as e: + print(f"⚠️ Error setting default dashboard: {e}") + + +def create_agent_observability_dashboard(workspace_id, traces_pattern_id): + """Create or update Agent Observability dashboard with visualizations""" + import json + + dashboard_id = "agent-observability-dashboard" + dashboard_exists = get_existing_dashboard(workspace_id, dashboard_id) + + if dashboard_exists: + print("📊 Updating Agent Observability dashboard...") + else: + print("📊 Creating Agent Observability dashboard...") + + # Visualizations based on last 5 queries from saved-queries-traces.yaml + visualizations = [ + { + "id": "llm-requests-by-model", + "title": "LLM Requests by Model", + "type": "pie", + "field": "attributes.gen_ai.request.model" + }, + { + "id": "tool-usage-stats", + "title": "Tool Usage Statistics", + "type": "pie", + "field": "attributes.gen_ai.tool.name" + }, + { + "id": "token-usage-by-agent", + "title": "Token Usage by Agent", + "type": "horizontal_bar", + "field": "attributes.gen_ai.agent.name", + "metric_field": "attributes.gen_ai.usage.input_tokens" + }, + { + "id": "token-usage-by-model", + "title": "Token Usage by Model", + "type": "horizontal_bar", + "field": "attributes.gen_ai.request.model", + "metric_field": "attributes.gen_ai.usage.input_tokens" + }, + { + "id": "agent-operations-by-service", + "title": "Agent Operations by Service", + "type": "horizontal_bar", + "field": "serviceName", + "split_field": "attributes.gen_ai.operation.name" + } + ] + + created_vis_ids = [] + for vis in visualizations: + vis_id = create_chart_visualization( + workspace_id, vis["id"], vis["title"], vis["type"], + vis["field"], traces_pattern_id, + metric_field=vis.get("metric_field"), + split_field=vis.get("split_field") + ) + if vis_id: + created_vis_ids.append(vis_id) + print(f" ✅ Created visualization: {vis['title']}") + + if not created_vis_ids: + print("⚠️ No visualizations created, skipping dashboard") + return None + + # Create dashboard with panels + panels = [] + references = [] + for i, vis_id in enumerate(created_vis_ids): + panels.append({ + "version": "3.5.0", + "gridData": {"x": (i % 2) * 24, "y": (i // 2) * 15, "w": 24, "h": 15, "i": str(i)}, + "panelIndex": str(i), + "embeddableConfig": {}, + "panelRefName": f"panel_{i}" + }) + references.append({"name": f"panel_{i}", "type": "visualization", "id": vis_id}) + + if workspace_id and workspace_id != "default": + url = f"{BASE_URL}/w/{workspace_id}/api/saved_objects/dashboard/{dashboard_id}" + else: + url = f"{BASE_URL}/api/saved_objects/dashboard/{dashboard_id}" + + payload = { + "attributes": { + "title": "Agent Observability", + "description": "Overview of AI agent performance, token usage, and tool execution", + "panelsJSON": json.dumps(panels), + "optionsJSON": json.dumps({"useMargins": True, "hidePanelTitles": False}), + "timeRestore": False, + "kibanaSavedObjectMeta": { + "searchSourceJSON": json.dumps({"query": {"query": "", "language": "kuery"}, "filter": []}) + } + }, + "references": references + } + + if workspace_id and workspace_id != "default": + payload["workspaces"] = [workspace_id] + + try: + response = requests.post( + url, + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + json=payload, + verify=False, + timeout=10, + ) + + if response.status_code == 200: + print(f"✅ Created Agent Observability dashboard") + set_default_dashboard(workspace_id, dashboard_id) + return dashboard_id + elif response.status_code == 409: + # Dashboard exists, update it with PUT + print("🔄 Dashboard exists, updating...") + update_payload = {"attributes": payload["attributes"], "references": references} + response = requests.put( + url, + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + json=update_payload, + verify=False, + timeout=10, + ) + if response.status_code == 200: + print(f"✅ Updated Agent Observability dashboard") + set_default_dashboard(workspace_id, dashboard_id) + return dashboard_id + else: + print(f"⚠️ Dashboard update failed: {response.text}") + return None + else: + print(f"⚠️ Dashboard creation failed: {response.text}") + return None + except requests.exceptions.RequestException as e: + print(f"⚠️ Error creating dashboard: {e}") + return None + + +def create_chart_visualization(workspace_id, vis_id, title, vis_type, field, index_pattern_id, + metric_field=None, split_field=None): + """Create a chart visualization (pie, bar, etc.)""" + import json + + if workspace_id and workspace_id != "default": + url = f"{BASE_URL}/w/{workspace_id}/api/saved_objects/visualization/{vis_id}" + else: + url = f"{BASE_URL}/api/saved_objects/visualization/{vis_id}" + + # Build aggregations + aggs = [] + if metric_field: + aggs.append({"id": "1", "type": "sum", "schema": "metric", "params": {"field": metric_field}}) + else: + aggs.append({"id": "1", "type": "count", "schema": "metric"}) + + aggs.append({"id": "2", "type": "terms", "schema": "segment", "params": {"field": field, "size": 10}}) + + if split_field: + aggs.append({"id": "3", "type": "terms", "schema": "group", "params": {"field": split_field, "size": 5}}) + + vis_state = { + "title": title, + "type": vis_type, + "params": {"type": vis_type, "addTooltip": True, "addLegend": True}, + "aggs": aggs + } + + payload = { + "attributes": { + "title": title, + "visState": json.dumps(vis_state), + "uiStateJSON": "{}", + "kibanaSavedObjectMeta": { + "searchSourceJSON": json.dumps({ + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": {"query": "", "language": "kuery"}, + "filter": [] + }) + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": index_pattern_id + } + ] + } + + if workspace_id and workspace_id != "default": + payload["workspaces"] = [workspace_id] + + try: + response = requests.post( + url, + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + json=payload, + verify=False, + timeout=10, + ) + + if response.status_code in (200, 409): + return vis_id + return None + except requests.exceptions.RequestException as e: + print(f"⚠️ Error creating visualization {title}: {e}") + return None + + +def create_overview_dashboard(workspace_id): + """Create an overview landing dashboard with markdown links to all observability features""" + import json + import base64 + + markdown_vis_id = "overview-markdown" + dashboard_id = "observability-overview-dashboard" + + # Check if dashboard already exists + if get_existing_dashboard(workspace_id, dashboard_id): + print("✅ Overview dashboard already exists") + set_default_dashboard(workspace_id, dashboard_id) + return dashboard_id + + print("📊 Creating Observability Stack overview dashboard...") + + # Load architecture image as base64 data URI + arch_img_tag = "" + try: + with open("/config/architecture.png", "rb") as f: + img_b64 = base64.b64encode(f.read()).decode("utf-8") + arch_img_tag = f"![Architecture](data:image/png;base64,{img_b64})" + except FileNotFoundError: + print("⚠️ Architecture image not found, using text fallback") + arch_img_tag = "*Agents / Apps → OTel Collector → Data Prepper → OpenSearch + Prometheus*" + + # Build workspace-aware links + if workspace_id and workspace_id != "default": + w = f"/w/{workspace_id}" + else: + w = "" + + markdown_text = f"""## Welcome to OpenSearch Observability Stack! +Your entire stack, fully visible. APM traces, logs, Prometheus metrics, service maps, and AI agent tracing — unified in one open-source platform built for modern infrastructure. Total observability, zero lock-in. + +### Architecture +{arch_img_tag} + +--- + +### Getting started +1. **Send telemetry** to the OTel Collector via gRPC (`:4317`) or HTTP (`:4318`) +2. **Explore logs** to see application log events +3. **Explore traces** to follow requests across services +4. **Check APM services** for latency, error rates, and throughput +5. **View the service map** for a visual topology of your system + +--- + +### Explore telemetry +**Logs** — [Explore logs]({w}/app/explore/logs) +Search, filter, and analyze application and infrastructure log events. + +**Traces** — [Explore traces]({w}/app/explore/traces) +Follow requests end-to-end across services to pinpoint latency and errors. + +**Metrics** — [Explore metrics]({w}/app/explore/metrics) +Query Prometheus metrics for throughput, latency percentiles, and error rates. + +### APM & services +**APM services** — [Service list]({w}/app/observability-apm-services#/services) +View latency, error rate, and throughput (RED metrics) for every instrumented service. + +**Service map** — [View service map]({w}/app/observability-apm-application-map) +Visualize service-to-service dependencies and traffic flow across your system. + +### Agent observability +**Agent traces** — [Explore agent traces]({w}/app/agentTraces) +Inspect individual AI agent invocations, tool calls, and LLM interactions. + +**Agent dashboard** — [Agent observability dashboard]({w}/app/dashboards#/view/agent-observability-dashboard) +Monitor agent activity, token usage, and tool execution at a glance. +""" + + # Create the markdown visualization + vis_state = { + "title": "", + "type": "markdown", + "params": { + "fontSize": 12, + "openLinksInNewTab": False, + "markdown": markdown_text, + }, + "aggs": [], + } + + vis_payload = { + "attributes": { + "title": "", + "visState": json.dumps(vis_state), + "uiStateJSON": "{}", + "kibanaSavedObjectMeta": { + "searchSourceJSON": json.dumps({}) + }, + }, + } + + if workspace_id and workspace_id != "default": + vis_payload["workspaces"] = [workspace_id] + vis_url = f"{BASE_URL}/w/{workspace_id}/api/saved_objects/visualization/{markdown_vis_id}" + else: + vis_url = f"{BASE_URL}/api/saved_objects/visualization/{markdown_vis_id}" + + try: + response = requests.post( + vis_url, + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + json=vis_payload, + verify=False, + timeout=10, + ) + + if response.status_code not in (200, 409): + print(f"⚠️ Overview markdown creation failed: {response.text}") + return None + print(f"✅ Created overview markdown visualization") + except requests.exceptions.RequestException as e: + print(f"⚠️ Error creating overview markdown: {e}") + return None + + # Create the dashboard with a single full-width markdown panel + panels = [ + { + "version": "3.5.0", + "gridData": {"x": 0, "y": 0, "w": 48, "h": 35, "i": "0"}, + "panelIndex": "0", + "embeddableConfig": {}, + "panelRefName": "panel_0", + } + ] + + dashboard_payload = { + "attributes": { + "title": "Observability Stack Overview", + "description": "Landing page with links to all observability features", + "panelsJSON": json.dumps(panels), + "optionsJSON": json.dumps({"useMargins": True, "hidePanelTitles": True}), + "timeRestore": False, + "kibanaSavedObjectMeta": { + "searchSourceJSON": json.dumps( + {"query": {"query": "", "language": "kuery"}, "filter": []} + ) + }, + }, + "references": [ + {"name": "panel_0", "type": "visualization", "id": markdown_vis_id} + ], + } + + if workspace_id and workspace_id != "default": + dashboard_payload["workspaces"] = [workspace_id] + dash_url = f"{BASE_URL}/w/{workspace_id}/api/saved_objects/dashboard/{dashboard_id}" + else: + dash_url = f"{BASE_URL}/api/saved_objects/dashboard/{dashboard_id}" + + try: + response = requests.post( + dash_url, + auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + json=dashboard_payload, + verify=False, + timeout=10, + ) + + if response.status_code in (200, 409): + print(f"✅ Created Observability Stack overview dashboard") + set_default_dashboard(workspace_id, dashboard_id) + return dashboard_id + else: + print(f"⚠️ Overview dashboard creation failed: {response.text}") + return None + except requests.exceptions.RequestException as e: + print(f"⚠️ Error creating overview dashboard: {e}") + return None + + +def main(): + """Initialize OpenSearch Dashboards with workspace and datasources""" + wait_for_dashboards() + + # Check for existing workspace + workspace_id = get_existing_workspace() + + if workspace_id: + print("✅ Observability Stack workspace already exists") + else: + workspace_id = create_workspace() + + # Create index patterns (idempotent - will skip if already exist) + # Titles must match exactly what the APM plugin expects + logs_schema_mappings = '{"otelLogs":{"timestamp":"time","traceId":"traceId","spanId":"spanId","serviceName":"resource.attributes.service.name"}}' + logs_pattern_id = create_index_pattern( + workspace_id, "logs-otel-v1*", "time", "logs", logs_schema_mappings, + display_name="Log Dataset - Local Cluster" + ) + traces_pattern_id = create_index_pattern( + workspace_id, "otel-v1-apm-span*", "endTime", "traces", + display_name="Trace Dataset - Local Cluster" + ) + service_map_pattern_id = create_index_pattern( + workspace_id, "otel-v2-apm-service-map*", "timestamp" + ) + + print("📊 Created index patterns for spans, logs, and service map") + + # Set logs as the default index pattern + if logs_pattern_id: + set_default_index_pattern(workspace_id, logs_pattern_id) + + # Create trace-to-logs correlation for cross-signal navigation + if traces_pattern_id and logs_pattern_id: + create_trace_to_logs_correlation(workspace_id, traces_pattern_id, logs_pattern_id) + + # Create Agent Observability dashboard + if traces_pattern_id: + create_agent_observability_dashboard(workspace_id, traces_pattern_id) + + # Create overview landing dashboard (becomes the new default) + create_overview_dashboard(workspace_id) + + # Create saved queries for common agent observability patterns + create_default_saved_queries(workspace_id) + + # Create datasources + prometheus_datasource_id = create_prometheus_datasource(workspace_id) + create_opensearch_datasource(workspace_id) + + # Create APM config correlation (ties traces + service map + Prometheus) + if traces_pattern_id and service_map_pattern_id: + # Resolve Prometheus data-connection saved object ID + prom_so_id = get_existing_prometheus_datasource("ObservabilityStack_Prometheus") + create_apm_config_correlation( + workspace_id, traces_pattern_id, service_map_pattern_id, prom_so_id + ) + + # Output summary + print() + print("🎉 Observability Stack Ready!") + print(f"👤 Username: {USERNAME}") + print(f"🔑 Password: {PASSWORD}") + + # Generate appropriate dashboard URL + if workspace_id and workspace_id != "default": + dashboard_url = f"http://localhost:5601/w/{workspace_id}/app/dashboards#/view/observability-overview-dashboard" + else: + dashboard_url = "http://localhost:5601/app/home" + + print(f"\033[1m📊 OpenSearch Dashboards: {dashboard_url}\033[0m") + print(f"📈 Prometheus: http://localhost:{PROMETHEUS_PORT}") + print() + +if __name__ == "__main__": + main() diff --git a/charts/observability-stack/templates/init-dashboards-configmap.yaml b/charts/observability-stack/templates/init-dashboards-configmap.yaml new file mode 100644 index 00000000..7682b755 --- /dev/null +++ b/charts/observability-stack/templates/init-dashboards-configmap.yaml @@ -0,0 +1,15 @@ +{{- if index .Values "opensearch-dashboards" "enabled" }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "observability-stack.fullname" . }}-init-script + labels: + {{- include "observability-stack.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-weight": "5" + "helm.sh/hook-delete-policy": before-hook-creation +data: + init-opensearch-dashboards.py: | +{{ .Files.Get "files/init-opensearch-dashboards.py" | indent 4 }} +{{- end }} diff --git a/charts/observability-stack/templates/init-dashboards-job.yaml b/charts/observability-stack/templates/init-dashboards-job.yaml new file mode 100644 index 00000000..15e87636 --- /dev/null +++ b/charts/observability-stack/templates/init-dashboards-job.yaml @@ -0,0 +1,46 @@ +{{- if index .Values "opensearch-dashboards" "enabled" }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "observability-stack.fullname" . }}-init-dashboards + labels: + {{- include "observability-stack.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-weight": "10" + "helm.sh/hook-delete-policy": before-hook-creation +spec: + backoffLimit: 5 + template: + metadata: + labels: + app.kubernetes.io/name: init-dashboards + spec: + restartPolicy: OnFailure + containers: + - name: init-dashboards + image: python:3.12-slim + command: + - /bin/sh + - -c + - | + pip install -q requests pyyaml && python /scripts/init-opensearch-dashboards.py + env: + - name: OPENSEARCH_USER + value: {{ .Values.opensearchUsername | default "admin" | quote }} + - name: OPENSEARCH_PASSWORD + value: {{ .Values.opensearchPassword | quote }} + - name: BASE_URL + value: "http://{{ .Release.Name }}-opensearch-dashboards:5601" + - name: PROMETHEUS_HOST + value: "{{ .Release.Name }}-prometheus-server" + - name: PROMETHEUS_PORT + value: "80" + volumeMounts: + - name: init-script + mountPath: /scripts + volumes: + - name: init-script + configMap: + name: {{ include "observability-stack.fullname" . }}-init-script +{{- end }} From 9f1937cbcf86be32a0d5f1da015ced7f6a197ab1 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Mon, 16 Mar 2026 16:05:10 -0700 Subject: [PATCH 05/64] fix: use custom DP 2.15.0-SNAPSHOT image with prometheus sink - Image: sgguruda62324/opensearch-data-prepper:2.15.0-SNAPSHOT (matches docker-compose .env, includes ps48's prometheus auth PR #6595) - Correct experimental plugin syntax for DP 2.15 - Re-added prometheus remote-write sink to service-map pipeline - All pipelines initialized including RED metrics to Prometheus Signed-off-by: Kyle Hounslow --- charts/observability-stack/values.yaml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/charts/observability-stack/values.yaml b/charts/observability-stack/values.yaml index 160dfa81..cbe192d9 100644 --- a/charts/observability-stack/values.yaml +++ b/charts/observability-stack/values.yaml @@ -41,12 +41,19 @@ opensearch-dashboards: data-prepper: enabled: true image: - tag: "latest" + repository: "sgguruda62324/opensearch-data-prepper" + tag: "2.15.0-SNAPSHOT" config: data-prepper-config.yaml: | ssl: false peer_forwarder: ssl: false + experimental: + enabled_plugins: + processor: + - otel_apm_service_map + sink: + - prometheus livenessProbe: initialDelaySeconds: 30 periodSeconds: 10 @@ -142,6 +149,12 @@ data-prepper: index_type: otel-v2-apm-service-map routes: [otel_apm_service_map_route] insecure: true + - prometheus: + url: "http://obs-stack-prometheus-server:80/api/v1/write" + threshold: + max_events: 500 + flush_interval: 5s + routes: [service_processed_metrics] # -- OpenTelemetry Collector opentelemetry-collector: From b0d32160a18103804250c81ee06024b5a0cd4470 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Mon, 16 Mar 2026 16:26:22 -0700 Subject: [PATCH 06/64] refactor: centralize OpenSearch credentials - Add opensearch-credentials Secret template - Init job references secret via secretKeyRef instead of hardcoded values - Document that DP pipeline configs still need manual password sync (subchart values don't support Go templating) Signed-off-by: Kyle Hounslow --- .../templates/init-dashboards-job.yaml | 10 ++++++++-- .../templates/opensearch-credentials-secret.yaml | 10 ++++++++++ charts/observability-stack/values.yaml | 5 +++++ 3 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 charts/observability-stack/templates/opensearch-credentials-secret.yaml diff --git a/charts/observability-stack/templates/init-dashboards-job.yaml b/charts/observability-stack/templates/init-dashboards-job.yaml index 15e87636..d9d54086 100644 --- a/charts/observability-stack/templates/init-dashboards-job.yaml +++ b/charts/observability-stack/templates/init-dashboards-job.yaml @@ -27,9 +27,15 @@ spec: pip install -q requests pyyaml && python /scripts/init-opensearch-dashboards.py env: - name: OPENSEARCH_USER - value: {{ .Values.opensearchUsername | default "admin" | quote }} + valueFrom: + secretKeyRef: + name: {{ include "observability-stack.fullname" . }}-opensearch-credentials + key: username - name: OPENSEARCH_PASSWORD - value: {{ .Values.opensearchPassword | quote }} + valueFrom: + secretKeyRef: + name: {{ include "observability-stack.fullname" . }}-opensearch-credentials + key: password - name: BASE_URL value: "http://{{ .Release.Name }}-opensearch-dashboards:5601" - name: PROMETHEUS_HOST diff --git a/charts/observability-stack/templates/opensearch-credentials-secret.yaml b/charts/observability-stack/templates/opensearch-credentials-secret.yaml new file mode 100644 index 00000000..a7d6e4bc --- /dev/null +++ b/charts/observability-stack/templates/opensearch-credentials-secret.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "observability-stack.fullname" . }}-opensearch-credentials + labels: + {{- include "observability-stack.labels" . | nindent 4 }} +type: Opaque +stringData: + username: {{ .Values.opensearchUsername | default "admin" | quote }} + password: {{ .Values.opensearchPassword | quote }} diff --git a/charts/observability-stack/values.yaml b/charts/observability-stack/values.yaml index cbe192d9..16b5fb56 100644 --- a/charts/observability-stack/values.yaml +++ b/charts/observability-stack/values.yaml @@ -255,4 +255,9 @@ prometheus: enabled: false # -- Global settings +# NOTE: opensearchUsername/Password are the single source of truth. +# OpenSearch and OSD reference these via extraEnvs/config. +# Data Prepper pipeline configs don't support templating, so the password +# is duplicated there — update both if you change it. +opensearchUsername: "admin" opensearchPassword: "My_password_123!@#" From a5b6e27958f1cbe858631d19e2e42f307ebb13a4 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Mon, 16 Mar 2026 16:29:48 -0700 Subject: [PATCH 07/64] feat: add canary CronJob generating GenAI agent traces CronJob runs every 2 minutes, sends 5 agent traces per run with realistic GenAI semantic convention attributes: - invoke_agent spans with gen_ai.agent.name - chat spans with gen_ai.request.model, token usage, provider - execute_tool spans with gen_ai.tool.name - Randomized models (gpt-4o, claude-sonnet-4-20250514, nova-pro) Validated: 20 spans indexed in OpenSearch from single canary run. Signed-off-by: Kyle Hounslow --- .../observability-stack/templates/canary.yaml | 114 ++++++++++++++++++ charts/observability-stack/values.yaml | 7 ++ 2 files changed, 121 insertions(+) create mode 100644 charts/observability-stack/templates/canary.yaml diff --git a/charts/observability-stack/templates/canary.yaml b/charts/observability-stack/templates/canary.yaml new file mode 100644 index 00000000..47addcc8 --- /dev/null +++ b/charts/observability-stack/templates/canary.yaml @@ -0,0 +1,114 @@ +{{- if .Values.canary.enabled }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "observability-stack.fullname" . }}-canary + labels: + {{- include "observability-stack.labels" . | nindent 4 }} +spec: + schedule: {{ .Values.canary.schedule | default "*/1 * * * *" | quote }} + concurrencyPolicy: Forbid + jobTemplate: + spec: + backoffLimit: 1 + template: + metadata: + labels: + app.kubernetes.io/name: canary + spec: + restartPolicy: Never + containers: + - name: canary + image: python:3.12-slim + command: + - /bin/sh + - -c + - pip install -q opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-grpc && python /scripts/canary.py + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://{{ .Release.Name }}-opentelemetry-collector:4317" + - name: OTEL_SERVICE_NAME + value: {{ .Values.canary.serviceName | default "example-canary-agent" | quote }} + volumeMounts: + - name: canary-script + mountPath: /scripts + volumes: + - name: canary-script + configMap: + name: {{ include "observability-stack.fullname" . }}-canary-script +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "observability-stack.fullname" . }}-canary-script + labels: + {{- include "observability-stack.labels" . | nindent 4 }} +data: + canary.py: | + """Canary agent that generates realistic GenAI trace data.""" + import os, time, random + from opentelemetry import trace + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter + from opentelemetry.sdk.resources import Resource + + resource = Resource.create({ + "service.name": os.getenv("OTEL_SERVICE_NAME", "example-canary-agent"), + "service.version": "1.0.0", + }) + provider = TracerProvider(resource=resource) + provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(insecure=True))) + trace.set_tracer_provider(provider) + tracer = trace.get_tracer("canary") + + models = ["gpt-4o", "claude-sonnet-4-20250514", "amazon.nova-pro"] + providers = ["openai", "anthropic", "aws.bedrock"] + tools = ["web_search", "code_interpreter", "file_reader", "calculator"] + queries = [ + "What is the weather in Seattle?", + "Summarize the latest OpenSearch release notes", + "Write a Python function to parse JSON", + "Calculate the ROI of migrating to OpenSearch", + ] + + def run_agent_trace(): + idx = random.randint(0, len(models) - 1) + query = random.choice(queries) + with tracer.start_as_current_span("invoke_agent", attributes={ + "gen_ai.operation.name": "invoke_agent", + "gen_ai.agent.name": "canary-planner", + }) as agent_span: + time.sleep(random.uniform(0.05, 0.2)) + # LLM call + with tracer.start_as_current_span("chat", attributes={ + "gen_ai.operation.name": "chat", + "gen_ai.request.model": models[idx], + "gen_ai.system": providers[idx], + "gen_ai.usage.input_tokens": random.randint(100, 500), + "gen_ai.usage.output_tokens": random.randint(50, 300), + }): + time.sleep(random.uniform(0.1, 0.5)) + # Tool call + tool = random.choice(tools) + with tracer.start_as_current_span("execute_tool", attributes={ + "gen_ai.operation.name": "execute_tool", + "gen_ai.tool.name": tool, + }): + time.sleep(random.uniform(0.05, 0.3)) + # Final LLM call + with tracer.start_as_current_span("chat", attributes={ + "gen_ai.operation.name": "chat", + "gen_ai.request.model": models[idx], + "gen_ai.system": providers[idx], + "gen_ai.usage.input_tokens": random.randint(200, 800), + "gen_ai.usage.output_tokens": random.randint(100, 500), + }): + time.sleep(random.uniform(0.1, 0.4)) + + # Generate a batch of traces + for _ in range({{ .Values.canary.tracesPerRun | default 5 }}): + run_agent_trace() + provider.force_flush() + print(f"Canary: sent {{ .Values.canary.tracesPerRun | default 5 }} agent traces") +{{- end }} diff --git a/charts/observability-stack/values.yaml b/charts/observability-stack/values.yaml index 16b5fb56..c96ddcd8 100644 --- a/charts/observability-stack/values.yaml +++ b/charts/observability-stack/values.yaml @@ -261,3 +261,10 @@ prometheus: # is duplicated there — update both if you change it. opensearchUsername: "admin" opensearchPassword: "My_password_123!@#" + +# -- Canary agent (generates sample GenAI traces on a schedule) +canary: + enabled: true + schedule: "*/2 * * * *" + serviceName: "example-canary-agent" + tracesPerRun: 5 From 1eb99aea87d5a9f40d615f1a6a670058d3ed9923 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Mon, 16 Mar 2026 16:40:10 -0700 Subject: [PATCH 08/64] fix: use opensearchstaging 3.6.0 images to match docker-compose Docker-compose .env uses opensearchstaging/opensearch:3.6.0 and opensearchstaging/opensearch-dashboards:3.6.0. Helm chart was using the official 3.5.0 images which are significantly behind. Signed-off-by: Kyle Hounslow --- charts/observability-stack/values.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/charts/observability-stack/values.yaml b/charts/observability-stack/values.yaml index c96ddcd8..3d237c9e 100644 --- a/charts/observability-stack/values.yaml +++ b/charts/observability-stack/values.yaml @@ -6,6 +6,9 @@ opensearch: enabled: true singleNode: true replicas: 1 + image: + repository: "opensearchstaging/opensearch" + tag: "3.6.0" resources: requests: memory: "2Gi" @@ -21,6 +24,9 @@ opensearch: opensearch-dashboards: enabled: true replicas: 1 + image: + repository: "opensearchstaging/opensearch-dashboards" + tag: "3.6.0" opensearchHosts: "https://opensearch-cluster-master:9200" config: opensearch_dashboards.yml: | From 4ccaa8010da59671db01b3d95e4e71ff829a5704 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Mon, 16 Mar 2026 16:46:48 -0700 Subject: [PATCH 09/64] fix: add all OSD feature flags for agent tracing and APM Missing explore, agentTraces, discoverTraces, discoverMetrics, query enhancements, new home page, and experimental features. Config now matches docker-compose opensearch_dashboards.yml. Plugins now loading: explore, agentTraces, observabilityDashboards, queryEnhancements, datasetManagement (54 total). Signed-off-by: Kyle Hounslow --- charts/observability-stack/values.yaml | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/charts/observability-stack/values.yaml b/charts/observability-stack/values.yaml index 3d237c9e..4180b0cc 100644 --- a/charts/observability-stack/values.yaml +++ b/charts/observability-stack/values.yaml @@ -31,17 +31,32 @@ opensearch-dashboards: config: opensearch_dashboards.yml: | server.host: "0.0.0.0" + server.name: "observability-stack-dashboards" opensearch.hosts: ["https://opensearch-cluster-master:9200"] - opensearch.ssl.verificationMode: none opensearch.username: "admin" opensearch.password: "My_password_123!@#" - opensearch.requestHeadersAllowlist: ["securitytenant", "Authorization"] + opensearch.ssl.verificationMode: none + opensearch.requestTimeout: 30000 + opensearch.requestHeadersAllowlist: ["authorization", "securitytenant"] opensearch_security.multitenancy.enabled: false opensearch_security.readonly_mode.roles: ["kibana_read_only"] - data_source.enabled: true + console.enabled: true + server.maxPayloadBytes: 1048576 + savedObjects.maxImportPayloadBytes: 26214400 + csp.strict: false + explore.enabled: true + explore.discoverTraces.enabled: true + explore.discoverMetrics.enabled: true + explore.agentTraces.enabled: true workspace.enabled: true - savedObjects.maxImportPayloadBytes: 10485760 - server.maxPayloadBytes: 10485760 + data_source.enabled: true + data_source.ssl.verificationMode: none + datasetManagement.enabled: true + data.savedQueriesNewUI.enabled: true + opensearchDashboards.branding.useExpandedHeader: false + uiSettings.overrides.home:useNewHomePage: true + uiSettings.overrides.query:enhancements:enabled: true + uiSettings.overrides.explore:experimental: true # -- Data Prepper data-prepper: From 6fc630c6b8006df0d8b705fd384139926d69fa07 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Mon, 16 Mar 2026 17:05:00 -0700 Subject: [PATCH 10/64] feat: replace fake canary with real example agents from docker-compose 1:1 parity with docker-compose.examples.yml: - example-weather-agent (FastAPI + OTel instrumented) - example-events-agent - example-travel-planner (orchestrator) - example-mcp-server (mock tool server) - example-canary (periodic invocations with fault injection) All services, env vars, ports, and memory limits match compose. Images built locally and loaded into kind via finch save/load. Validated: all 5 agents running, canary invoking travel-planner with fault injection, traces flowing to OpenSearch. Signed-off-by: Kyle Hounslow --- .../observability-stack/templates/canary.yaml | 114 ---------- .../templates/examples.yaml | 205 ++++++++++++++++++ charts/observability-stack/values.yaml | 18 +- 3 files changed, 218 insertions(+), 119 deletions(-) delete mode 100644 charts/observability-stack/templates/canary.yaml create mode 100644 charts/observability-stack/templates/examples.yaml diff --git a/charts/observability-stack/templates/canary.yaml b/charts/observability-stack/templates/canary.yaml deleted file mode 100644 index 47addcc8..00000000 --- a/charts/observability-stack/templates/canary.yaml +++ /dev/null @@ -1,114 +0,0 @@ -{{- if .Values.canary.enabled }} -apiVersion: batch/v1 -kind: CronJob -metadata: - name: {{ include "observability-stack.fullname" . }}-canary - labels: - {{- include "observability-stack.labels" . | nindent 4 }} -spec: - schedule: {{ .Values.canary.schedule | default "*/1 * * * *" | quote }} - concurrencyPolicy: Forbid - jobTemplate: - spec: - backoffLimit: 1 - template: - metadata: - labels: - app.kubernetes.io/name: canary - spec: - restartPolicy: Never - containers: - - name: canary - image: python:3.12-slim - command: - - /bin/sh - - -c - - pip install -q opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-grpc && python /scripts/canary.py - env: - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: "http://{{ .Release.Name }}-opentelemetry-collector:4317" - - name: OTEL_SERVICE_NAME - value: {{ .Values.canary.serviceName | default "example-canary-agent" | quote }} - volumeMounts: - - name: canary-script - mountPath: /scripts - volumes: - - name: canary-script - configMap: - name: {{ include "observability-stack.fullname" . }}-canary-script ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "observability-stack.fullname" . }}-canary-script - labels: - {{- include "observability-stack.labels" . | nindent 4 }} -data: - canary.py: | - """Canary agent that generates realistic GenAI trace data.""" - import os, time, random - from opentelemetry import trace - from opentelemetry.sdk.trace import TracerProvider - from opentelemetry.sdk.trace.export import BatchSpanProcessor - from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter - from opentelemetry.sdk.resources import Resource - - resource = Resource.create({ - "service.name": os.getenv("OTEL_SERVICE_NAME", "example-canary-agent"), - "service.version": "1.0.0", - }) - provider = TracerProvider(resource=resource) - provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(insecure=True))) - trace.set_tracer_provider(provider) - tracer = trace.get_tracer("canary") - - models = ["gpt-4o", "claude-sonnet-4-20250514", "amazon.nova-pro"] - providers = ["openai", "anthropic", "aws.bedrock"] - tools = ["web_search", "code_interpreter", "file_reader", "calculator"] - queries = [ - "What is the weather in Seattle?", - "Summarize the latest OpenSearch release notes", - "Write a Python function to parse JSON", - "Calculate the ROI of migrating to OpenSearch", - ] - - def run_agent_trace(): - idx = random.randint(0, len(models) - 1) - query = random.choice(queries) - with tracer.start_as_current_span("invoke_agent", attributes={ - "gen_ai.operation.name": "invoke_agent", - "gen_ai.agent.name": "canary-planner", - }) as agent_span: - time.sleep(random.uniform(0.05, 0.2)) - # LLM call - with tracer.start_as_current_span("chat", attributes={ - "gen_ai.operation.name": "chat", - "gen_ai.request.model": models[idx], - "gen_ai.system": providers[idx], - "gen_ai.usage.input_tokens": random.randint(100, 500), - "gen_ai.usage.output_tokens": random.randint(50, 300), - }): - time.sleep(random.uniform(0.1, 0.5)) - # Tool call - tool = random.choice(tools) - with tracer.start_as_current_span("execute_tool", attributes={ - "gen_ai.operation.name": "execute_tool", - "gen_ai.tool.name": tool, - }): - time.sleep(random.uniform(0.05, 0.3)) - # Final LLM call - with tracer.start_as_current_span("chat", attributes={ - "gen_ai.operation.name": "chat", - "gen_ai.request.model": models[idx], - "gen_ai.system": providers[idx], - "gen_ai.usage.input_tokens": random.randint(200, 800), - "gen_ai.usage.output_tokens": random.randint(100, 500), - }): - time.sleep(random.uniform(0.1, 0.4)) - - # Generate a batch of traces - for _ in range({{ .Values.canary.tracesPerRun | default 5 }}): - run_agent_trace() - provider.force_flush() - print(f"Canary: sent {{ .Values.canary.tracesPerRun | default 5 }} agent traces") -{{- end }} diff --git a/charts/observability-stack/templates/examples.yaml b/charts/observability-stack/templates/examples.yaml new file mode 100644 index 00000000..a3ab9c08 --- /dev/null +++ b/charts/observability-stack/templates/examples.yaml @@ -0,0 +1,205 @@ +{{- if .Values.examples.enabled }} +# MCP Server +apiVersion: apps/v1 +kind: Deployment +metadata: + name: example-mcp-server + labels: + app: example-mcp-server +spec: + replicas: 1 + selector: + matchLabels: + app: example-mcp-server + template: + metadata: + labels: + app: example-mcp-server + spec: + containers: + - name: mcp-server + image: {{ .Values.examples.mcpServer.image }} + imagePullPolicy: Never + ports: + - containerPort: 8003 + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://{{ .Release.Name }}-opentelemetry-collector:4317" + resources: + limits: + memory: "128Mi" +--- +apiVersion: v1 +kind: Service +metadata: + name: mcp-server +spec: + selector: + app: example-mcp-server + ports: + - port: 8003 + targetPort: 8003 +--- +# Weather Agent +apiVersion: apps/v1 +kind: Deployment +metadata: + name: example-weather-agent + labels: + app: example-weather-agent +spec: + replicas: 1 + selector: + matchLabels: + app: example-weather-agent + template: + metadata: + labels: + app: example-weather-agent + spec: + containers: + - name: weather-agent + image: {{ .Values.examples.weatherAgent.image }} + imagePullPolicy: Never + ports: + - containerPort: 8000 + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://{{ .Release.Name }}-opentelemetry-collector:4317" + - name: MCP_SERVER_URL + value: "http://mcp-server:8003" + resources: + limits: + memory: "200Mi" +--- +apiVersion: v1 +kind: Service +metadata: + name: weather-agent +spec: + selector: + app: example-weather-agent + ports: + - port: 8000 + targetPort: 8000 +--- +# Events Agent +apiVersion: apps/v1 +kind: Deployment +metadata: + name: example-events-agent + labels: + app: example-events-agent +spec: + replicas: 1 + selector: + matchLabels: + app: example-events-agent + template: + metadata: + labels: + app: example-events-agent + spec: + containers: + - name: events-agent + image: {{ .Values.examples.eventsAgent.image }} + imagePullPolicy: Never + ports: + - containerPort: 8002 + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://{{ .Release.Name }}-opentelemetry-collector:4317" + - name: MCP_SERVER_URL + value: "http://mcp-server:8003" + resources: + limits: + memory: "128Mi" +--- +apiVersion: v1 +kind: Service +metadata: + name: events-agent +spec: + selector: + app: example-events-agent + ports: + - port: 8002 + targetPort: 8002 +--- +# Travel Planner (Orchestrator) +apiVersion: apps/v1 +kind: Deployment +metadata: + name: example-travel-planner + labels: + app: example-travel-planner +spec: + replicas: 1 + selector: + matchLabels: + app: example-travel-planner + template: + metadata: + labels: + app: example-travel-planner + spec: + containers: + - name: travel-planner + image: {{ .Values.examples.travelPlanner.image }} + imagePullPolicy: Never + ports: + - containerPort: 8000 + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://{{ .Release.Name }}-opentelemetry-collector:4317" + - name: WEATHER_AGENT_URL + value: "http://weather-agent:8000" + - name: EVENTS_AGENT_URL + value: "http://events-agent:8002" + - name: MCP_SERVER_URL + value: "http://mcp-server:8003" + resources: + limits: + memory: "128Mi" +--- +apiVersion: v1 +kind: Service +metadata: + name: travel-planner +spec: + selector: + app: example-travel-planner + ports: + - port: 8000 + targetPort: 8000 +--- +# Canary - periodically invokes travel-planner +apiVersion: apps/v1 +kind: Deployment +metadata: + name: example-canary + labels: + app: example-canary +spec: + replicas: 1 + selector: + matchLabels: + app: example-canary + template: + metadata: + labels: + app: example-canary + spec: + containers: + - name: canary + image: {{ .Values.examples.canary.image }} + imagePullPolicy: Never + env: + - name: TRAVEL_PLANNER_URL + value: "http://travel-planner:8000" + - name: CANARY_INTERVAL + value: {{ .Values.examples.canary.interval | default "120" | quote }} + resources: + limits: + memory: "100Mi" +{{- end }} diff --git a/charts/observability-stack/values.yaml b/charts/observability-stack/values.yaml index 4180b0cc..2abbc7a0 100644 --- a/charts/observability-stack/values.yaml +++ b/charts/observability-stack/values.yaml @@ -283,9 +283,17 @@ prometheus: opensearchUsername: "admin" opensearchPassword: "My_password_123!@#" -# -- Canary agent (generates sample GenAI traces on a schedule) -canary: +# -- Example agents (matches docker-compose.examples.yml) +examples: enabled: true - schedule: "*/2 * * * *" - serviceName: "example-canary-agent" - tracesPerRun: 5 + weatherAgent: + image: "example-weather-agent:latest" + eventsAgent: + image: "example-events-agent:latest" + travelPlanner: + image: "example-travel-planner:latest" + mcpServer: + image: "example-mcp-server:latest" + canary: + image: "example-canary:latest" + interval: "120" From ea56211f703793fa5834e512cb8ae897cfb5e40b Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Tue, 17 Mar 2026 15:03:20 -0700 Subject: [PATCH 11/64] feat: add Gateway API support for OpenSearch Dashboards - Gateway + HTTPRoute templates (replaces legacy Ingress) - Two supported providers: envoy (Envoy Gateway), aws (VPC Lattice) - Envoy: TLS via K8s secret (cert-manager or manual) - AWS: TLS via ACM certificate annotation - Disabled by default (gateway.enabled: false) - Contributors can add GCP/Azure support Signed-off-by: Kyle Hounslow Signed-off-by: Kyle Hounslow --- .../templates/gateway.yaml | 48 +++++++++++++++++++ charts/observability-stack/values.yaml | 31 ++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 charts/observability-stack/templates/gateway.yaml diff --git a/charts/observability-stack/templates/gateway.yaml b/charts/observability-stack/templates/gateway.yaml new file mode 100644 index 00000000..38b9112a --- /dev/null +++ b/charts/observability-stack/templates/gateway.yaml @@ -0,0 +1,48 @@ +{{- if .Values.gateway.enabled }} +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: {{ include "observability-stack.fullname" . }}-gateway + labels: + {{- include "observability-stack.labels" . | nindent 4 }} + {{- with .Values.gateway.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + gatewayClassName: {{ .Values.gateway.className }} + listeners: + - name: https + port: 443 + protocol: HTTPS + {{- if .Values.gateway.host }} + hostname: {{ .Values.gateway.host }} + {{- end }} + tls: + mode: Terminate + {{- if and (ne .Values.gateway.provider "aws") .Values.gateway.tls }} + certificateRefs: + - name: {{ .Values.gateway.tls.secretName | default (printf "%s-dashboards-tls" (include "observability-stack.fullname" .)) }} + {{- end }} + allowedRoutes: + namespaces: + from: Same +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: {{ include "observability-stack.fullname" . }}-dashboards + labels: + {{- include "observability-stack.labels" . | nindent 4 }} +spec: + parentRefs: + - name: {{ include "observability-stack.fullname" . }}-gateway + {{- if .Values.gateway.host }} + hostnames: + - {{ .Values.gateway.host }} + {{- end }} + rules: + - backendRefs: + - name: {{ .Release.Name }}-opensearch-dashboards + port: 5601 +{{- end }} diff --git a/charts/observability-stack/values.yaml b/charts/observability-stack/values.yaml index 2abbc7a0..5ca03112 100644 --- a/charts/observability-stack/values.yaml +++ b/charts/observability-stack/values.yaml @@ -297,3 +297,34 @@ examples: canary: image: "example-canary:latest" interval: "120" + +# -- Gateway API for OpenSearch Dashboards +# Requires: Gateway API CRDs + a gateway controller installed on the cluster. +# Supported providers: envoy (Envoy Gateway), aws (AWS Gateway API Controller). +# Contributors welcome to add: gcp, azure, etc. +gateway: + enabled: false + # provider: envoy or aws + provider: envoy + # className maps to GatewayClass installed by your controller + className: eg + # host: dashboards.example.com + annotations: {} + tls: {} + # secretName: dashboards-tls # K8s TLS secret (envoy) + # + # --- Envoy Gateway --- + # Requires: https://gateway.envoyproxy.io/docs/install/ + # provider: envoy + # className: eg + # host: dashboards.example.com + # tls: + # secretName: dashboards-tls # cert-manager or manual secret + # + # --- AWS Gateway API Controller --- + # Requires: https://www.gateway-api-controller.eks.aws.dev/ + # provider: aws + # className: amazon-vpc-lattice + # host: dashboards.example.com + # annotations: + # application-networking.k8s.aws/certificate-arn: arn:aws:acm:REGION:ACCOUNT:certificate/ID From 33b6b603835ce25807589fb57a302a26cb10e5e8 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Tue, 17 Mar 2026 15:24:29 -0700 Subject: [PATCH 12/64] fix: update NOTES.txt with port-forward commands and credentials Signed-off-by: Kyle Hounslow Signed-off-by: Kyle Hounslow --- .../observability-stack/templates/NOTES.txt | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/charts/observability-stack/templates/NOTES.txt b/charts/observability-stack/templates/NOTES.txt index 5747b3dd..89e75f72 100644 --- a/charts/observability-stack/templates/NOTES.txt +++ b/charts/observability-stack/templates/NOTES.txt @@ -1,22 +1,26 @@ 🚀 Observability Stack deployed! -Components: -{{- if index .Values "opensearch" "enabled" }} - - OpenSearch: https://{{ .Release.Name }}-opensearch:9200 -{{- end }} -{{- if index .Values "opensearch-dashboards" "enabled" }} - - OpenSearch Dashboards: http://{{ .Release.Name }}-opensearch-dashboards:5601 -{{- end }} -{{- if index .Values "data-prepper" "enabled" }} - - Data Prepper: {{ .Release.Name }}-data-prepper:21890 -{{- end }} -{{- if index .Values "opentelemetry-collector" "enabled" }} - - OTel Collector (gRPC): {{ .Release.Name }}-opentelemetry-collector:4317 - - OTel Collector (HTTP): {{ .Release.Name }}-opentelemetry-collector:4318 -{{- end }} -{{- if index .Values "prometheus" "enabled" }} - - Prometheus: http://{{ .Release.Name }}-prometheus-server:9090 -{{- end }} +Access OpenSearch Dashboards: + + kubectl port-forward -n {{ .Release.Namespace }} svc/{{ .Release.Name }}-opensearch-dashboards 5601:5601 + + Then open: http://localhost:5601 + Username: {{ .Values.opensearchUsername }} + Password: {{ .Values.opensearchPassword }} -Send telemetry to: - OTEL_EXPORTER_OTLP_ENDPOINT={{ .Release.Name }}-opentelemetry-collector:4317 +Send telemetry: + + kubectl port-forward -n {{ .Release.Namespace }} svc/{{ .Release.Name }}-opentelemetry-collector 4317:4317 4318:4318 + + OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 + +{{- if .Values.gateway.enabled }} + +Gateway API ({{ .Values.gateway.provider }}): + + Gateway: kubectl get gateway -n {{ .Release.Namespace }} + HTTPRoute: kubectl get httproute -n {{ .Release.Namespace }} + {{- if .Values.gateway.host }} + Host: {{ .Values.gateway.host }} + {{- end }} +{{- end }} From bbfee2f2e3bccf048d2376fdf192d74e0f09b1bf Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Tue, 17 Mar 2026 15:27:49 -0700 Subject: [PATCH 13/64] fix: use correct OpenSearch service name for datasource creation Init job was hardcoding 'opensearch:9200' but the actual service is 'opensearch-cluster-master:9200'. Pass OPENSEARCH_ENDPOINT env var from the job template. Signed-off-by: Kyle Hounslow Signed-off-by: Kyle Hounslow --- charts/observability-stack/files/init-opensearch-dashboards.py | 2 +- charts/observability-stack/templates/init-dashboards-job.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/charts/observability-stack/files/init-opensearch-dashboards.py b/charts/observability-stack/files/init-opensearch-dashboards.py index 4623f0f2..75165d75 100644 --- a/charts/observability-stack/files/init-opensearch-dashboards.py +++ b/charts/observability-stack/files/init-opensearch-dashboards.py @@ -376,7 +376,7 @@ def create_opensearch_datasource(workspace_id): print("🔧 Creating OpenSearch datasource...") - opensearch_endpoint = "https://opensearch:9200" + opensearch_endpoint = os.environ.get("OPENSEARCH_ENDPOINT", "https://opensearch:9200") payload = { "attributes": { diff --git a/charts/observability-stack/templates/init-dashboards-job.yaml b/charts/observability-stack/templates/init-dashboards-job.yaml index d9d54086..c0ace5e4 100644 --- a/charts/observability-stack/templates/init-dashboards-job.yaml +++ b/charts/observability-stack/templates/init-dashboards-job.yaml @@ -42,6 +42,8 @@ spec: value: "{{ .Release.Name }}-prometheus-server" - name: PROMETHEUS_PORT value: "80" + - name: OPENSEARCH_ENDPOINT + value: "https://opensearch-cluster-master:9200" volumeMounts: - name: init-script mountPath: /scripts From 0bd57546f97a23bee37530493615ccd6041d511a Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Tue, 17 Mar 2026 15:47:12 -0700 Subject: [PATCH 14/64] docs: add local TLS development guide with mkcert + Envoy Gateway Signed-off-by: Kyle Hounslow Signed-off-by: Kyle Hounslow --- charts/observability-stack/docs/local-tls.md | 101 +++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 charts/observability-stack/docs/local-tls.md diff --git a/charts/observability-stack/docs/local-tls.md b/charts/observability-stack/docs/local-tls.md new file mode 100644 index 00000000..0984abb8 --- /dev/null +++ b/charts/observability-stack/docs/local-tls.md @@ -0,0 +1,101 @@ +# Local Development with TLS + +Test the full Gateway API + TLS flow locally on kind without a cloud provider. + +## Prerequisites + +- [kind](https://kind.sigs.k8s.io/) cluster running +- [Envoy Gateway](https://gateway.envoyproxy.io/) installed +- [mkcert](https://github.com/FiloSottile/mkcert) for locally-trusted certificates + +## Setup + +### 1. Install mkcert and create a local CA + +```bash +brew install mkcert # macOS +mkcert -install # adds local CA to system trust store (needs sudo on macOS) +``` + +This creates a Certificate Authority on your machine. Any cert signed by it +will be trusted by your browser and curl — no warnings, green lock. + +### 2. Generate a certificate + +```bash +mkcert dashboards.local localhost 127.0.0.1 +``` + +Creates `dashboards.local+2.pem` (cert) and `dashboards.local+2-key.pem` (key) +in the current directory, valid for all three names. + +### 3. Add DNS entry + +```bash +echo "127.0.0.1 dashboards.local" | sudo tee -a /etc/hosts +``` + +### 4. Install Envoy Gateway + +```bash +helm install eg oci://docker.io/envoyproxy/gateway-helm \ + --version v1.3.2 \ + -n envoy-gateway-system --create-namespace + +kubectl apply -f - < Date: Tue, 17 Mar 2026 16:11:35 -0700 Subject: [PATCH 15/64] fix: mount saved queries and architecture image for init job - Add saved-queries-traces.yaml and saved-queries-metrics.yaml to chart - Add architecture.png as binaryData in ConfigMap - Mount all files to /config so init script can find them - Update overview dashboard on every run (not skip if exists) - 20 saved queries now load, architecture image embedded in dashboard Signed-off-by: Kyle Hounslow Signed-off-by: Kyle Hounslow --- .../files/architecture.png | Bin 0 -> 70401 bytes .../files/init-opensearch-dashboards.py | 13 ++- .../files/saved-queries-metrics.yaml | 73 ++++++++++++++ .../files/saved-queries-traces.yaml | 93 ++++++++++++++++++ .../templates/init-dashboards-configmap.yaml | 6 ++ .../templates/init-dashboards-job.yaml | 2 + 6 files changed, 180 insertions(+), 7 deletions(-) create mode 100644 charts/observability-stack/files/architecture.png create mode 100644 charts/observability-stack/files/saved-queries-metrics.yaml create mode 100644 charts/observability-stack/files/saved-queries-traces.yaml diff --git a/charts/observability-stack/files/architecture.png b/charts/observability-stack/files/architecture.png new file mode 100644 index 0000000000000000000000000000000000000000..0259da802f69b63c4d23922728117ae510bb94f0 GIT binary patch literal 70401 zcmd42WmMeFw>1cjHx7-vH|_*?4Fno@cXtc!!8Je#p5PvWyM+)4?ry=|o#{OP`@HX6 z^WK@Y=G%OtX*u<)Q>V^8b@r}kh_W0ykQfL91B0$0FRcax0}p)->wt_1eMg2{U5HPLFj_lZE>%=8BsY)XK1FftY_wK)8L z`{yhoI9UGw{^q~m5SI)N2ecCWx3B;E3Di)C5`Y#o{`%@WA|k zaKReR-{)ce|NcDxFVhGyKpvljYZdvQjsE}5t>VcnBK&J@a&JOlvcnJTuHgX^=$fAB zn#6e25D0*}?!$8)Kdp$sz1RDmUTRZJ_j$)y_4h(*!7*hzRH&FdyUp65?(-}|M@wAVcEek# z^>Yjk4HYCm2|=A$g#0hXwkDN@b|!52A=TxPoyn8I`Qp+4#DBgtmb_v#8f<}GAy|Nk zR)kK_fHM3}gR>n~Jt8>0kFrb}{yvNnI=TS`=TXwTs>N$Z5}sO3#1Eh^W?1s?)>^>k+x#cq^80bIV$~$la*;Q$XkXORh%>TC39$0A&Lk*7JbW40xBxZ?`J4Xrc@Cg~ls%~1Y zYb)CHQzXavyZ>X5LI{@fTVoGJ;uenHW z#n9O}E`w3t5Zo;2aQ4wIAX|T5PcJ>#v;-@L2KKbWPjsLi(`M~vJepx(BIx4NUz3z7 zfWXf^s_KJYKxFupO#b{5`&A9isoEG}!@|!0#6&PMhy-w6&_u;HFCxOWjUR)!l#*)1 zRwfMJ8~YB%nDk>S#KqF(HD7j%L2nBG!@#i?Sm33zv;Pj41_3lnQwg4Fyw_N-^YP5#DxnTL zeOJMW7#gG4tVRH=Zo|N^<8TV~bcGcUOcw@7V4g??M?HS$YrAD<_MK3b{wH(p<8*I^ z6CdDTMi3)D#~mYO5~n>Vw2~$9{tGE@l_*&S0A``eQqsZtl@J+gg_alRbPy6LEeI5) zN=Vam)U-^*2<86*C~+@%HQZ3j zH_Qx7YYIj-Y73q2Ak9L{-(Cwm@7}F7kZr@CoCOD#Iq}i|LEHyR`~*?|mm$~)>R=%{_(#@6<>-K8GQ9Pv zvIl3Z8VClLwLI7a3nw*o5U)dH#O!1hD45!B&G2P4lms_Aa z_=8UeGq2#@vOX3u2Fu~odM-7Bk|Zoiyq`+0cs*5Le!HRVnV=h?M!4jMxMxpM(*XwOGPrdF{X>gsbN_s`6diC?CQBzLO98)(sl$bh2@1-L1Irz?ZyjY8ec5u7AxSb| zaplK!)`(n3y}rDq#=fU(60V5Q{;#ynr@N@v6gyFslap^E^&h zR}*VX^iCxWo3&=_VOja$dh$2Eh>~Q3aqO1+)Y!sTmvDI*HThh_8a6r|g2_frx>LxguddcVJjUZcaxgbM`!8TTTH*wD#BIuPt9}gb!5B$)yx+GXQ6v%!e+=wAJ!9LB zMkQyqHgb2S8_sfu3(eCNM(-2>B!1-b;#cj^;9e`;c3~YOV4&aNRPRvJK3r;Qi$7Hn zTi}$U#79N|Yu79?k~^E1>4K9|pTc$Enf?y9q;c}d zPTuzLBwy2?w)a-D;Jt@jNl4HZr}r!Fu{mzaqIBA)8O7URm*H$N1qW(buQuk;9}xxHBEydb>wv{TW|e!t-AvyJ3T1 zVXGqJ6OAga-vMSR)V#t*@M!CP4s5kn zveS#Ht1j+|-NAcL_QO0QkBl%oJIkb^xR7jmwC+?{*z0*9nf!ccLQ^B$j*~1Kv8vDf zjd7k02^obE=Gv>qZ%@vbb_$mQ33&&Iy}$8VXP=(B$RYnJ=7}-|^;6qMT{&NT<43-- zs|B(CimTYMM8pkFq04e7<#f+yG3ooyK3(1v@q;W^!`T(;#WP>-`U$TOYcO6T2bHcW zS4if)&(`aN7p_0+WLH;L*ly3XOD1yr9JQaWPV5(2;Ie8OzG`V^ubj=(b-{(HP2#?- zD6>IqazE<&ICybjGk4ctTF%5|=SoJ3R5f6p`%J+XOBqg3L z#c1ki-*`wk#_E0Wv?4;=!+mh^xn|mX?oz0=T{R$_F5=eVafUr6<_$GnY1=)}-pb1!4Ql;gUlobze4t=0w zq6&{qATHO-uPPWz9H5Qlmf(zEn6&Um<1#Z1_*&Pgv5!2CUBP%T1!wq4 zX+4i*U&kLeRy0;7Vjy&G<9gmLMkb0=!_N{9xZOq7qP zF8te=bbfUqf?yU=opim9v@kv&TbA*UKZ-0$>o5TdEJx%DrnfP{E8LMst38oVbhlGM zb+l)Xw1QMDJwIxfp}u++yID3OZvVR5MWn3u7#Tc<$T2zFv|QsnQ(dO}2gOY_yo&*p zUtfkqU)x1T6;JvvFcvW+rGwCVRJ7?zegw^1pI-E8u8Ko3k%kf*$TyOU(8Z#M#(FRt z5vB5MGP*ax;rXZBIXrZG8nkzPNXth2-8M5>G6;dIMy3Jtk zdEvHhG3>$}G`TG3tMwW)ny<5=y0V#y3^><{Di?V#c@4xfwKz56c)xKf{G}kU!Oj4z z^Tr+VX}Xe8rh;e$I%h3VF^K<)jR zsZp#cenow7VPB;xjcHsg_gE5GAwC?ikXXD{!Z0jDnMDmM{n^QV5Q4ld*?suzIi^jO zC^vT6{yRfFKyf`@f8g+8mdvicFoGy1fmF{)61J5_^s0iiy^!tSdJIe-l4#`qWm2sd zW687Yjan{w%3siIZt#A?vhWR0*|4ieIy7$~e2qAAEax^J;u2*^z_R$ANYtPuO<*V; z8R5JV5n-0!W?zuB(9CS(d?Ujlo{_0q;gzlAibwg`S|t}_ecxG<)h^rc{^(PZ`f9Ggc$mXFEakIxUCBIxX%DVqOQQT?G=?b{FlKMT>7x5Yz_lKi08&r%x-tdn>xuu67zX zcs~6h7=YD$f1wlJVfzR1xcn-x2<8MFhYA}->TP$Rd1RsMxD%xL{?E3dlUYxT?n;4F zJeThUiy%G5iJuq(E)86%-Vm&YW)ZuV+3ZN_?h6(UK8hyOd{{%o)%nWpd*ERsMerwF zgMt*Ple&_x#l19nyDW$Fcn2CafJMEmY|TK2Z4@U_l?S8Oz1`+d!}-=F>Eor!o`i2= zS@(~ofvw&)u)}YjhBNe)$wfHwIxMrk`j8P4@AX`{N}+CJ8cP4Iw2}~DRrQ4Z zo5eUu!xx8--Y@IX$BJHT-XRqf|i2(4}n9fvDK% z^k)I7`HPga2O91hG%6-O{FIcGmVC|`PeK;bH+<~3igo-^(6~jP`iA<7dIItM&=LdD z^vX%Hq=vu%OzR|^enTrC{a{7KZ8~R6!QVLLwuuc@U8`>qNk{`SbpL!zK67w8(X|tZ zM)VzG(rK_|S2C&|$7ku54y%vgE#?63D#Ac#UV3!|6FEV{UnUOf-gRidVeYHghr$^OD2ZgC*1F z{gui(q>#Xx-7~ug7(LyxG%EcEI6;x(Zbu{_)IuH)MauYz>n8>@&K)4wKj{g^u8U5LSN-wiO;ww0<*ECw9HN z<+44YheunRaHiyql72%LyGZwJqp6gZp9!7t`v){DZa*CZtfd&n)Sd)O?yFBcMAeRK zKkNzQiP?Plw@xq;G04K1{`?B018r|-xIcOsB=P%4&-rgH00V@)EEGxc-PT-7kU-eA zw4k3>w0ZDibyzX*&I`rmM2Yp#c|}lDDr@6@VAiNDy^$?gP6^|Fak6YhogDa_ zDQ{9;Y^nQ|JRfo<%tO{6^uoUnt}uaSUHGX!xAxhs6LXtcb6OAECA9QWa?BTJIMe=Z znf&TvU#p;z|^-QmVjZdblA+hzM8QI ztuTU`wVoBZPOKs6yU2zLiH@v9}{ILBAtH6`e>Z z1q32yloi61t4Q2Gr$jE`HhKz%d~9KK(BnhB@@O#K%*I>yT_bsNCfn7;v@u^QZy9O& zUSlTPl|*C|>$>+1iCEx1==UY8PS)o$hluTMjdU4o*y10N(Bgj zk`gyBb$RY{!uL8RJ91hDFaf>qIX(_A^=0lWu8)@uySU9UmFllxlh-O?E39QENAC)1 zdi*3EJdKav6>+vX7DkjsKSFu-^DyZYg8gy*aEvrQHx*HRoD~uh_~X~-52b|7K3f>` zOL~{CKbb%#k^Q^l99|UDFy?)6g3SNv%3jc7QM^YDeb(}qPY5us`89S{RvJK6kdM_r zH919BxqHQtj0 z9%`#4!gw2ZQ%5%+N+{U>4J7Gj(Z_ez(RsdpS2QF9Kbf$9e(e68F_`BMymjX58XS$Y z`}B4`*D(@kr?>N_(*DJ?%+laW|Nd<_Kst}2Ze32Y=d_}!cli&=#!VTSKTfRU(MVCP z=`Iy=vmZ|TlhK6&w{md<5AS{W?fea%@CsQTpYcueFntu?`sEQ`e^)0;eTGx;%;#HJ zE4uY+**aI0vyo?xyzW~f-uPS2>x`c9mUrq9b2DjwN$NDmadfnWaNO5tuS%T+u?VMm zOTu+rkL=Pe|4VYC6~S~yK)&PE)^iy5@@trD0FubQ7-PK$N#{(>ds^*s098X0YOtbfQq z98a^xdJ_D$rY#A3@z#1RhaT+|HSEXO?>LHqNM8LL&bnOz{|iCPS8_qPzLzE#a-;da zt!}TotB<6E#Rh#&XFG;i6n+K=9In!0E<_?^<(h0`GI$mp=GVgWjj!;y8G*aIrm^n> z$;ulKhPi<|%ETz~ZLJqd=uPoK-+J3LOw>_c*BBy{;6W+k(~ss@*DrfCe!JEUv5OKo zz?dxlYX)~{k6xbQ{FK3fS&Mn=V zKeJ5Z-)dRh=e|FEc>BOd_TtB57;b8T2c%umJ4B(fnZ0eYII6$kA?u-e8`>F?`;=c}8fW5Dr{8Ru(ct~Ns8foh<%ZVn=_Qgx_W0%jLjaRsS4ZlPSNKER@70Bs`v0hG_&4= z20j2|BlY^qW8!aQ_+MV=ih+P9InNn`k-8|H>?^(wgQJF9g`>~zue}#RUhX{urw=_! zx0Krp#;yvgD%FeE??;*%*~(=lB~+7(17qS~ zP!J~NI@Y&81b%cpICc4MTS^shXpxL8AJtv57kuUCJSs4D)VxvZ!Gh1u%nZV-k|AHKrF4ZzkkQBHJ8$Z5W!Qe?i;$`)NENK_ z568{CjV*o#)YJb8c=OG9vk2~=z9}u%m5-O3TH2aJ6l@4{Objv#f{aYo10&ZzoAdg# zB#7LoP3f@8t61}^b2bYOtSt@}u#)sQnV8~~g=_7#S)wl{I_qKAF|pILvuS5G>0o8> z^5SiO=&2=cK5G|&b9Y%0_fkYYOWt0Wh5gxX)jWJV2Fx&Zl2SWoSiYSllFgbkY4go7 z@GwIOiS<#|A4k+w+W13U%pdzz*A72r$q++E=G+)(M_ZtEjnBn`2T zf?nAxC{Hl5XymY4xG@~op1J)(7^8{e!G&bOJeQZ#e2y;r>dihMsZ|`skGXy8TZ&k) zmi@{ta{-L5N@hp)@z3=iQCRi#YB~Xc3gY7~66>3nSG_C!VhD%h5f(@3e0IrejUhk6 z;w{Bs2DyoBk54-K#hYPZi?12U+C6Lp>2d@Nw;)$1DDTm35T%kTX=?tv^F&f296onH z9eQ&=OhgfX5T+55L`uVW^{I|giP`Srefy5|7r~jkNjMTk?#?F|8Nw05{pH58>=(CE z{lER~?pu_v{asPF-h=)qe3#f=@X)-bRnSi` zp}ObZzRAZBV33%A&amn?$G)kEO$&`UHW(m~0Q2d$H*CE{RW76t(aGosFQ&jt+oioiXwOy6={5ebkf8@i2Ea@X1t zS%EUo7|!@%e*mKAYeivWaQEQwHTo4CXWb!S!qWo)|Ip~-Dq>)GVAn{f5N6(4UuH|K z`ETJ|st}dC3B=Bzt8D=SRgN9~+W?b03$5DBd))#DTmB9|^t5{Bk6&IJ7SXqH|HLBA zT6@LHAh;4mxX(3y62kEy?BkS~sG*>7ag$0$;aPlmXkorm0EiLV z?zag03_TyY&{a&*VBuVnpLDhH``az?E)`j8p0jqiRx}$dl`c}Eme}IQY&vSBAjj?O zR?cnHg0|D&C5aTTk0WdSHj2VyeR8JLnAHf(-rm?+Ox8GEMde zM5m5dqR$OQ$x4T|M7!=-?_r9fj|Dm@=d7m&R@w)De7&4<;mS$RP1nt%dThYq0>$9{ zs|7fhbR(VHHxH3yj;%q(-`x%}dj82edh~4F6y3&Tu`(fXA;5--9WC?3&C>osNXvxp z=Y->Mf!tvFT66l+yvYvY30nxdqiOP61(Ddw$Z_p`;1KT9v4o)teLG5pI2Evr*pt~2GRNune{AhjVYr`U^maI1YmdYIumx;9*7#c8nm7P z)IG0RytWS!QIt!U=>r~Mo{EMpy>SYIWA2`<^_Ak_&hCF$g+Q z{Sb{p<_LZtH3urYo@o<{imt$G+37_l4)b`{LuSBtBGl-{Zr>(sqC+9o!qR+1x*K|8s7+)HjNO=@2Bd>FX#J`encv8B^MOU`T4E<(2_=i`{H-X>m zI}!gm#k*?x_%Q5mPlE#lIh+mU^x{$$?n(p$ zw&b-qRDsrahEO^RO6q2JNBQCQPN^3%q`BTKF$|9?fRpG9zp)ZP5e!7yw)A_v;>d2~ z_UR`)+^I*hjb)crx^_%Gq}K7->xC!a;Aw6BPkjc*q%WxXaWz;yN<|`q39tLGlb{*R z74|f0)#ddD&ZYbWS$$9m+DEY&ucW#`v+Qo#4!JcC`isDI6Uv9QE zPe)kq<|dS!pEbLSNmL15D|yCn&lug%MQ?roUdhE}(<`BkEb0dw))MIq?nh>5QIYIFOmg*kt0*3$cEH93|@*SyCeD*aeLxa<=nZEc_m zxgf%|&UpiJdZa;#DH&F@^t$hFT>p;ee3_*kNQ<3ir`6r^xjWX~3EZ@^8in0Vh3?sB z`BiNu)p!7uny)N&dg|@m7gg7cuTdZQBZAg)&AzY#GdAQ%056je;!pFpaKnPq#q0oA zs$8ravpy-m3<6J!eK-_^Z(kJ~|48P4MMe;+MzA?3-Csn3mlRTM{i#d|`sr^kSqH-u zrn8)bp4|Wgzwc1NrKPC}%1KG*FMB`m)14{Q0oCgdSMUAd)#c#W6Jwx{4L|9kJE2jR zKlhkjn~ruU_G?wJ7LP;f0r;)C6C$0;4M5aJP{oU>Qy8_oGVZ z;B5=;uE`d3r%y$xj91Bcy9Bwx@pM&I4eZe;W&wKg!V2{BsvRbcP=9@+w!8~xW#K`4 z-*|)&CW{Vl%x;YAPl6o9P7$e$E@b&*^SPVEZn_FKOco=&-r*uB3x%4j)f1)$$w;X* zbGY#}>o=^KRevNN$@=u~)Y{|INZgLh-cU6p_ zk5s)ASHbc^ejQ% zF#RzRT`G3X?p9PSzI2Z7gB&`o8!Gr=R0drhaJi47VGT>M#y5dFM{+)QCuUs=j#{Ki z*K0A<^S5i}=ZzkHRbF;_S~2Sw_t0YMzm#Dh?2!4fIm(fs!9U+dRhE90D?%YzCjIDF zytpeq5MveX`5pW^3sxUfMzPgnEOFt|Zw`9I4@hDmT(=al zA->b&MQAc~;;_AWy-R^nKz#wB-Riyz8FdIvWTm2lrH{`>i_dx!+EfEKPy2$PVLFUJ z>)8DPmcyz$b415wUvi7-oyqM`kSS!YXMN)kgFxAET@se9uDf|`O=OInAa)>_h#(Vt z!r5j`>2srlZ+}uA62bMc_TEccS>{IlyK&{Y@}V0w&)InpF)7i1gbn{BX*}3EFy5-c zDfTDg>lXh%u4m68(WuyXP!>B_E`y0aWRnTh>>HV25tQ^XWpi>D!^xS*c-mt{ShB5_ zCA357Ycm5@Da+ne|Gdk!EDJV?!>6xhO9R zGhAC|on|{&lY^Ihr?5*Fkby`L-ivMnHin@vvI_a5!ZG*{( z6gyx@Z$(Iek!W$&7OA4KGJFx;?s7=jiZPANeE1M0o8JRR%h714Cg6 z)y=8T7bLLU{vMZ@#}S5`V9dl5sO(l?GD}gPZ+Wd%yF2X!9=OpwohuU?xp(Iq$>AZu zuQ41s-b%o%o8aW|7#T0@L_l3JHtAP9!)O=v(|hc~Cs@o3CGiR9eyRP`H@0$JxdakA zSMHfe@%v9whC&`fNK#3KSt@l0!P51)J`-vb6!H)>zcFe922{cPKFlMsEI7ud3-DDB z&;yMpY)mJa2WAC>7H)Y956{iWsvk6fYwsxV;n3bw^!Gve7c@jPif((5JpEkqMly$a zuaeL85nc^m@GxezaO=~%c))UTt#wzNS!7D2F0fhiY8tv)+7Dwy1A^MJT zE>d767S#j4*^>()uj{mSU4B)Gs7ir~kaZ_~Z&{>zF(g=#l&$s;YEaz+(56i_R@Q$3 z4oEP(!@BFVeqI&TXoauF@*q@~B*G*SA3FEBo6>OV&a*{};axZr(c=&Ge_>1s|h>K{!vg24ZUMig_fdIw3L}Bb#g>Q5 zKuWV>UbLe>x}mO&v$PfvzGk0!<7{TtJ9oNUvY{TdywctKEA*5pSmm+asp==gx`P6o z6{?As6!GV{rt*m_mF=px0YL%xQMVM`Bg=8In7SlZvZdvvp|BOPkfg}xFYJP?f}b+J z1WANY^km(quHB(-hK*FK0-)!0t36YV)}{BucJ8J8AtO$-TuT3yn#9jh;C-$V6EOXB z8xC}k(;AzgZM4teg;b#sGbWIMCSFgl(eDw_A+mLE(zC}W+A4kmZVgbivr$Kgt$E31 zEBq9`k|wC>Z8(K0NTRFcO1>rNF2!z|^yWetC^rzJ2jex-rUF*mt zPj|O~sxPJx2W}uJ!tt#eP+C^{;t0-ek`9aT1<)?G3_j-tK6&d4v#z+fTk<-Vn}b(` zK2-iMb%7DQnV4ay%WDqOv7u43{E|Bk@awy@qoEveS6pZg2D-x3nZ&!B*T-?)0_X9w zi8s_H%1;{0i-=z{DyoA6Qxt!_Tlf5+(t7K6YwIrPY@n;AAiM^+IcT+9IsQz+qN!&7 zwfWzKj}_96E&uKbIm9~<1^i}RYN6dQ`L;eL0s_@6oNH>HwMWTW|N0D5y~4-K#iaHo z<}+k{2YVSYJd~U*{`Vk0W2wJK<5`-$l>HYV1F@h7-|(QN2NO0}4PaY%^=8nCtz3BR z9`ZKX%OE^QWCg$`Tja+EJ#^Q5V0$SV{ySkx0PyX4&hGk$ucalPO!46`wymY(l07NM zLhIA$-eBWu7I?&=BCK{EUFITIq(`o^&5vSNHZwSZWaC}(gh>#}fRRhg8&zi=&+ZE~ zSq$8-&N+WuHrQB|p5Hy5H1%dd6QdPmt(Qtn@p)zzbBfy7&op`e+X01iXA|~XJ{sc) z)`Q6|npq;ul@JcAeXhOL8nN$oE5sh2`;SRHNFAk)GP1J0J#a|t9eL~{WG76QC(sOp zZq|h*rYKKy*!L{rAQ-C?u2>7X}#!H{Pv0AhyqtVtd z;`SpcRMYlf+Gh(D%A=Xa?b7ZqQ^S~OMl`=yL2BS`rBFe~C5hc?!y+Q)g%s|Ps?Tqv zdfS@L+)s_TuEU{cZ0J&;12moU3O<{o(Ad?{d&;AjWH>xDUKd3#cU2Bb} zx2?%Men}5+;t#9C!ieKOn;1eEXbR0umdj^cwzr2uv+@r?Y2tQvOy8{)74Q?u5gM4Z z(?4ZEi4-)}H^Q$+AJjY@mny;LJG9r1Iu?fly6m>_^KNkCT2rrs6Ti+gQtwzQN2&3JgJDy^bTJNY&}uuM7$iM_xR5|Dx>CWM@1`Edc%K^`0!RmuqjKnu?J9M-DGOo|R&%VInIhUBbEiZh+aE z@jx#>&1sF9>B|8}SN&k&fEoU``TB$qd=diFM%SUZi0kM8b9L<*v#tEmWjyXiqED}O zOuAl{-$SX_1+fUR!~MVgvU0e$^b<^Jk(RY zB(&-y-Tc!0&u~H%1g@T}FBA9HvRYUlpknY=uZ2-wIWnB$SdAJ{u}Nk&#=>0j(v}*S7nN6hSNj~o zTC0US#lN&XSMQ=8=7rH)hrRW|mKuS^tZn$K$tXRo7m_{07qDoo3@D*a-)KGRxw87S=4<2=Ss516+rv+Hs?tI9UwnOBkb zHbxnFrt|!Qzsa%E-n}|ns^swQCGB=rY`$M##(1#5Ugkbmc;F!JHer8%lZS@yK4`M| z+i)&>-BYhwow-?W+f4%8oAtWkv1LTAkgw%2;=C6n2YoNwb?t(Sy!4{&oVa|GZ5kAu zxQQ!QVoTK2Y^5}v&PD}~0653`@Tk-3X^a^Et+>H#2a-xgd$sG-)7?i5XHmcSeshc+ z%_hw{r?U%IW~cqj^V!$4%m#^tKF=flwZwP8srH+vd4RB0xwU73YRhQ*!wF-JV{AXA z+SD9Z=TxCtjk(X!QL15AQBg@I7wnHFPs`nvzkURElc2UmO zNQew?xqAFYtT^5(S!+i*@Pmfr+XpqcI5B&_M!0gj{TX4$IU7_IgoTaq(|^- z6yl;P26F-3=;b9POix)_bCJPFAtj#+;zQm=<;?OD3|XzDyP13i|GVMR_@VuzpC
2Q$p_%A??RdRsw3L6bbG@}~l+C`wjV^oQ%i+jwuCKd%F^ z1ncYTFj-kd5i=GO)HF0yf`Y`q(imw_FV*T5Bqc_{Q1o16m5ux$9!(GBc%sRTfG0xm zYLor_vO_4PwS!qqcNBI|FZ*Xe88fwZ0T;KmuArdq=)kQX@jX*hsjr>(b8IeNUXiu6 z9A_6705~`}HmvDtLlUhTqvv=e_ni^gY#4UKU-qQ|GYMouh$4PB+k8lv)o*YK`;KD)nr0mEcvhL=?;n~)HXmiP$@Tj>5fI31A8x&zQgV< zuKpm-`S)Qt$%WlS#^9f%CVLpbWW6}yP=)!ZcX~nbGQ0<`rg*p4Vx0-e4Zki*&xD?4 zO9MNQfK{Rb{rg@Am#oZf|~rMj@JQ4L6d> zNmIYe02Pmmi^F`kpG&Yt*`^11oUW36|Nedba(B$-c#+qMhL$$`Hli1<@o=t23Tov} z1%#VGE&|jn*MPORx0jcfUr&9tn<;k{>zw-rM0k05S!OSQAfKF^j92P%JE7KNR2y{w z7Z(=|2qh?p`5cggp05d}%hZvwv%l)ma?5GsX@DW7qh_Ah6UNy$7GiZcuYxz5AJ^)? z7au+I-7Jn8;aDOwuTVU=jRxG{Rty=Bj#a&tlnAt$Xpt88h4!cO+s$jxl1oZrHhx7fg3T2UdjJDRgG@`-zy+)DMk z-Av3N(_wMGBrHCbbOB6L+v86ozXfKPsJsrB%ZI3=hUAXMT5V)Mk;jW%XMsUIiW|Y7 zMv+GiA3BKZv?GCOgV2y7mzc1X79aZSs_i%8C>Ub#(eObAN+Eplq&*|@P4IGW3lze{ zl?-s2*RI2oxnpwx9VNH&c(6nw&4CbRz)% z)QO&GCr~NAot@qIH%rB4kA0k@m9{s7MM%(K+1-74JniBz>qiD_SCft@{Zs(^ygUi4 zPln|MKA=EMx+8bZw7NG3VmDUWyde#Cn#kzr2XsQ zMOa16g_Akv6b0e@{5-#+0u>t@Tet$I9=+COYbceWAn^Ht7GxprVa~38^NR_@V&1g{ z;7^D9WL>b%C`3??sr4a+BOFx z9}!mfALKVDcir>!-{BS>A}ia;hX{gw;MK|@Sxg-c_sP#k8z5Jw8?J4Q^p;%mj(XJ|MaE!5wS!lk5p z4#AKwwR$qM=zfFO(Bv+!T5Wa>6DyDlfn_+n!(D=I(R-Xw$EDg6L)p~%0Z}qCvU|=- znqa?(2rA+_DaNH$0j$RG+xS|*nzkO0~{BJ4f{=pcq!McK8Hc_k!+>l z6A!3M!}osK>>1u=jB}EN;tf@a6DSP|eq4lv$@2Z)blo|a7vB&Pby#-v88#P#6#9+; z(LPB81e|>^^;j;414roGsNZI(x*48z<{_KWEkbElr0yTi{FV50=JACB*_B(ffJl}|Syl@3Q?HWJn?Q+I) zhlx%Ar%f5&-KThsCrTP=@!Y=*J43)h+c-SP01Wo-%DTyTUBTi zX4q003D;><8ZoMo@S|&Ik%7EZJsKHyct5eYKWVu@F%f(RlkC?5-N}^JqR{Kh#u(5V z$(IUh$_P9mFc2aBHj7ZpKp{+&>=2(IV@L&o&SiB4>AgTvNCOoL0#HzTTMHWtN(CZQ zsuO&pfsBL$9Ayjf`~-+dNVKPBXJ^e{D!vsFhd2m)AP@?S>L6DL!-sZo^5ywHhz~J9 zM;W3pw~R>&2Urq2F8p{a8RBrCa=j@b&KM4;gdLyugkQexZ6Rz(k%sa1-CJp39w9mb z?$G~$L_3l(#^PVCi^BUftJv53i8;Avu37bjhwN7>OXoq$4M}JQa;Qy*pRMiQOw!q< z;^E;5!An95xZL?&(&W6hX_O^&xuh_AsQ?C}iL|C8gE65AJH97D<&GgJ3K<$p<<-Y!M3-k{ee*+ zF+B7M=gVbuup16Ec?%47e3gACDWO2NP?!iUbPKJOMVQE8i?D%M&rA7<9D(<49Cfwf zg4c&q4i1%@@yNbcQUF<5+4aC<;(m^E>uh(5@t1E6<^uZqwRIe?8`lT|R9%u1$BPZ# zPY;JR9Y`3YrTR%Pv~yPnGlyG`x2I8}K4+B1X0Xs6e1Y;D4(uOjq=j6>HE>Bo*PbiR zZK5Aq8Be$r8-YFttDzw9gD~?%BccLTsA!?OF6v~nL=2Z8JgI-R0KOSCu$t*2COrq( z{PXFEA(ZAp`kuVJrJ6-tuZ*$j6a~REG&+5YwGv^7`-*)TT2*=-Endf2+^e0w4RX*3 zSD5gHZkE63zr1BUS;$b3|24k!X_KlzJn*Q@dVnh5LlQOy?l$*M@~vI@W*op*4Gdsn zl5}A*CP(u?y*?Awrnq1NK?{@HTw&h}H7)CWhrrOcF1W#Z4PxR|7jdNQ%*LHh=0QP0 z%cqPY>|P4spQv<#QBal>2e?NLR`{EV6}~X8=|HzKd(eR)Co(t(LfP|ju;TZ@Dq?GL zauQ@t`!m)b7ox6?@Xiw*Y;KM&sHb<|DJn*=tqJCM^Cswea{v`dN~%xzvs|+HWw1^!{CYw#qIR^j2l|U!)&(WiL~O zULIAGYuwwSF38{eq@EZfCvF=77$5;&$S0$0tAh6i(@87y_+1NC1I>qGb;iHjs^@)v zv9}23sv%)^)EKq%hiR+ z1x|;2ww^yVOa58ntbQpaCYT`#J-Pi`=o;hDK-Z5pq1@k$x@kqX3 z^5HKiL7CH~s#~Mb>Xqz*I8>T8DLc=QW!wup$npRR(A{3D^Di5HQ5#|hTlv;|UDQ~W zhfDKUFNvc?W6;AG#OQCPJe|+zhn8yK=r2rpnb0M}^ght6`@4sAwj@@;;1FSCgzGl1 ze zxI_Da5aqvP>^5gimqN!VqtU-|O@=_OM#|&{{#GTM@hHAuga1y%gNWN|WE7o*k8pS9 zIlCV)!gOb(sH}`AB-EzT)6+BCa=qcjeMQme{Ix{Ud`iUkI${(`N{9>mjq~g+qwFvE z^%g>}^1QXC-@`=8{(uy?X?7py7p?T8t;brcP*SsiercAX z;itC*7J(tRqKT({xyOeayC_oy2KR3fzEPdVLVa(_YLbk^sf)r zGBd(IV2aH|yh$d8^f94i&C>@r>itmb*kl1YtThwAWWVa~oMmQa^{sBrA0KUd(OS*p7-3({d4BS?C<`rz4qQ~ zJ{w_n+H8f(Urt}?`+U~wLF{i_a_S+Q?>Y9+FxC>d>xMA1tpYl0wZ6W{Vh(-L? z5YTG2HB-e-NJv;-%A04=(TGT#ER9T>Az@yn{!Rgh<*QW5zi^rO?W;2o%-l*Z>#Jb= zXYi3k;ve6Pkcyt|&ZsQO{pSTqGED3GnJiUr!^1H=_ar3kcPiJYKe8k}zR76_dji$Y zBF!Yji)Mw?d)gIn3~a&}d2iWeq|@z|KUKauZNOB6= zlMqz}^Jt_Z<_#wqBdP=$5y5<;TfGdyq2}?rNQkF2PqCgAUQ^mtzvCB7#wi?~oDeD! za(Z}C<>`A0&wQCvJ1jr@_~y>c`y|}t&H8>FgmITMS+#{YGjV<^zL>KEucEnNi0tm- z8HstvACmFlj7{`VLMn$9*?a@zXFxx zBwcrPbaC;$XUQ*OyqP-ppWk2ROz>F=NnntHnl@hHPfbm+u(C3l&hn6)}z0blg9=qt`M7n>@Zr*k!tA~x8-};mohq-W4(^&8DPKN5NOd`L zf{$-ZQP_1)r)Y)xXV)Y@6e55QGnpdi==k$?^i&$#%4;&dpw7T6O_8#3hw1>FllG#n zbA*d(Px3uT`~Lb+YF+4jXxMB;*cskO2*vSljm(fJ5h^@=zevGHt=@O%?`12otN{@0 zKR_H}FQrtNb$QSm3sF)4)3MjEF6ZjvRgdrPZj9CLiC}jW#=KMXb2kr1hwc0w)^9QsQm#$n&iPA)c?(8D@ zA%bp8!j2CYsW?VY85y{xdj*t(dVk7st~7r#;D^fw(&Li+3G78}9`DamNjNSR^0=M_ z(Nd8AJdo!$sK7Bh&cmt2^|!vnbC_$qz@}htKixdZj|tE3#Eg!?QO5i1xHVmf$!pn^ zJ*_La(#dPwmVd!w`=g7_hzDxJIT{mu7_3EKU(+W*|ReGBW*uCh(4-0xmoWE^9jBl)YEl;SiSX#5m| zn3%Y%*=>HN)uh2*>ikv6H^op!CMhr5b&xP$)c@jFuL;>lR{t!3OJ|4B{~GHoe7ie8 zb%_f7{q+p26+!@nS8(s$$#2!C;RW(Xw(^M!D4?i z&uYUL)X0zG$6*bp_ZVX$9TbPEC09f8X~UHi2GJ0!?yQ3XAnvm|Huoy zxjMmWa2THf7cW0gcdQB4WTsq{LK&5Ib*0(+y%zv{)qhmlc{UR;A=m!g81en%3{}#v z`KewrLxw?qkoU2_04eBmsr*gLZKOpmo8a{IyH2FgVp^`zhjv1i*l8C>6Aq@=BYMJD zh?;&vtB(Dsf*d2Y0h`=uPbg0a_=jRQ3f@_#l5w7|&Azd$yCY(7)@EpvRDO zxv6TVZZB>!%_kT9w0BK$_kU&lM8J(RxrvM?CKkRvpXKj<3e}o%-H@R`XN7;pq7!$* zFNlN`idMx^{Y6N%ItyufFN|3zKoj^ONG#oD>|}#&Uj1Q z3APZpk0kpP9?Xp$0XHl|=h#rrNqTdLmRsJpS3)txK+5$iUJ01pSvKwr4rEFP;UgXt zO62?Hjfw`Z*`sp+`*r2MfB)ld3$EG5w;~U{h92nexSHFr>p!`8z}l6(DP?}NW3TRp z_rike0YgTV+Dd$-^_zZfT6>p%14OsrLak*uRAfMT`GbTk_v?UKZ5Rl| zuBXf=e;&z34;fdxZ6e2s=)*Vp=Z6CTX6>jWP zgIIYj1SyI476*ROE<5;N$^U(dVIaYdp9^A(EVH276+98h4a8Ca0nFa7`G)qQk?*ZDkBE zHg*7Am~B&0tes-qLi&Q#)MP%AdtQaxrEd}cPOoy{1EnFLKA&M91nTp(TbGx&7|qRyt#3c3 zmE6{daJ7$Er)$rhs-(d^36RL4bKe+K{7=g3;^vq#!`}Lg3|998B!Sws#`UY_h7D*` zkVLcdG`FafR4e~yj)2U}fa4#8PQ%9oOe4{eF9W3t##nD%_BK-pSnK*-&wAvWV6Qu| zrS&9Kt*jUZ?yXY^4M^I=+gmII&U01s=#3+>yeYSb&3E|niF-SdGaN-~cn&zNVpI_ZShxi{ z%I{(I=pH|*LPMY1tXg^|a0lPtG2mbo8hpBs$-v0iGdv9Q6uvyPQ+=6JJ-V>Ku%F(+ zXWl~E=CchEbD6pPDPkTLd;HBX6n$pLNM93ibF@vNFx%S;85tQly%-!+f0mRKD{MEs zKI9+xx_A)OH%?4W{(4he+ZFn~(aCKfXh|LT^c=^SCms@p=BJd!%e_cGRP*aCLM)`BM_(+Ogm7@9BZRV~Oak&yThz z${*a_|NA!zWZHc%@2Xj!>@9l!7b({cd2s%nQU%ZsOTzUVMwFE4LM58%!oR;1#nmSz z5!aoVKf}kzf5a6dq#@(Ct8SZ(EAg%6rl;nE9%>8dGtw9=)UeZ%Z?6@JpWt(_Hdq`< znYKfh+eB}Mapl00;K*1}8BS16$q26P=Slxaf_4Dsl=ZIiieADU+s(}lbZ3!nbp1vi z91;@fhbbPs8g{hxz;Qv+MvKl>H(wUhaElns`%?q97X@f$@Q2v6`qkd%c)<$z^71Jh zCMzfR=+9C|N`5{uCAWU~{akU-D}o#*Otbl3Rgpi+A|k|aerahbJ|0S?y`}E(nP3SA z3Jq*nlbEZTAgEJEoURnmyJ85!|-k<4G=-772C zX~mG-d;MP3(Lu^e5s`$RrlxcPDDUmB?2ePA_e+hcS(p;&C5Q)-na7jus9}Od;c|&M z^z`)nywx*vCf9S97bt9z%sQChJV9Fh!WC^^Zu_KRq!{ZR zwR@lr^XpiN0Po{x=?C?G7-oHA;~#A@GBWA_Pc%XYH4%Z;3Cr^<(#SpaDct}8;L1#} z6bxz{gfZLK(CFW(uV`VxXxt-vx>*nwa32CEs8itedFBR~K{j|WD^Gt3F0c}!2>J6_ z1=bzZ14-73?YNR{h!smnX|~ z16O2R0guPK9w$arx5cWn_v8%1llAEZ`KoXAaHzckV0u)b=I+2~#xJ^uhVc0<+Xrhi zrx1fX0A9HH0&Ed+jT)o`Y(udscvwYYS`I7drj~ST9pE_seG>8U@q#{EC`o{ix>tRA zue#74C?e{yB>Okm%*^aM&FbNZ=dYQ!b+$v~C^=vO-5c@?7I7HE zu6|JGHs7pm_NooH+ zuRfFm-v_IL-WX?|<4v>0#caLWa{(J$In#a zGc%ar>~1?R@ZcW*am7=BR_zOEXi)c6Ryj>ng)BzW7xKho-1QtBWx)12+ji?mi98<6A-2ZQ3sP1i z7t8AV(fZ9@eJ}FoWIzXcNQ+AjTkp$znobT*4zMx4zP)GvEd1er?u^*H=*Zs0p=O!K z<$9mV8pWmw9G=w}Izuw$+O3V{gd8hht{3A;KyF8Y{{N05^v7pHm=9~i^*-!&6jKN* z*^~XM=sw5Lm6beRprwvjIx2a^L~tY_1s!@p>lWc z6gGGLTV}g2-T|k1SWUCzXCvMi%=j%4)J^W9mT60%Kz~s`SBl$CW)lw zPGzNygm6AjFjtW~nyhe^?*qyfE0uA$x9M=4=s#`>NUdc@$zq|y2~kAX}?_r^Vg{=Z9LccchF$jg+lMi(%apptrVf{E+(vrt;q&f0PbN7 zBqd{6yz&JQ3)5L_KBA}Gyo_&#A8qW%;0ktT!?v7JZV(l^C%5lFr+F97vh;|6S~1MX z$Z@`g(b3UyFi!FxHzfpcsdZsL}J@%0f|Xn+DDs zsavCuVWlXa0QYgw_WZTQPOVq?=3|I%;&Z_CEAC?xfhFH-U)6%7q-R8uRaV`Ww3^XJcfWLRDLlfu>6B0sL{ zGVe@PP2rVSR35rUN+%_;kX{yaG=#t4zF0k_#a6+8QCI)LtY^=UZ)&>9X4=BS z<#&_IXp@VZiuu*Annf7RTtpKUN96N2EOIyA{PKRt6k7XOi6?uTf8}@fmpV4D_XNkT zmMG4Px(Q)E&9)0XhwJUQgH_9;-9IKqPqeAr)N1`qbQKX=D4oHE4-69;;@HI7CyEe7h} z)eBdQc*#eT`x%Vw4E`9&m;LpK0|)BFEkQB>zEeft4o(%X7$MvCw_Nr1>37Wp#WN|b zp;HgRZ%u_uw(k?dzUsR(C4lb|khi1rl7rv)Y(7ZT2A@=$tzAKS4l6GKziqPBoXn+x z7gsFE(%hOP0+?jogza`lV?}h?;8TKY&zw1!CrS(a)_EwcR6+=$uGbnU2)+}9KxyEE z4eIu|FeN4BmEu}~cA(ICHi!t+-+a*YkR`QAZ2gH}tg4txxLsRgMbKw7vWk5z&IHr2 z8wu=e9YE0c-z^V?U56Yqa8#Q^*)=m`AKHCZGB(bVt^8YV%4O~1LXb&zY~JdFFRpt= z#L!?jS*hU5WRF58X$F(r&Oi5dNj>^$;517)bt=^uJS?@hZ}jDdE3)S;E{XNd!2LS^ zAEAM1KOgwkAZ=~gM4jeH#qa0f>3sV~KtKcE3O;L5zI`(c@MbjKci4m@6eCvN;+uU0 zwm$?biDbGO#eoy%|CLE75F>!W5iH4HWh*V=`zqs+zj=(hcMD>-O2R7iaG+1qGi4lr zM`go-N2Tunpvm~}BSGNzR8zrlh8p)&o-E5e!rLappx7g(9zciC$qm6L%e$=}X&ECp`wL+!%)fCo{ zrybfUiV>E!8#-VCkoLDr*N&k(9^Wi!QK+iW;)q zTjSJT2fi5lKIQ*w7cqiIEW5D!2jDJx`+M`2-i@DZ%UzJP_U7TKI?NWft8{jA`~8~O z%#^-oEjt-fKli4n2uoR6c|7F$@{D^dSisU_MCCt^`1*f-L@I+A8yhzNqu%~%lLn>j zuO*lgT1b;CtM-6{=_~!8Uj_y^dVT@j&_u13f^n-C`!WYl&v;~H0N2T!sFvHpt3kIpNo%#Wf5Ov0ACx0aIp zKfgv7-7SQ4D~Q8zPx(kab4TTQH%N`4JOEwa3S#6DH?bd4q`A8w=YGif?c2oA{_lId;MVatY0|})#hj6kdociU?l@b+DC+hdZacXjy_reAXj|7#N`!{kaLM|^?`rwSL~1qeJvJ}e!E#Mm4A@O5;BrL=PiW}v@?qAem=Tk+G#L{#lr4`cxdp-O((S+*?b1tC54LSP zOc0Tx7Iu3x>9}k&u#yTo~PHTzdDy;Fc3_R9@5~dqAEa&DjUki2dVuhEs3NsBfjKAP>Vq!=oU+ zR6SX*Xzwscb^@G$ZtH<{V_JA<7b5Nx(7wj_C#TJ>ZG6tWgR8z8`-sYFgOpX^4bbx{ z1944^k?yK$0SO7SJod2ID+v$FCmQ)G@qgHpS!85nPJtWVrsbbKEm)qQ$OrhO0AzvV5v#jho&OUAI`4qv-RuA^ zAq099JBCSp1p`bg&V8|s9UoRKKP0n~tZ+?w^^hs5#^S@kCM!B9^y0S!+3@nOUpz>~ z7#OAX&Gprt>PG;)8FJTNPfL-5bsGM7hE)6scJdkP`WsDL)o&0iER{jxt5D6iVK>*e zs=(8OPs}R$FFz*AhF2QZ<}cexckt_#5|UyR-|uh^I$Xy`olXG1S+zPoJR~nv*AR;g zN^6{&oTLCUM{ICR!~&?sah>v>%_Yl!(@Oo1S!v%?fOtVhFOy+!bpo`>;FvxjrGIz_;f!>2p_|Kyk7LzZ1PDt^K? z?bSnrr4w`*V|euFF$#HxgU4HK+5{_qYOb%J)-rezuEWU4uodKHUUVPNK$N8x1NLNV zx+=3Jt{oTgXeAsIYk7aAPtbE2Nh9I@dTv?$5k5AB%GzzhHw~T)s5boZ5C^*Bl5XfPj8Nw(31qm9I&7g3d6S-u++);? z6vOUU9#(Ece4%sym#}~CpFgNNSHrT|H+pN>dg^z zS+%%kRtV)4$9*WQ?B!CLswQnxtcmNR;Kt^;1`Rl8KO|I7)R6!JEetqTKD!nbCiC}Z zH+p7EF8)6F>sc@5)=7cVb2g9YbtGnt@JmiCcJZsrGZ8qCAt9jOIJ$8R z#0o>pUs99Ba6Bl$X4r0kDl30<`D%h9uE3Tp&Uyqf=E$3@*vXSj6RAn zlit=El9k3E0q0t9q*7af4iUvB96dXD|NGR!vu{Pqzx=h(W%mG|2~{3>)T&Gxj$Q#OWhe?= z#|D8(xG#`eY5?&Fxg{~ZxGO1o84w@6RmkCS>x5s~uSe6?Fgq$jiHM2wtBL|2h?}^y zcj{Od7Ga8RtGc81u~))t6}m+($1*w9sOq$Jw>3usUsMn6WFL0alQNPK3q;8yiDlUJ zN6Vs>l47%YhSH28HV0sXZ@VW+dFGxjGWFW5UR$A4qKMh$mA>r0^vboTS`$GCzOWNt z)DT91Sa$Y9?lL_)NA<%J&0v0rlZ|d5=AfGb7iyyPKK)9mZXs;SYjccGDfAWCd3(B2 zN_g_GrG2?WE7!@8QZTk~mL9zCgL>sk57FUBNDRqKiSQ{Z0ZU$47V@ZPAJO5i93GtS zFc3vX6)mBd#2|&CTdAS~k!~#`#pbrt zG$1IRDN-YOIUbIIY_K1_H-`~JiYmw4;Ab7jmOdzZR|d~Vrc)qnBXq8ZY?gu0AGb0Z z$r3qS^z{maa>x}e{o}MyoxlfpbSUO@9wVw1T=?^Qk2K+absj(DA5q5`PyE1VTj&Tf z8Ue@5_doA@0yrZe!~Xl^wQmFD2C7Ib8?`zUW;NSqmQ_$I_?DGu^xf-uidxeEM7#$a${)zxHps)8}6FanBE6w{H`q~;Q!bx$>B2$k)@cc%DyNfy;*XK*Zva7HW{D*M#_0`qYn#3{uC-Mxpya2>c@tzQvGP*e7s|)sB1Se($9_%eT z1qy+R>gL?tP-6{e-9T6;LzO>0x7v#@dSEIlh3sklB(%@{oopEXxh!VmvRsOL`Q(rHB|9wIG)B@RLaIh#{97=k zaUd7Ly^qG-6V87|jIe3#-Ko$lG`h2@{1(J!`=7-mCLe~2Yw_iZ#**d9MCl#7q8J(m ze-(AK_>eIL<()OH{QTYaTyG{W`;9KhJyXTMB~?7fG~h?u#_Wh|nrfo8#A*u8HTV_q zuM%XTzqxvTOy}iMm z#lHlPtZ{eLHyrg9F+=iG1bhiCcOoO_0|Ww@QM5TX?`A452DE&vOrs*})Y*CE8HpHk z^fgNQFSA;C>NY{kP~%C+ZnacqOlueQ$fIja=Etm(yi~nKLb$Yy6}e6X$*O8HBleDY z#4&=C_Srg^+XutXtv-9%-Ou&ZZNPvYg;vgbU|c4O^&5P~$=SXbDK5Rw`9}GG%}+a- znFbzgVuAzZ_(!!lG*}%@tk}cydRVv}aTMMTZC;6Sx@2HT z;!7_wmWUxFpD57v3a`gJhEByf3wJ>S0yTi9Ecr4bk0%4pqPfC;{){ z3Vt(mRTINYgb~V7MBqR53#0r+&Ft<2()TrlFzwB&QIVHcn2aD|{}C)@dDS)Y{1t8U zrxzHQO~uFvwr-+Jf(>@Tr>OnSGh>8RC)=Fen_iLyW6V)XJfh_27nvSPU3BMHk=3ns zm)JaLdZ_r>KxD~%al`{j>N)#jC8-7Dh-g^!%I5N-8s`3XpV^irZFE_Le9pIU2)V~_ z0ev_0bB37X9BN8|Gt=CCpbVo`?#Bmz53C0nOxct3a zsu@o&yj(wit=m?uM)FE0ygkAxx2%wy`!*3q31;ZQ15ihp&c#yfwHReePvlp|?dEJs z#Ix>zW!hiY(SId86s5Ja0+LL+T4@U)F@hwD_ayB@1uW=x3OO;wv_AWwruv}_@9d{P zMt z+B*pwWyTQ#u-C22xNchakbY)=hcbI#c$~wkkK$vT)3q9?Ben&abR1|X8L>Wf)b^#A zQl$+jmW@^;F}Y57Supb313-u-WL>y?W9@V9u30p;rLXyNOQL6N28SCiP$NMArnS3I zynv4O;gR`Zr%)@Q=rbtb{mJ{lh13^b;IP8t3_C2vK&Q98)&@L*iU5%)4MMcx{5u4(4hz~|Hp)xVuYz2DWwaG~g zlj#!+Tyn>k=;Wqt*uvb1#+14jbk(ZyouoaTA{S{w5rGHH4Z=6_g zZx%A5OxIfXQn#%7L+)QsqO$Cn_aX>&m;OLmV4J%LM65wTskp~dyEeEEHAUSQXaFyD zoF0M)Ti*Io3?(1^!!5N5j5CCK33RdN$J~yzORYIsHp#Ig3VXXA8S=8`WIm9_I@5_oTKy0t87DQQ|Z|aOHW8P%;ke=Q_mjg~}uEj#nr3#=K zy=x$0DePVoDO;JIu9DmFSh&+a`}tAqm!$ZAbbOitU9B;!xa<=jSPA7|`C|;z8}_;< z8H}k|&_+x~h0ykJtj3DZshlKH5V=~Y%S$Xc>-FnfQ+FF6&^CNr??JSlhvmr5yaEOb`uTC27{flCaR8fcz zHh!_afjy3KdT}H!{yfOCO(De)VA6-*;0gwk^{UVaTd>JZd z%83FkU`lMA<4y}@*CahNGfZ^LSw44kYo(;6m2LIek|;6lc=RbxzGltL&=x*U&zY#v z5W|F(j+)_jZjK?-cbYmO7>)5=zE`kaX&)vXrbQCHfotsa%C_F)F5SL)iHPN(V?ylC zP_BIMs(Bhq`I9)yy-}7AQ4~+P6Ub^8+x^HuQ7qP%Qz_H;kGu!-X4!k@qXq&%N#3q} zFs7fY3D)}d^1(Q5U40Y9-|qQ8%F)-ZHl=>!608y-g!Kk>nxG&G29bE0NB`ZMO z8&hMTZ5U9zJMQ^1{xwJM!|!?BDa9t}?duyNSf?CgH`snhL<^4&yC$?Fn2R6X$$5Jb zeXkbFQt9pC8QNxcNzkUp`H9la^0gZRRbIX?jj(*jdl|9{Gyf3GB;Dkx+=8;W44L|9 zGO^UbsV|VH=Nj8!=O68Gos2g4z%LkfX79La`cPJv6_c^;C?O6zca|^4X)P{3@!Zae zxjy@vh8)s|<3nci#pU-xQ1@>CuV}uWKTFR>gEN`1x;xCQPMJDbl@vmv8CnnVYQypG z;=qEO^mI?Y6fDgaC`$95@QC5VUw`=buw8UNt{D<=x`ZF?%R|+@)YZ)5I>=>>kr;IK zTmV$wXf9~2xy}dC?HF;fEe{sq`uD3|O_iw8_}ag;PAJ1OBEDFl5`2t25w8nqikoo0 zs#H0kyYfEd7A-f$-W72Gy5&B!di|?9_lpQy--3Vh>Idzj3SZB=x9L-8AL)2ql{<^u zjR`l&>(ao>*(rY5vH{3j@Kubkr^qN3oZ-*G)e)r0aO@WoU%Lb?Kty` zC9$~{u@7820(?Ik`z}VcF*am)XUlg4Uh^Vo#GKRSx=VcCRfSQw2jz-QPt-(aCo)vAefoD+J=MTx;tdz&$GpCpXp zqU)@spHe00TDx2v|HktBv$Tk5b@atYk&qTH8*)fWXDEOZf}0rqCDs0u?pvjmYSbH_ zz`5p4TZqy1+h0b{CQqUq+lgFLhDm zZhUy}(S5pt_3>x!_u}jqg=T+xAS`JJ17hj5fw0T{>xo9YAR2l1HpvQ-fPV~zygAUg zlRvRlB#|+E>!Wp~5#g!cI>ybOIG6GCLEo2CXR*ZEIJKI0dyo6QIs4v0C!q-RMM(w? zT>XUc@R-698=Gkfw~Nveo3d&u{70sHG+l|eVpEQ<^_QLBZ|J2vK#gI=K6aMqQ__hB zC;_#WAWn>gdUy;*U%mH<_4UZX_kPcnsdI_xiLD)BFfn zZ^s{{27Z~BTAzJ?)TK$3->9x`YJ6tv05p^jLiILV))dfv5ugWVw?y3Syf2DzJKOanETunQ0-a z)V%~9RKP0uWZ(@pI$XT7amcd%Ags>sn{54AnJ3h_)oU|~Y|S1IB^bObx*|1ErsW-Q zeH4!YefYuafH#5eE5%X=QRJjiK62yDHkf-g<&Byp_XIXsz;g?zg#Op(5 zEGDSB&o)?aep97&MOvBaTF(ZBl?#l}-7i$~k6w*wG+dL`To@W|_t;E2Gq?zu2OVK@ z+(^?TF)ie>A&i@BiAM^An5e4f?#N<#t>MeD+oy@o0ji@h-m1lmhe57RZQT|{^l~s& zpbL+}ak5T0jmt(o`P-n^`CnXR6x3YMmuZ!8w-W zV>v0dQs5z6O}ZFLldepTo?s{oJoR_yAQ#sxgr;$#1%Fbf&?OcUBY-e7MoLc=D+WFh z%(dAK?ZR-JFV}q4#yR!?*(_k*dLNmwe*R!ud+|g7(75TV7YT}BAqVO-V?ByYz$um= z^?P3Q{g`W)sP0B(>!D%o%bxwr1U^wisJv_#PO@$RERm3C-Y{q*=XyYzM7b^~&L~B} z7~v{-_smTt!7DO|u}s8jSCbq*p&=F#3B(HYk!SX9Sm`p=mL1aCdD4{6;arili))$} zAllZ}R~t;@O4Zm>-e3gHjf}eYF#vB5pPRd+W=CTITnzvPPn$qRg34R{zvRUM4Z}o0 zkq`&iTBgv;)B1sWmNp!y?aWG?xaW3>Sg~$d^iE)Jq1V`W$`?)i3QXu%QLY}Y)-#zR z|L~dbnJxcQ3*ek;el}};9btTgQ>TMnTYiXXz%=-C|M$DJM%L1Y35FSl8cP>@_frrG zjxN-h-0>Kyl7^~-ui0ImYpH0^QX22EM>Y9AKdJP_iV?9Lw7!})ugJmeznCn2{ZpA0 zud<$Lb=6XXuJjbuZL9(3J%Y1@3W5vLh`vI zOz??UjHjL1A*xlo-fj$x^L7HJBV;$&^lF@S!Y@<#)`HA(lKKl36-nvf&x}y66N@qe zZ~Ej(s=ETszS52n27GT}HQHUkzbSd2@y*a~Od@Y0htin*WpEqTmk-NKm{I)yyrxa- zte1{HZjb@8yxp3+uPPEUJXauKWu8N$2ZgRbK$b{WEEG+FlauD@{`}?HU(Pb@PW=u% zeQ$Dv>kH%n)sLj7Zzke;C5c+?oVxm5h#;MEXJ-XkG!E~ z({pXwUyqMhUe$kjU{2O^wDi|W>MnaPM_5ens^-19PZU%sp$;8H^u3c`!>5uqw*S_| zkU!qx>ggp;p5o@=g`YgpvlMT-+TNDZJi99JcuT%SvS%DR6fIQ5K}U`_P2ddEcjtGR zNz=vuSDdQoTLpu?&dXc#Cf|a=QHwzQ@#AyNdtixl@mnd0sF4BSj|4=^og1!uDokFx zl*#_y$!vbYbUwLkyJE45iVDg5aF^lKh)|7^3w; z_=J5zU08ukTMYo#DQjEsH|6O@v%IC07(D<&oS$un)y5I}T+4yNb#-4gg`}w9?=&3P88BUljT$r|2vxS`-84&QGT;JXF_V-{(yU#BfP(*rYVnpYZ#nc`XEN- z5f7Z%TGARiA^((9CDPG#eNM2q)W2>O|L(HtZ#CCakuIvv@3$$9La1!U{m*xb=Rf!h zEC2)=sBJJoyRtjX5Ox%x-`f!BXFv@Q5D$t?`1Z;^M!`#!LcV=t_jxj7EB6or-6GrP z;h1|}rBT3r(xzW+d?5Ri?35&};?i@sU1mAxN;$H|Ko0gW68p#+;7gV^N_|wPv!8Rm zmy;w%CT~5cJA6pRG7cfIh0bKuqxk$K)4ZlS)FMj%Zd}~I^`g@HB-s~|P#;VfUb{l? zdWk^3Ld;h?o>Ch3bkpAovBBY0nPX8!-oyyw@~knXO8wv25+Mw+f#2eHM2DI-$Mc^J zb5Aiayw(%@At&>``*G}gmBEZDck>QBvukr4EWynCg?IY(c%~%^UTD3Z$K@Fo zjTC3>oE@S8pv9fL{=G1oE=bGSSP6>TCXB~(0)!4RB$XGIgOYm(XWGW zE1pQ~n`W-#yYz)MCwKUZg+gWAdM6-NfmG^GfUx-E8xKub*|G!7bR5Fpa@os@k! zs~v#$gmG-8%`kIDs@@{Lk9evncdzJ#cIU4^^P3!5&c!Mcl;?5;pLu9Drm*#(q8Oo6 zdH#inZQT^Vnxga1f$^*wfE;Q2yNy1v{|*kcS|v&YCWT7@?YFbxj|Bam4^Vn{adkNh z8?@ECj7JCt9zV3V(>FM)meP2lq(%{40X=H=5#6K}~UPLAr`|)rW7A zInr;YH>AN9{G|D$pgQ~|SKYtR0s8S#QR>T9OPf?UShTY)R`(uv)=^F>Gw%&N)6HLXyF;LeOKj|BZ z-Pp>; zkP!^>!A~p;Um|{B9QuUeC#U+fgpx&6BngdI-G-_?VviKR=`be#68qsKM|R;rZ}DNo zY^`PY;=Dw?DUxh3Q=#UPGadW#RH$$kw2_cSd}rxeDpVqXzE zANCtCHh`=cOaEy-$hV9O6|WwqPD!xKzLwr|a9p^QDYvk?(q!0~OV2nFM5t>u1@eaQ zp!pD6Ukb@y|CPu8o8cCDm|odasqpjJyUL?KrF`>IC)-7<-dpGC1jdz(c={h7Haskf zuH{VHrC`^tSk;1us!m*Oa2ryl>+ONAe@_!lt<#INQ5;=w5vgsDCz*iE4o`cl@?0u# z=9i@Y+XwWAZE$$Oi_H+7_;DpOvmnQb;`H(brT;e$2Zh-k_6SuRxwJl?UQI30$$d{A zJf0W9J?#?mPf8QL3{v8{UlW0nfj$b_v=^!65M{eZ6_JU5zKIyW{8}sg zvq0W9=b-9fa`dwWWS+9!fw+h7wkl7LZ!TKIh zAmy$4+Ouj&YZ_yahowRg8j1aN+;4XdGc7&8ovwHm$O_on@tz*<;sazKels*NB$%kk z3ZL5gC>wTt2|=c8Vk%-B!W_RhR$x%ChfK73J7CfSIb)@PRyWq6HH!y6x08B4u7G7b z>5XD(={5!YwT!DV{&9Fr|$FD_}Kp!U+*0aXY}=bGmJ877(|I0 zF}kSHTl6;3d+$AZiQb72L@%Qg1kt05o&-Sri)Cf5 zbDg@+-rxP%I2CEOARAPZPwvmlE@||{uw4(*PMC7)7t1j-dh~0Pyu`S%t#h!03}dKek3nm5X-c@4;fu_Fw5Gb@9CW(M9*@<$bivv%PcW zZ#B1~=-{LxGt=EpCqni=0Oac=o-+oisgx5~ledW7Vn7ABT<=JRyqYu)3Q>*?<50_8&6Prnhr zK`nE<&t_ee*{$htQ#?4yM4`&KpJJW=56w)$;LJH|EhU4bkTH5ioG=TF4^n?pYzCaP zCw)$TdP5+X-GNMkMAr zO@AT{F4=jnNQvo*rOMZiJ-36y$)xpdB07M_KU55VctC=lUP??h5rx@H@HZY6?9iYQ zN1nlD0$cmg9ef?&cX?VVLd1rj@kMYyJUqWdDK8V_$qB z;dH|X$LO$>hJj@1-gr4ai`6})pLlhFEnR*xO+*??F)}Pymvv_4J)3pNxmH_TXT$cV zFK~UlMj5&yZ>C$U-lIahawjP^fA3SvytZVv_;DI3=Y^wwo+}ZJ;vsjOaYabPluR|h z%yK@AJdAu2mm)?2G3)Nrs%4T{dMta1gOs9$dRfYy5lbm#Y$yVR-^CgOf3+$81tAq~G0T@%LZ$!l!|H#S z(Q>{Dq2wbArV&(k_S#Nc+8p%EeEj@Lu80qsEr&kAf>3q;)U84tp~{!Z>;Lk{hUbHm zC*q?-%h!b+EY}_E*e^m&N_-SdUNjtNGI5&gC_dLKhbawH{EtIw6y>|fM56_$Dr1&4PF@@5@q;X z;4vckx9HkFi^Q@$-;KmM=dp)dG%Z{JbdP)i=-uiDXHI!0Z zR3J=pW-rvsz5qwO=g%j-cj!9tqB7TmoED$gmV29=Y43|a7d+mQggffFht1>IZlasm+nV41KZhS1S*MF1NfWAYjk7g3h4J~2WLqegX~ z?R-;-q(_cL0HX|+EY%Zn!5|d4lOcF;%(1RTUm=|j#iZS8yT>GtL(kYtGPAJnSbW2A~v>0y=d4(XRbusEcG zeQf)WG}X3D;^=1v+0*eS)Cea%PRC z+Omu(z`Xi$YlN5m(XHZk`8s=VYvffH-VAEjmC0qvRwM-EKaDZ36WVA?Q3JH|2Nl;d zegctuZacclkt_r^HXY~UbP_(<=bhzG!`5)US5KYpPdDVvT}^F5nvC0y1_J37&(5|d zY4Rn5QdW)<7WWhgwpnFr)qibT0Uvf$i%i$m^MC17sB%(&5-%4my$_DlkfUvTdNORP0{1?&!bKW5G&1$dFxNksJ_3L z2+j)&cXN;~zoz81R$K|z$cL=`R8L$%H`&Q;X7S}lhd}UBF5AKO%Of-I_0WdtyF9UM zQ{}vTZEBz}Gh2zgDf%>yy`p5X^Q~sH={*0$QQW4CFglJtGB*5?S^+9_k3Ppfcuj}i zPK7B6@>uTJ_W6LoEuOf9X3*1w-E%BY5_}o%?7aD%K@`1uF9_GfX&rTI7{_HCmv`m$ zQ;Znh6eiP3%JM^@nsHJdMx~^p9vJCVCo0#q^Bokw7e8PnY+OAuw%tzf)?{(`L@_3% zH(%r4%Z=u0e3D2co-7sifyJ!*i{W1~gyPPc`>}?&TrO8oy-@!kh~?{nk~%ez`A`T| zs@O}PgE@o@jLqUE?4*hUkBDjhB?Y^wVJHMfx3y%)^YXH^+x@!y%g(&bkvU~>bM@dj z150GVns73UTNtLfG#PyFFA&crq!ZuzCOw$ZlNuxKB0A*n;C#_1i|?^ne!%n}r}@PCeY%)R!v%q6L{15%7Yv51v&c)Cve|=>MPoa?}e9OefaqxA@UQ zFYilY$VhZhpqf;6MT&}~t)HF?Nux*E!8xB{0Viv8A$e!0grw3Z5wKzwUWyT85f#vf zVg}`F1!HC>;OMD9;K(=73+HS9X*VQp=KzjKvI2TkICHZLW`MQ*{C&9n+xyGou*^&d z?UFPGG2D7bbK!a6BQ2v9e?hBv#si}UQZEyDbuDTek+u_S{w}` zy%^6u+?=WmJZOSI^kpL2Sy2UfXfvksmC15{T=Ld9`mbumLPZgF^!r6%DO?~_y(+T; z6?r3g5V554@ot^r)K*3>#Y>lWvu!*QNJBc?t+78~|B=PyU!x#)Z*I0tkv|@T%c!mGpli5N~v#{_IbwWyb#ih~6Sv_R_ z?)rtwynO!gvRAVGmq&au3vra{Gt*FCxA33hF~g*Bmb)3(|0tqG1&G`5ZC6Jw!@7TE z@~w+yNrVV!bH9;|B*gFk(#VFOM=7tQQVsl?+Gu|i-1U0l<|>P{-;T8D&lGtqG7>#g zD0GkmS%G*&{(|h_*-9=SMfLJ?#YL4R+=P2+n!NJw6V2@x8iKVOue1knd6>)*{W)W1x+Ag{zC;DQh1*Lith2US%?C#vl!jCEoXZ5OFV+BrInOMJ4WPPKsN zZ&6@J{f!_=yMdL;;S6>tR2+41{boRhS`9@&u6&8-j~a=>!?z!y!MqAz7oPT_=(%2E zvg2J*(a?zL4eeSdjqsQ2yfbbiOCRS5{fGB-J+WVjl2vSD_%%~n?onJ)QWC`@E}96R zs|;ZcmE!yGeq6=!=U!|IW@(};Be$=wZ%J#vjt~trYN`+dLwDvtMQE@#GNL%u-J*1= z+O|gYVVjJmU*GY+s;A`7-&RYpsMhT0D3ZWA`mS(8T+FyVu*BqBehO=as1XOCevjyN zGZX`tLRvxCPV5xLVt7=7tnNz+!ID9ccG^hgoX`xj$Agvp8QFckcvU)q2rvPMJZ9iA zHCsn9$*rWUl6s{w`oMlj)iYsRwgGz^hc|1T{NyKYP8533cr>iM6}zz%0l7E{1|+IK ztWvEj*DsXlAVTgN|J{1kHcKyLvIt&I347XGL9u*a_CiGCi+uo zI>H0eagQb{ny2T|{b;bDtgPdbU~r7l)uh5UM|xL2(pH@RWlo~#Yx?=>8 z9-}5OqN4hUEG>tu2BP`##FL$t%6Paya&0(dn=^&T)BHtD8{6WugQEMkMg7S1as72h zrbn%+F_P^`{=BD_HiU0IJQQXR*s-9utjhJp>=fhVEb3nG&MwLsiMq*kr~ii26jZ3{EQl3fg^`*wud<>iX6zUd}vBeOcQRu4vGt zCGxL`W8;r7jE7RsgC)jGX0F9a#bO~o@s_mo3w|Y#sg-;)H)_<8(nHa+iUJ+^I%384 z!(E8tN7H&%J;-)PaUo(=813sPetfAkT{fMYwHi~}cr`Y-<HS0uE5F?u`TPypZ@+c2& zQr;{twXvV&nP%sK%l!bU=kF(#^7(@`S|d~t$A1XgzL3Cze$Y+3Ow~dU z0=jH`l$IAYwdio{w~W@T|9ArZZL32KVHJ2Vu?{o0`t#$BO?5SuueP!?%|ApdJSc>Q zRFv-bqNHZtDEYf%LEndlr%}kq_=3#o!c1PTQW&fvt*8yCemCAvZamfi4>>|6+bMy5 zSUE#yH{XC&V1O@%7_#*gbp&#A4x~NSE3kuxD|3ryN-bKEVczkG5Jo;g8ruwodIWjrfDpSmCaeM*7QP`Z^5k1v*)NH<^2pMAkN;#BVIwSKe0;nm zQif?e!@_~Ctrz4Ftpau>tMWv%ixKdBu<|=hV%`-H!6WAg1u`|rV08I;HH}cR=pneQ z-uYz%=uC}}q0DHj|M5)1j1&^7pf2-UkEo(&w#y5GCr36Equ*i&n2CC0gA|Lqz#+a8 z@V)(_*IcO{GPG&K0`SNWx2IU{(s?J!G6W+rF)EB#KEa8 zkTzWhF1Gexi=q%oK*8sV0C|0TkeFxo=G{Q~;sxoEXc=JP0PZS^p;-7N}t8y%)lfMW!z z@gz&}?c~f{gR(ae)w%D?pY?O~iLsz+LThh%#C_i`BOE1$X`zm=ftJz=AVR*A7Qa*k zV<6wCkLAo35$?t@MknbKS2fU^NTyf{k>uVcPZ?F1+I#FR)k|25Qi->>`JsN2D^Z$d zv3wtAKdV|B5i+oYH{(bB#=AXwopRIH!@J@Y%P&3=U}UUp10&Vu@ zL+^b;nb&)kmP~VvBJZo5OiHc2rgp6|Ks^4 zrTzhY*?qtXhyv4d0w5?cn>kH8^qrkcr;b2jBAGl9Z+d?IyQ&$OAbbI_v$J~+$!FK& zvdAbXFiWYuM}tZ{2F;Oc>wuB5;c(&vtbNMRa1I_Je_!>iSRO(Q)X{QMi2xFg>lOnZ5m9*_5s>-Mr1}g9KIb`}^Ysj~NGxmaH?+9|PU)cxB|`o*=FK7XSjToC`*UF2 zVT_+b>6V09IkpXOattG160bb__HrXv!x;g212q$#EKVe`R4H$0td6Jr_Q}Z5z%cv!WmtKYEc6hQkKFQ2*w?R* zH;VwX#n2FON|?EPl1Lacp`^ze2FZX8EGy$w}-D z$)TGUD$$MnsKOjZA=x}bWnN2vz`neB;$*$aBrek8Sj&Cqx0%>T@gQ>sYyozp6!mYA zvlrKa>E#G=ja_?qz1;Wa9^dCN51s3n+!k}W0vXsGf+d*rBtqbye!CARoZHkm_fV@~ zJYO6GMv9VPC6WO_qJH@d-byQ6vFCKX7~f_~F72mOe4Ypk#~I`LA8n9XX%$a-gp+we zG<;kEF+k9H4h{I9!a)E852w&|PEL8Z3NI-ftSwwAsT^O!n7TS1^l%`jV(efHK3|RT zGr=aHdWeRGo(%#O82*IGF<3uZtVdEXFH#?2Mk-oO>gw=8!~q5pp3k@gc4P-A%(D0B z1M_s{J&O1~Fv)8mOI&t<1(E)>`?3UlQnMwo2IwKx2aG*j1L?fYe?$zw%+vbi#RIdb1wx8YcC!Tn2yquFF zB}S?7yPUJSp-V@AC|dk%Dlkh;d2B!5U!BxQ6g=}PQp@oUhpeuyGHOdZHY)8ecT0k- zcwlK*(gRob;X(H;Bj~M42vf?9%&aDDcS~QcwtDc+dOF2J?OAi4qyds)5`5Mz^Sd@{Mmt-to9IsIw zAQMRD@4@L0aQ>##+eHE zHF;0WA9}eMsEPkN=hCI8g!<65;uKRG$Xs|`{YYAVI9LnhVZY2|Wi!LRM})6`Z1GVo zVEq{0XY+S}9FZqfz`y}pFwcfXc3fh%jQu!EijV#K&%$!yfWRjG115A$j<_oF@s?^m z%9FD^?;`V8daB!zzQMp!NvEJGg>+%=&)+*<8=v$8zC4@g@b^SV-|7!u?TcdUDlq=X zx{2nZ2^bV~L@>mSI0(P}p@mx0xhQWRF9A5n&C?vgreje1U7Hnj_G+n?=Z+PqKV!+) z?fOIHk5xG7T0ku!0ys=92phjLYd~PkDE|rhY|G!QOZkJ9hD~o3n-v-0joXj5q!arD zd4Wg-K-L7Z7@}lV3nWZJ2_|FylAd2);#J5bit*ySL=ME?xu2wD>q7f*hL1vzYjvv9 zTQkr`fI=+_djP0XN$usS@AA>F<;$57^a(S0Nl|}oRKhzsqMAn$>SHnoPbnjb>c8{_Kor@WJU5O+Ya_V13;>~jivR0pqv4-Rp zx>@gnIoo&!Rbcv=e=AkdHRv-@N4!aw6`36kFfsEyIu1@4T2WS&F@Ic_Y9x5@^cN73 zA%1t5_lEa%nl8wfUp6|FkOn%gm8s`wiA3p^KYN8f@24_8#kaPhajkh3_*0%1Q*#-U zY50ffiGlZEN#L;j2klOB?aQ*jVXS#YEhgXKZ-L()u#{ff-w;so6SraDcp;0V9RbUO z5iV=;pj+S-aF*m#tjRG57z$+{wQR`kvTo<3*|L7oxDZ~KMIB;6c#oHg+msV~pnb7s zUl*+2eYmEa3UVNTsb5nQ&`oVd{e<33&1D{28m;TCblpIXYu!U9@l`U9=rY{Gt*%9I*3E5l(iq+Z@=?cM2w^356a)ft>jzlNMbQN za%KD47C)Usb#~;NL-b2eAbUz~muy>^8vH^`t)mL=RWXXe_B)9(kG9|i9bn<2M_C|$ zqmMG6T$tR@-x|}uK~6V!Pm5_35~S^hp@Dp%X+*^h-}!48o`#%s%p;xYkk|ujs5X2q z=$IGZrvYsNpT?)h4Sxz`aD_Qk7cVyO7$K$N@ESxG#@{Y=$l$h7gbfKvLLpZXE2nMP z5aH*Dmf_ z3qcswKd%(K*|pNtkF721C%6mc5~-uZ4j;I_xI5 zb=PlQ-e$*c%$4Pmu?~HsvL{M*))XokvS8iQ=la~>m$jn^I&>+li9_lSMtFEIubM2C zpaD|q&xYj9d2wuFtx?On&$ijihiSK=Z0y7c)MUgoLNd&8PeR95zrmJZZ?R3wxh}E( zS&_79lBSq7HikU2+Z}u0P|@=vfk;xUosOsD;4@)}<9UU;+bk~PFyO}n>9B-A&hl*$ z0rjJ?6ex6B2Z7MuUc^&tDgX1tuCtcRX>< zabQ1OJM}z28%;I49NR%SZFqttAXV!pk-Al>IUhB%^lA=(@TokJMwnN@!v~oD3TZb? zdS^gKhG*ZS76O})&ms{Zp-06NOmB^YKg%w4m~Fl;9VG;)CUnDca2h`g>b;-c+1Z(y z3RjjEOcg@^B5Ks^j9V63w}Y|aa6kUgy7=*DH7)9?4?EHxr%^9-898FZCUBUaW(pH^ z5O3k=Jitht(NdT?GuJ#T2UFhY>V1w-4%B+2qTSEPWNt914swAsU#_vQ8_uPOZ*`Zu zAGGDP6Lit-huvFCD?Jqd0s0o`XRh-;9!o3Vpm-LdY}!t60ioW_P^J{poGe(`FaMT0*HI;HY@N1X7@-0PAKI|72C zZ!Fk*2q+Aw41xCpQ@eqFjCn;Z69E51&%=X^r0p+KVH3J$lM=>8;D@RIAT6k45#X)) z1j_aRWew3!>!tfBbZO$hNCs}$Uk*hYUFoBe0xLw{9XTh{navSs53fy>)ee9FvzSWq z#s z7?kbYO$hhZv#w@0-|w_52+X+cf9Uf^;XHu_zHDU*G^7o=7o zlzw>!22%#^rvHL#g*9r2_7M~QcoiA4jJD=I7OiB;hf~|Kr9U>6mcDClZ)H%#vIg$y z=$3t}UysZYnI>_i#5MF*0Y8|zRmJR15ffAGq3XXC)Xt%Vw+EPT1!P^oHEizs%&g9) z2K%!aF9ji;B&Jm`B{s!9`sugNs11eO*cXi*B0~O*Lfw5v(bcF0DaL6*&fQX1Af8XN zPv8Crmrp+KW3vh!`7FMsTmORShVAq2Rg^4t$%cG9F`9kqU(3u+!Hfq>zv_a_xEK&aSLEFF@qkvQgatXARU1xN@C4`Gx5bsd(j%VZp;e{%3KmL&G9xe# zjbN%NLWMwo*O|^?@R#oEe>4CI4}OhdJ65@?vlYXYsQhx|0et+81KoP#@a&YMVZl7Co$l7bHD)F6WB{ z;d^I)i+tg4jFL;$MK*k(p03`2rEhaYDfBky8TCKPB?%J``b`wryib~Qd?Onun;QLP zWf}V{(njG{f}mTS20di|1b$098B#GfJ#NY)8%abX{2XUz)&XZX!!0`O<3(R@@nxa$ z_Fo7&Qe~{ze8$bQQ2t{pF9+GEPI%QUx2(_^9SW=&99T47&R}9@?4ObE7Hj7gKP7g2 z-Mb4?8((kOi~=1|uwDyrlI#Llv9a&Q=d6F9(mAV^ds}^VSEyme9e(#7sppbunh}1G=>A5_Fx;tsAIj<=YQL_u4iJt8#-h<=Q z@hbb_9$!{cEFQOW^)bl5q=t26aEH#W9u*PoIPIKMPIP)|_#LlVqzG<|y-`R~9$glQ z2nOi{HvmhgxzXO;42a@^^j)Sj;o#Sea1smDYeT4bfT5yR?@q)b{Mq5b$IFR2?*}MkD9xB~;V&YW zLf4v$BApG3g%7Us=4T}R&+ImF$H`9JZH8tCH-J}VTN()W?{PVi%nGieE{&(78W#$tGw zxNsAx1d3~P=VKHa6mKxTD=_aZ{4wNC+RZ=Ds|e*EN@Czbl%QMlRQ zO&S{DU}CWTv*;v$rytCLHN*9T{x4kYGz1^dmJ8A{_>ErSm9dm5WgD)jLkK<5OtUA9TNl%_eIMu#UU{R#C=CG}Q=f@z|#{K~$ zs9~{-&^!E9Bt&{kt?phYu7rI5o10J8WOycDtLSUR1IbDTAbh|XHR|hIrz;N3u4WWwiEGaa5k|)_ycmmPpm#Tg}cj zwOJe&4T|8Lv|;0IOPykw%Ay>9YIf%v%XnXTkRdI-)AWnw4l3ISpYva!b%mB;JE<`1 zn|s{WA)uh;t0bW14RhKfp@P3LVet60PkNCSnKB$@f=vT3Y}O_rw}>8wcB}A(ioeqK z`UvZh>TXTX?77TInwRUFj^Pl)Mw|}D69H~;7-YO3Ts|nNxsiPXTgSgjtJe(oIBk3f z4d%Ns_=A58*bR^%_G*8yWcX)a0$iy+`ji$hG-ph{;Hx*+Cl5)$*t=Qj5uYLXG5Wn{ zTRO?yeCbguWqJUsu%%=Adr=#%hz@_UBIf&xxlO0~V>eO%?gP`fI1Pyx`>dw#{03~= zF!jSThN(jne-_95JbA*p4+Q(8qRMASnXyZlIJmi$cGZnnY(G8y>f3_iL#D447mHb0 z?^doZP#OFW(-3jU(XSavX&h`raPKRuBI}H=qi5FRyWB7IYB9mAsS149XNM;Lca^&I zP~;(nk1?c{%TIXK8_xZfwT-kE;^+VC8wvMYgSfyaeqkAG2>s|=pWX!J`UmHKuQE$s z+8~p6nEAJ{SzOOCzQ-gA-OuoFN3l{afxATiL{4LyR^Z}ODI4H~YgFDAPz*0=Xhx==eZCTi2J z77}49@;%WCNq!n zNg!mETDJ(0t+4ww;7YM(AoBm_Rezo~#)8^@I5bqvkq#sOxxM`iY=$K1?RN;};=ADx zQvtM<{{y$3nU(zs)uT#bn&1F-3K>wc0~o^^eI6}$ld=)?Z?fI?9ia8Uy22hE23ygt z0u>+HwFh5PR=yAW38$HC^POLxNb+hW zQz9s!r#rzSB0QX)kr6UeYeshrf>Y%{A#X|yI6P_(0L;6hIOX3G0BgUPm1yF#Lna%p zAD@BgckS>=yy0$c+zoBMH=ii7Zorc%_gI7w76r*amrPgirqyAG##6APKYR9UqEXuO z{GAPU&4uQf1%Z^b4FyGpnn}CQm)RCKvm8*1BiJ(*6!hweCu5fZSsf>H%geWc9p4X( zh`?<09r%0%;;0*Kzvn-LIUu;Ry$J|iA7)QR7VfuRvynq24OYnTEbcx2Ko=Rv&jx+} zy%d{0L4VkR8jsmSU#gxaUET)Zo&YX9pG=4yc{sEKMK)Sn*w585lr)B=+6ntL z9?*iY9tN-Bd}yH2yxDF`B?fFF0TokKwVNiFK;8yIlnP7{cX2YHT?qy{l#^$`*MF?| z+X4$pHR9+a1W<(S%(DS~V0hAzn-9QwQd1IQk`RVd7xwvkgDe+zVBMHGu5S9?v$sZX z5^G!L^qC{&pGvPRBoUL);L}NV#EiX}3{%63`lB|8&tZ$CA7{)VK9uW zZNte9KKtRD1Ncca8Ie+-X6mhmGC=0e&Z~cT>JNVdVRHJGVfBtjeXjYgMFVlnmvMw)Cls658Wq?J}*60MCy= zI-u=w>KD*WDI>YNwo#`|n)A=}qu$*fT>HAvU>Maip^FKAJ9T>h>n+5i9gb#)FR(HQ zvoGqg!&i6`GD37QJF&*vrMG-%U$pcoow)%H--cCJU(OcD{T=gFqlrq1e&5UUwM*^9tnuLvW!vd-DavCKDfscx_>+y3Z+` z#E5oa#Bp?Z`FhR|qolZ1T&DBDcCoXbI^YLH1H(68q(LXQJR*v)TzL`tK$}xfyZzO< zLxaxBOlSE@iUcv+2Y3AN!VX-pFEx<-AH;TWGZX}76!qWr=U<7Dh&cM337MM8!Scxse zXZx2{NK{^Qd+m_aqaomXC5p**EHC*d@c>Q(3l8=zG*fFD5B;oH&7Um{jsX&puRQ|q;ByNL8ioMExdw7}tMV9Iq|c7}I)3F;$;#;SJP^klWZ?+{Qt_`~ zfOEl4CblVwByMDxk$GI^q!!Z-JQ?+qgFje6FvnC<7$kMk$LuWA0+Yta>HTRg$y;4E z0?L}2#@*NgK_}=df0KxO9W>rDAty?F+rM`Q1nhqu5Ic(K!qU=99saci;(^W8A4-Xt z73tN)A9D<^y}Jqt+(f6<`5LvFx)ezCu;iKSL8ottA_c1^6djsoT7nxMibBX`qrA|i zrIJlc6hW0o8Tv>sO(pKjapRi-Ow#x5Z42FhzGExej^+AqGG-m*dzKNu0W^P5Pim?O z#Qy6--p2aea@Hp+n!Y=j?e|AMKSvfzm0+u=e33#-OeSg-FA&3~ak?`r8Q^KI8XL4@ zMd&8f$)91t+hrf^A27Rb9jj4k>dC0YUMM*@IIQhoi`0no@}^9J#2k8hl;Tm~=^lgW_k#7RgvXs@-;u>gWEY60 zZ|Xk{Yp+Hokl^`g_qH1XK6o*ZNErEUZrfw?J~mzIuJnxYJyCcXR^48hb4?H5+UR2Q z)FFnYDd{~1{R96ss9>|nTe$KNG!@sfB(0}l^Ea~gZQGgyQK1xRJ`4xC^BsxLhv=l3uq>WK;tG``{Ea33Z;+9gNi__!w9$mFWeKtOU7A zjUf4a2ZUM&1_vjB6;TPQk%ueTjpY^woa}s^H4EXyfns3cqcSt^J1#VHQc`v43LX9Z zy9Ew?YYPkMy?w#;&CMwg#f$5VDr47shXkUas930^q$DVsbN>yund!OdGb=4k&DhKS zAvK_4Am-uVo6)pqk9+&3o(mFCx1b!% zK2yAt=h)pYg@-T}^dtL^N)A3d>E3(Cyx4w1U9c6CF_SA9GInA@O8$}&p`Y5*^FhqO z3zzHbth`#bjTg1pWejDps47=I-)?9u!~n7F3g+8~_LfmY%1DUKYn??dFo@r!5y;#J z@-m2tH9It!umB%ynXvda&q8zHp6S;j?mGX8N5yCRa-}y61KS>GV)_bmg)lHM0?rQ^ zt^q$cJaeL4gAQEGPdNQ4xQtORU*f+tXn54*wwx~`D+|464k{d2ySS)zw3>oUe|ta! zoCZO0>EsIiddB*eWjHM@7tQ%HEhwRB)=S5w@3lAg8Z2NXpbaW(a+-yC*1@jAC9igk zA67goj^-H$F*hC6!)}>g?2nI--C@5*K7W4cT=W#u!Da4hAz5|r)P-EPGBC7#@ZgEbMmvor&n_V1STL zjo}+V<;aMLi1_VmUlD;aT2M@P{oQiw^Wx^>;$I;&IbD0`CY^c4Tf0yunDsqezewDK zwN@Gul1W9~31XNvbLszOVgG;51Qji!7cX9H0c%bs9$nJ?6C>PR4}g0efIa86KQFMV zchg)TMI4q{6lpgxvgtJo@=L z!ZY< zeGECBoaBVZ01sCRlAdtCLd33?zkJzO+vx?>-r^om#hDB!=)Bdw=c_@_cWFgZSAtYA(QFPPpShOz=>^y+pr^8 zchx0c4Rm0_E$Z34gUN*PZnN1^gW`gV=ZL5OnR4KD3^Xt!At!s-kq|OZyBKALlu$hA z)m;j<+v;qMQ47rfw&Rb`?N!g%F4-X!^y+M6WCZ@;ftp#+30b}Jxy))GF6pBQpm~cF zy6NibI$m(=#+5;@W$5YFHGV1$s)pR~odW$he3CL-$c^hM$)TjDD9DZ^kHlMQP43yH zKfHitWo7m4;}|a@?X((@J#DvbK>SG9L5S(5`C5t%ju?$T5GJ)GjE;)(&KSZ1wax6e zP0he)wUm#V-UFjWn!YRns?236DMPQH|JXdf+ zF9-|`kB$A7H2rh%he)!Y!Wq1q8zaI|RY`^2k%NN%JVt>)(a6}CMp#(5v&|(84@WFM zY*q#7tJQ++nV}no$ zwv(;g{*_k1&p1;YXq$flaWwa{cA;iDPhV>Gk5G z_`1ml`|E?ntC-;L#~49Z+x&mI&6N1t=WTIh!Z!BizEO}{Tcl`wGyZe?ROcWbyaLV6 zUTxXe47`oxf`Wp#lvWbVB<2hkjUM0GcD^?ymEk~ZZwp%*az>tpKik8?ivUMzz@G(# zv5AQi<7;AW!jS}BAhcr+H*w1vb8S42Gx2Xz#aAVU^`j~}1%VL*q?(^RfrZ6rcGXeu zAn&zP`36~zrX~TIXM;PRTTpVE@a%uF+?{rlVV^xfW|{N&=)3J)uHra;En~X&KK7*& zZKQCyy~h9MGNIi?Fj2Tf4CA-rb9^T2kJS#I3DcF9!)Azl*LT3G~*X5v&k8C=n=&^56_jn#^E)%hOHi?L; zpcvmiIAW;`F5Xzv8B5}NY$O}dQXCv1uQ_{m*|6K1iV19=-&SdPc>^0G_XbtKUhjJ; z5*000lI|kfu!W5c`w*D=)6xp_b=X3%`)%_|?VBcoADt&e;vFF|5H{_ydDSZ(etzv= zsZ7h$#IUtr%svNt2GL%UOKB;*i4_r&Pa*aHMjbWWA|Zqi&;Q68UR*Q6@bOCwo1E?u zUCK{MK(6+$z!KkPyL?G2l3vYG6Dj8x_%FFUrSAlkeyo7=L_*&DoM{~a6JjylzC#ua zqHhAB*!x0}-;`@MF@{jLgUQ!JGhU+eFBhvlDx2Kqse7dsqQZ|_`bPTaBB^vwP+=bj zH9@`mDB1SZ2KLUENu^d4;r5Elr|<`q4D_pJk{-XglrjVQh7`L8FaK`*HqhPr#02-D z5mQ+Zd{+kLW5EgO3O-@n7xQ}cDCsgbUow!9X3uHX>FG6g2*+mw_p843UH$%E_n)t^ zLkdKdOqaTjdA10D1gFR(bY128xLn4C7kh56xGiia)JHhn{;Bd>|4DffumpFUXa%K= zso2h4d*98>r2CxjiNlWhE#I6rDJ)KqH_QDSsg+RuBKT1KQYi)3;J!*lxR&4~!E{Gu z05jZu2E;N7*=MSQvhrD zRGj5v>v1F`Xz^sShEhg)`;L`N?I7~N2Q>bNkg+kb{CRylCN=~|IN>Yai$ynhui!Bj?wb$ug2G)$ksHjBrGoS0vBORbxI1Ncj~B*0f@j++7ves$ zwqazj`uKv^hN>;{_!5n@!1d|ZTGMWhl|%~Bsn7P6(T zZWHd6OPiy(5U%9}5=OMQ|MdARR`{sCz=%8wL+O4hgb11tV~z$~;>h&wjaWrSh=p8^ z6J3k_=()Ug$aWF&A)Ta*CxMj+z}Q-;@nodi$HXJyQ9V0NBTK6%Y7zhQmNepcc%0gw zk!*1Hdl7h~#{4VSnkl4})fwE^hx>Xm$`};nHzFRPg0)7hCA4b|UN|c3P9h-}_c^nV zgk9!>GqYQMqXa@$!S3=!xAz${$%IOwRGgTFp#ms2Mnm@!1{c(!a5~w~R5l)I6*EGqm0q*%{RBekt7cq4S+U&qACe5scxc zqYJH;y~j=c!nW6IzTP|U@cQhMux$G%{h6aOuK>PRxe^~a>?>4G4(?tUI?JrzWlwHm z6?M-)r_CHnK>3eV?fwb+1L0PMSn~U#4!VWHZAc#b^Cv!mbIS7#L8O29aUBBs^O-23 z0?foFL-7bX4SnHzH0Qf?43nYdw1+d*I)R1_Hj-J{S#dWUh)?u)x8W!g7e?UD$??JK z0_kFrkfy?+eh=bMBt15V7a5Keq9+2Ch#B1Jq#|6cE@T_8%-yKTuQ&WL|9SaWhBEz31g) zWn#KkdmAn@_SS(o5+V>b)JP=Zrt zP$!eA2>G>dCm>O{fv8;xFf1nz1~04`K=a9!w~EhzVYhY7gd#%&V>lq_kd_L*%TaVi zl@*L2f>{bbL>5g3ofeCjJ{%cZj)rLs>_I6k#jPSA@T}=BsNp7f{h9^dCs%}b5BH0` zYnA>(3x%A!8L;sq;((0_{12Oo$InZb zf8lJEB8kaJ`8XRdLt`}uCvyUV!31}K+;7=meTO_$5Ys^s!$2B+0Qd*)lDc4>em=WT=)vn+l0 zK1B7PF~kWE_w4!5J4s-ZlT-Fz)X#VL2JE)yy;R! zZl=_~uAejj7t|-bLjf7=daj|Vsj1XeNN1x;&C*=p_xV*(fsobGnrhA9He2!sW~46-|a z{IG9sZ2a>c90{92OR2>g&=39oC}t`22Eb6Gpf}nopa^baKRo`VRhXNv{R9ApBjb0~ z*l2ZrSn6%?C`cvrQI~PA!N_oJ?3G? zGAz8k_W_;TJPBK&vW~tKYGh@#y|TJ`a-GEahB=CaZ+{6rOztp3^63cZg_*0E_TorW zOKS#Ok`O*yMA@mC6H_d}elM3<6I_0L9cN+RqnCqK=2Oh%1OPOj+Vd6rL>dd|#)BubF&FNyO4xVskHKqsxIHa0deMv51Xl~5EQWOYQr#f^h`X%`JH=#H22lQElVxlTe zxRJLewCNr0}niGC@c2 z)8pgjo}5P|;FS8F>k^nT@RGHv0`TY4sH3|KhZ$ZHJvF7+UYe)}WwJ-#w$ww_{I{Cw z)$6ARcie37**8!=1DLa{&i&!mbg|?K@Ph?beqR_0@^1BsugP#jMFQE>DOV307mA63iI_9TLC{F?9W9U=fBCR_?1 zK!{m6^>Xmf5}7yT>R=1ZLhHZc1faw1j78b;0@z)aQAa^{W*EE$10mWWOJ&?yt2iU$ zH+$}jYtXvXDWB9aMr2OD#7B}|-Tda;0U%zKfp3VthX09znf^otz@ueMpe=@~)5!ph z$}f2J05kvZ84l~5|C)SiJRMS!f_bvV81S{wVLYqZW6jj8D3_(XMkLD=T)eD#FgdZj z$buLgi_QxN0J7B1^1Q##xG`I!3)dO}zD!2fS_BSTcU6Mg7WQtx_)MijA}!$ z)&ywG58){t2R|DtbIQ;Y!Vt&q9Kw+jZfRlif<9Mqg%qS1XlOfnW27zYy|*zIal+mr z;*CToNOYrPLkHrFBSSvw8zlfp{lsot8YChkPp%(co8LSc|0%2}ipN-S68G~pV_K`i znCF9&{uHud|(c3xFGP!SMcaIfn-)GBF@2CRAEVDuE^Jc6S&1jW8t_ zk*%fWx#&`_D=OV@9BMRK>b$lk0Qt0e6|xV5YAsw~L!CY#efs44KnHvizYdEt@ywJc zy!LF9%Td1_fr3$9Q0doL5fLG9iEFF*6N?A9XtcPf4uD@5GW%);mKg_+`Np57}<7B{&d>#K(bJpNW+Jv}hcpn4ljePFrs<_|0EVn6*AG12QkS0hQB<9)qzA$n>0@xL|)iX_71`sXcn z^f!`R0)N|VOrO5vxwBkvWRZoSp-s_!oED`JzSlIY5PaD)HpU;lEs2YO5LD)knlE#F zoEeh#uIag3MS$L85(NUZeTw~8ppT_%FswYZ{zGz5Z8GeWs;_TUYu^Y3qA9(RrKyvSwVz z?Jx;W*q|kK#wvRr5uo$ZO-u|5EB#J}_*_5}o4Mw9EOxUsuns5TaE$>=c9A8cwzyr`y{Oi>75n3O|+c#R9*ycxAtwt6ZUu!=|2zHKN5VGAR?uK z*+bu&lNX%GYQ`6HHOWi;3S9zsYcJ%Du@gWiDfzh8F&GsYmwAxE!I4b@2~g*ROb{6x z2;|0GG@FuvLApp8^r%J>mEW2wC&75i6|7TlO_xXQ2p3q$a9D>x{?_q)Pdl5DI+}TT z8~uhCS-}=;n*GnSmq%K{p0=l3A4=bg%9EntK#xhV!b(7krRTsho$<9yX5Gu=Xqr3Q zKOFJh{KCGw`nLXTcbu78R)BvB>e{B6@Ey!SGXD%TXg6mXkUfh-uqSk}JN4j-M6R=JQj>}-e3G^6e zfLEI&TBpI5X##uM;hKz;G`P+`hnV8q($&dC@ULL_yLaS*&m0y7038~S{jv(MrJ}4w zs)d-KMR+PQ7C9>lJn-dH8Q=vJayho>iHuAXm}mPhaleh0v_T|jg^BfBU)4NzIdgTQ zasgbFEH3IrCt&q96jOPB%O!Bt$dn0=aA#Aloo*JnS_i$tIvsCWtBu?6UCMG zZQIC4OcWNzbjS731EI^a_PG|ndlpl6=>!nq-lDFFF<}!H5$HTOQKmy-9sLp+G{8&; zj!F5SCxx|zl<+5IlQj5b=120FCI|OTaB;d{gE5hiQt< zPn7`6Jh#_PxgvZmcr7Hk82I zk#JU|6&JNBZYsA3w=>&e2kZf^Y7!hA_M?X{eageVvG+!q28|Ns<{X||hXpUM4eGY1 zWmZ})Ra8_hXFiL%oMo~!S6Rp!BD_jMtH*Gc=mPvqoRFavJrqzF-hV(S!Igr>lNu<# zWQIm3Ni=a%!zpO8kY6)Du*Sqt*13kHQB;#WAGE|N@+mlQSvA-Ga{8($Bs-=!NiLWT z8#}UdwRBe;p1rIxue6a$;NruM7XS3m-36n_Xv+~J6hx6n|MMkMih9_ru;Pb>b-eA& z0Ud;aQtDFyv2A{SzOsY_qIwZjbQS&1(jO|a-XE`94VM3~R+n@(e3TmPM>M1`d#X}} zkD%dbSA*@%J4rhTkD_XDVCrM9Dd(fI_7-{AA4d@!=OAM_HF(bG=nk2PCeA9S!Zr&D zo-(Rl^735S0!;O&EAEA`j!@zn21kT{KGlQ*rSKp8mb2wgKihqRti!g^ACRRlQ?F1h z1EHOy+(uT!(X;BYzGHEi+GzOgHpZ;6G+v;k2Z^>?7lVl~dv6xPhlR=gG)dor9-)(E}-oj`0TMZo;v z3s2v(S{vWnAju)o=@;3QFX@u$2Z`hul+vLTc7tUQ_|^ompeqck9gF9EJ+txfH~lPT z8HJ)a5U17c37qh2h+9XqDyTgukRLzpXiu$yL{7r`fU9bM8xVv?mhhvgy(mc1o){;r zOJ;3ty6I!FA4@3j!<%+WPzwOk6GB8}nVynzbpc4*x6bd_)M`Y` zWkK+`4S&nRuNmNh^6VFo`4N>u137)Vxw7OMA0$myucveK;`HSiF*j^~-pfay&H=0P zAU}gbk;;vaf#1Utqx|OWE1wQ~LVe+D^p+I0cF*v~Nu5=14mjuP&5rzB`x@lYcV-%J ze&1c%G`Tw5O$&pXlSv`{g+#^M{;k1FAO3X`oknY9Bf6!Nf3*M}@oWZ%j9bpK{VXMu zd*Z}=B^(N**V$b2xC`b^dxBw@f=yK_p?0;Bd=*s4;=T=7M|MpdoqcNylJ-2^`k&$8 zDIfjM7br!n-X=0`EXFR~2uEzOZKx45L0IpNnJ|dvqaVVQH*}2UhJa~d6>F|z(`{#@ z`O_RM@IJSyM6c;s+4Z++IjB^02Esz|S!oF*vEKoJ!AJgvh++4I* z@vVJR(l0=MSE|w-uGRd9GH!8qB#w5kgA=c#K=4{=q)VTr$ly2g;6_ImJrXQ%e6s@e z(VNaT}X>xis!3$Dmre@h-_ndgB0t z*CK6ME|eEzTie}*D)VPUnG(YfJ+qtEVihC`;~ziWg0}5P-t?Bwl5iHhpbrx`$S2Fi z5)(btC|or-)2M&mSgNL@M}+lv+7J+9GxmRkVa%Ss>!CVw?$L{qLVF+}Pq*thPbYI= zY7a@~)9AX^)Q)R?I6Bv>I>a-ZG+hX8d|1av=jvg6TzTY>l}ge})dyX3qR@l5=}_=2jxs(NnyKL zrPw$d+H~=`L=vWp>$UgK1F?T;wzWLd^pv7U(m2l*{^N6RZYz4Pcv~GpTu&xdTx=DX zUQKfiHP_&^`{@xZFrU7ix%ThFsC5GP|&LDHZm(K$4f{v+d+oH~nUZpOjuT;nHw zOVfr$s#y{AmC!>!ii>o=wppG}L0bKCk4g)T268ZJa&vMb-k6v~kdl&S)EeFc&=092 z=(vvShP|IY7~bR^YKBTaf1CEj^&_2XcNHr69#o`O*RGXSLd&mzYORKP+r;wVgk^w& zYX4OEl?Lm~J5C(|MIFo0s zOZbwnFdYgCi^jcd=I}*AMf!r2(pQ?|KUu{tmVX*ub`_t!jageC|1?pyD#P60k?W_Q zxAKN?q>ZH|sc2cr!^2~*DpQdIFBZla*n4I((vZ19c>I!$EvQT|1y!u;BKkU3`ZB7{ zrK^y8bJLdsI+@{WGDS@td)DVf2+7J;r6RM;te9 zcL%8xrCv63Ep}EhVgWAp`=2z|+3C=bG8ONnids5hfn;7$qy8(;k+PSgg@?!Idv=TU zcm2j|UF{6y6<$TS;~6WDbTq=?4Tg<>)!~Z~BUE0GG+Mk89bsg&hd|Kg>WZZ8nxnFB ze(5It@Z0P1n; zDf6yWkXEUw@+DdghhR1^*2N>uUH=RoF!Ec<3}*48qt1!re6zE5-`e zdGEsb0!$K<_EV)&vI#8k=u_g@&byZFFGm<+LhYY&Z&TYmU*%*mX8RY?9O~VFuhm3d zxfBr0G8mu&0zc7RjUU)zHfnuUPb8W#dUG49}=&NPm41z>cJSo$2!NDnXl3^JK zN7zR=T&ET#WRn#m$0|2Pt!?zNBpMPS`vR`YQH8W}I> zs001WHSO8#QX}ij1RJ*Z%(aD27&Hl}x^hvZFtvC+=$OZc(SM%WkKjuvKgPJHQakv0&>M+JuvBl3yJ!1zykHm7 zkdXw6BLc76szPe`f`S55E938&@)=@U4O~E#bhN!FoU^Elm7^WgsYK6~eH&_E;d3Z( zsk&EykfAxev^Jc~8+&|mvV?-f#2V`R>lDAvYspaTeqUc_1X;tYv!{`b?aGN&;ojRa z`+Xp2A+B6#kcry#r{h3Rae=>)krr}@z$Gkm(zw}r*Lj2>{PFa+hgx6eME)>hb5|I~ zG5yf?z45Jm7vJK({-UHh$+_2azWcdf>#49^G8V+RT&Kl-F=Me}C>UG@9`MXt30l2;bJcXvFdU&IbqEx66T7*U-WAD5{II<#hpj!W@q zEd)!djck5v#Dt0rkvmM)zWZjJ^5@Jmy8(gF26P-`)Kk_b{S=5q)#FDU-cW>9{MdWF zOIIH=vApebE$oegdUlV%-=QX!SzcddV?tH>_=K)! zj`@QIb*WotrFdQ44c}16wYjaWL`aC_67%Xr~E`}r7%9&Cy=_&zT)LStQwg>X@W+)I! zDWs|e$@FhWK}wrnwfmttFm+v#k<4aI9ClCJMUCSB=;6y!C?L?>DSb!QcE6BL6c9=(2&vx2M6bX3c77`Ys+$HXD4NMZ7q(3g2I6S5ARJFi8ioL~c1AUc2d!G94IX}2goSv>xokV0d^biPlw&fkw0#_%!qskAYnP{iv2xgs?i zMmnbOcyI3=GwvV!onT57$m%9SAbJ|+~;(E^AnSSmUiSLl{jd9e7r^- z{_Fy(4X~Rm)|#r)aKuw(M)%JF{3)*{^wcXxk@ftt!m)U!5dV4PA)&09F^qny_4!p> zgIB6S@|v?hIQOksfCP2+Vo=Md@bta%MzgUl2BOKEH^mmw26dOAhVLg4pec{(P?6+m zXXZgIWe!$L!X@VMsXnr)iVse*T82U20U%XFmYUz$+}~dRVPJeC`oPlS^S+Ds7eYhw z=AgL$<@qK5mxa3RfXgyWYxM6Nxx|UckA!VnRTKK2P+$uw&&o}JhuD7!zns1LukB%| zrWO{elbt1Xri)G`>TQ@$iHVtef^dTa-?9!FeF1F>*KtwvqEghZ!{kc_-B;Z-Gzb70 zLA4L?iMDX`D)c8pb?yt;ce@+dq6PJ$8L`+-)d}Hz@X-Q&b3)7$%o?v7 zW`5fkF**~(H#eZ31pYg9j3SiJw}mA2GH+()SsR))4f~;;!FuA9kUkiCx|}#r<+1`U z(tF@CHnr5tY7qa`Ik?YTYHLsI2;i1=3gn+QZEWBm^?X1YBtP34UHuLqL3hz=@1`e- zBOcS2GwOTo(YXjIO?ONM`W|*4fHI|>M7}HfD-+XGN!j%U^jEqzXon{`b38ptL(PPz zN#rnr+~95qXc|%^Y#83*pk1O`S|hf)M4neE2DgcQT)XP@lZ$N0xtt4?2_;8B;Tilt zlb<33M8Wff{j2wKRSP(9wM4Qi|yt7%1FJ)9VWSP>u7nHabA`Nakd=bVW_$ zR+#2_=|~yDMnT+gRN`gq7Yq4UBrIJ86?OjyCt3$^%mk-HgqGQ?pCF@Wpr8M3s%Rpt zf}xQo?~6&77=iY)lgMNxp-b8;kg|8T4Pq~yP3zg@I?@S$VL}aloZhhsQ1p!%YWs9U z(Asa`a=@euKBh`P<>FK}KnttUD*9Ow5D@$&6Bd(O&N-CKrN!+w3+!==pD{|g+rieDYfPSbU0;te7ZQ_L5Palpqy)GA_~D142+_P7CiORJvjmb zNeW-C=e1Vj=eaKD2R~uvcC(-9^DRtOx$tn%8aOH1q{;20VC}t4m_B;mHJ?1ehMNy!#2}0baZM=QI!=*6>j%} zRPdImyK1*suXai$^BeNt2Y|9p)l~*pVFCZ=tC$#3*fDGgqaXU3GUAzh}>6m&hc8G47Hy}?*KyzgAyc7jnXcVAGE?>DJ4W=iL6_#}II zDqXtncVmauW!Pl?De~8R!~(@+iG`n<`Mns{j8EvcR`7Hh&uGi9e9OP^b`!nunKRuP zE1~AG(~~oH27)t3$F9^Nq35-w=hbds>vO^q&>YZ^|GS0AQ!)s~YV~)8&zBX6#)phs z!7!MfVT%Qjj&RL(-7E$bLaw;OU>hka@o0-{zf2h<@PmaS^KPYojMr{*PSlC=5Ra5% z%+BiI*RY2`4+Xdqei*c}NuV8iwA9(iBvuMjX0%f-4Aw=kmgu*&j%@b+#y&QU!L&=6 zH0OaMjnfc;5<~!o#N2kK%6JY?cy^{o@qd_PS(n8`-`WBH&?V^EeS5LBwMAi;p8MIi zoJ?gPiL%#$b5XAZyIE9=f>Ul}~fHriZVg^{oLug*7Y}W7)qNI`ldJaAFb_2K%D&=(ZSikW_7Nt$*VpY z8y(`C@(lZ5ELz?_KVP_on~U|8ptlvQ>jSf2E<(0Q@9pL&qEd$BcxK^UcG72Bs^F!8 z@nrtG5^*y@{b1n(V~5lG7rq~C%h;@DKvTwA9S6i`y}HXY$0$Xo?rxHE zD>55Tk$-|xkR?4_B%N5`a=eF8V|LTdIti+ zN(CE%t@JHQI5XqhOQc=gxS1xA$L5&?zzfgK&FxYmu)4V4a;+DK{xn5C7V0b8QBV;s z_eQR)+5`p3hVM=)c})zDp7EBKusWgq$Q9^|1Z)t(}f+B2z>Rnx-G81-S*x8Pm(f9&}9p|Iw#~C4K+1? z--QC?NvE*VnkO?^1k+^!%XRQCz@*0Y#o1`x%FZgmCSaCv{fJ0^!WF z66^X1mtHKoP*V1>^u|gBjGj*c`K`R!lfI*1=}7JKsfwX^=N|cO3S+JRA&?$w2gX1V z?>;3QCLC#pOFyDTb$x&EY?rx96l7qcWsP0G(N)d$RPP&{2&T?8tiis2l;CT`A%VG8 zl@mb7zLbw3%>k9SrLMPA4!t_Fl;myAs5;%*Riu9Nv-Wcywe}9vDGMpCvpY#9d(|1*&Z+_`&>0 zbdjHMKFlwuF+l`Sm9D0G`Zz+(`zYiTxS36#GTgY@JDX%kC`p zwa&$j6Aj{Q|7gGX1S*J}yh#jvlq;_o2E0M3NXi`4kx77m#moe)g^Wu1z0k51x{e%%7U2Cg!_ycP3J&}-kF5+nb#}RD*s-@Yshw)usUy`L#xw{Mv?D_Li>%4RzwO%1 zED#V*N=nBU)LNj1Gh_gl=XQPI7r3>z@#eLz1WNaN>9x*CMJZ%&J&n*l2xUzpl=L z$nmy?`T0f4m@vc}^qM?E$hTu~vY+ViyY~W0sImg4)}B*wGBG$j{r4X@zKD8*{O2!D z$J&BXK+J*qcyhc@-7|jA&#@psA2oC}I2h5%(b4_HM^{%Dcy%D{(xx@B zz{@6Hz@-R=cV|eND6@muZVp(ZY`mDLni{0~r}$QGw9v1Z=Yhjxf;%JG2!eihdf1ZZ z`wMcwg4unaaeH@tFi~ZEq!~t__aYb`MuCw0SV%|uM@YgClJc~jE3XMXBE}OD7sP+L zStwdS+Nn%fE6;Oo4wikld#qex&3;-mVi=IBzwnj$ZYqgz8ITtLice231$R&eLbb=jO;GBd4?XwwB z#N8TgkL70|CHJcoRpkj9g%)bjp4)?T7jmwF86mn8WGfIdS5)a`6)w=wqbJXf+NTnL( zIiSula;FEl^^s0q;8X zG;+2`0UKeyYM;in8GXWH`oP3xb6|CH>HZB34H_BIzoLOjY#A>PHlCgH`L&z9I1l{L zd9)$oNmGyu8Y^egtYoF^)zx z2L}SPs^N(a~sKtZSlJb=Ho$DG*xaup}f(xg6@d7~1iD!wMaY z2Z7j~NBF>Hf{~9pyO^#JQ2W@{CnV4QDZ35M20SGM`4@Ska1d5F>a7h$% z!)Y&RNL`vnMsg(4eK8825&FF`P?HEyV_eq#0t)~=7e+uoW5nz%5Fjy8uFI#>O<(T$O?awR|XLlWbiPWA~qJBB0UH3PVCETzu?2EskS2tL1I*QWzAZspRfPA)gysO|IM+J z6oi||KLGR!zJeFf3796foGih=1&P~rmZQ{ESeYH1ewsU?qJYB~LV~;NO+!bT=U^8) zY}^ByMF!u)jqGc0gT+Q8V)56*K_$WP89HBSsNJu+z{UcrCl&U+QP_ba#H6O7fu-Q4 z;e{`H&_Eo2+}PY`V3P+Z>xoN$0)%9r{X3UjPN*|g0F8tX8rQ5T*dewXf=p1z zV%|yP{S8eiv{U@V1M)@F&_|u|%paYUmzUQCv>?eAS$+r%fC~yI1W~|=+c?VczzfCa zUXJo3qW0HyXAc(nhpuTiz}8do1WS_<9ix?&y{19CAY!jNL_uSZvz>vun393@)}Y6f zPHz9~W^iQ2wdT{k^mf!8LJxYQN?7Bwe}U{dX-bY)AAX7ZqP>pfk4a2Db249#sZ$!) z%zi5->LknMb9we;&TW}b!Q%1#8tunVpTzBc*1c&Ijj-V3`K;of>Eui9Lktm>#DC7Z z#qKrG8c#-~TNRYRHyn`Ic*Iu-LJh(P`}$t=Nd$5pM@=`}feqmzJ0Pr`-cX*pY)`G9({g5&l)`O z&{kyydsr3U-4h{NWF`AdfO;j-&=<#=@Q`n_C!hSV55Zt2IvndJQlRn^v`Pw2*z!c^ z#!G-FAat)*_a!1uAR%&Gez6$ivY*4tT(E7U+AK%vKW9ckBzxt`s+OCTHLQ}I1>g$! zR~fxsyCRO+q0^l1Ex_b%_AP#z!R2}UFrQPBnUU#ice|dj)A8>ECFDv;3hux3M-R%H zd9SkAOl9>bKbsc;#SyO^((u})!^kaOie|3UavR z{~%m-xJL(Vi#HrLJyPI=Cr&((1nm4f7Bcx(+(<>oy?HcDQffyv-`p8ae;g!G>D&bi&IyKJ%nZjBFbiDr zd`Ly;g~2DBixChI1Y&pD&>j}O2POz~=aXGzkfp>Z!QVHgMg?*7Yx4iHDl?#utrm@aKD>XaCpns5zv4_2oJ9Z*W^ zMmIF{Zpgfb3V96vBR`TutWgnXTJIT+Tifg?+}tSe$xO{`tDchy8!g}g!idAKAqPco z!;vT2yxiie)`V~5zM83XWoIaUSprE=Z!>jxY;tkLy6zjZ`Aehj=?0!UbYaFQ3J|7u z@UUG795#Lzs7wp}Tq}E4%(}k(qSRA3GIDw*jk=nG^Bv%=Dgp@MOml`)#rIlzk6du> zR!jowj&!ju;F1oWSCVz+;^u*&{R1@AunGUb$k3&5Hq+yGj$K~|2j!>o(WfB>ZH~dn z=La&A2m`fh$Sldx{%P-Na}pzKP+3|Kr9P1kEBn57TL@jRI=76f91z zXS;9AHJJu}l$NHKd;R1>4oPudugJ0a`r@C}21SNTP!VB@`S#_LA?Z;6K!n23ciH&_ zv$T%<^O4{9fOx%wL5l*J&O2Os zN4FD73J1d@hBpLeuLT)Rex?py0Nu)&=1v%t8xQiZmYhBy#Zk$V{Ly?(MfHJ`&M9LI1;G6b1N8#7f7uKItDe%WQS&e;u7y zra7!qqe{VG+J-hTS6f+!`J#IG?v!j#tJM z5-kd0VC8NlS<1>PN?7|7iA)*=N!ou(FSpB19vU1h);OD!J&~f#7O}ds$~D#}M5}wN z^m!N?4e*I4vlz1y>`l06M#8w4#8uJVBXxl>)ks*+jf7vkF4NkH!k1I;GdF>|zC^|N z=V4m+L8P7Y^|7|PVGwshZc6VDZmYP=x0#Yp5Np+gpOQeef^h^bdE&r;s2^XHjDnRW zpVPYhV*oc`3sVQlWe}@Pj5UcC|C?MZboKvUZV2K=(=ExpGH_U9nF5qni5L#;D*j zXHb}FsFdsyHbt?1twn)hsD+X~{}0jkUZj)Bzoy@>#D{}+&V0Ne7f*Q`29gvhE5v)J2Y)#I@O;V zXZQqYKhioBUt#KHzuOIOj}%e2v=~fMq`y3xXjc#ip2WZ?kH2i1NEQ*i4i&`ckBA8Q zgV`mUPS+5C>uG6vJ9B=-WP+};1te&uC-NO$^)^3ag|0t8emV1#kri_IlYEh)uY!O6 zPpBRr`+J3U@#`sW*BP(99objFj`tNN`1cWm{qezk{xTrDW#d7}e@FfS?7Wp3(h}oE zA4)BI8cU13FOS8T(Neq-FXAI_LPC_jMdwEgS&G}UWA6Dod{OaauGz&OT7Q0izop_~FFOf{0{a`_Q%5^4pJ&c4g@4?-k2kBdh%w5Z7WO z1exootm=8aXS!IHmpl5Ww0lVT`4&aNIiGR`n_j$v7B;;5-l&(XQi&uMAlrcP=|9{p z_`U@PjESCA->(`ZP>_Bdd`=PzP-E@u?>Cl|b?$c4_{Nr!LQL3<)Z5qh5&ovNZ~SZi z1WSj@8743W726ty={)ds%&XfaBRasA6^s$|-#J?rez5yxkGupk*1e=s4GX-v@ox6K z6*@AcB4BJV?Cq5@HMbU9G}vF;-zZCpX_PvuLvH72wHI}DtzhIe+Tc|)zT^l!M9mDhMA!dkrvc;i z{=pxc#BD4`^I#SBBLP~8VF*FbgN(zwpbF(LK9u?3px$iXJuRVvyzglHgLQt{_q=Kx z*(-5J>X&Y)BsO-ah?6b0WN2EFO7ZxVLw7X_%U|$}GVA~G>wkXl>NPBSYA$KG{`2eq zrw086ficp*$Mi>trw@zxcHS-BfCpSoBi|Nuc&?#B{ZD-He0kc%lrausHx)LXrvTT{ zPBwj`d#k0S z>m8E$zb4~5q68tD?}dq%rUl>iB*v@h`U4ZF?&HlVRzT)jpin?v&x;L z+3LnN<~BnzlYpx~@Z{L*=7pmt*Iwij-?}HP=G~N#lm6av;4g6*{-Cpzo7HY0!mEKR zFYZ5;E5DUJ360?pO=&*HBuEls>C5%W4A^noOWqsJN@XX4{!2=J-t)crvUUu^&5A|Z zStz0rOl$!+t)#LM%`vg0KAQ>IARDc{{D*166n*c12mTRCD}5Fu+o-1-BkFt#lTWK0 z1fR3_iFAG~Acp5tT8dWmia+`qI^N8#HWYCsIPmrBZ3waUY(ztqW8-t)MrY1iN2kHz z;ZX-htu_@@BI0DItEU)lIQ4sv_#0N;*wm(Km2baGL{`YnRM_Ec?C+;X2wy1Gtn#h_o4jXr;( z#r`kl^sg@By`o65Sir#{{Q7I zMI^wMED?+f(!BoHK>kw}{01AC1000 tokens) + language: PPL + query: | + source=otel-v1-apm-span-* + | WHERE `attributes.gen_ai.usage.input_tokens` + `attributes.gen_ai.usage.output_tokens` > 1000 + + - id: error_spans + title: Error spans + description: Find spans with errors + language: PPL + query: | + source=otel-v1-apm-span-* + | WHERE `status.code` = 2 + + - id: slow_operations + title: Slow operations + description: Find operations taking longer than 5 seconds + language: PPL + query: | + source=otel-v1-apm-span-* + | WHERE `durationInNanos` > 5000000000 + + - id: agent_by_model + title: LLM requests by model + description: Count LLM spans grouped by model + language: PPL + query: | + source=otel-v1-apm-span-* + | WHERE NOT isnull(`attributes.gen_ai.request.model`) AND NOT isnull(`traceId`) + | stats count() by `attributes.gen_ai.request.model` + + - id: tool_usage_stats + title: Tool usage statistics + description: Count tool execution spans by tool name + language: PPL + query: | + source=otel-v1-apm-span-* + | WHERE NOT isnull(`attributes.gen_ai.tool.name`) AND NOT isnull(`traceId`) + | stats count() by `attributes.gen_ai.tool.name` + + - id: token_usage_by_agent + title: Token usage by agent + description: Sum token usage from spans grouped by agent name + language: PPL + query: | + source=otel-v1-apm-span-* + | WHERE NOT isnull(`attributes.gen_ai.agent.name`) AND NOT isnull(`traceId`) + | stats sum(`attributes.gen_ai.usage.input_tokens`) as input_tokens, + sum(`attributes.gen_ai.usage.output_tokens`) as output_tokens + by `attributes.gen_ai.agent.name` + + - id: token_usage_by_model + title: Token usage by model + description: Sum token usage from spans grouped by model name + language: PPL + query: | + source=otel-v1-apm-span-* + | WHERE NOT isnull(`attributes.gen_ai.request.model`) AND NOT isnull(`traceId`) + | stats sum(`attributes.gen_ai.usage.input_tokens`) as input_tokens, + sum(`attributes.gen_ai.usage.output_tokens`) as output_tokens + by `attributes.gen_ai.request.model` + + - id: agent_operation_names + title: Agent operations used by services + description: Agent operations used by services + language: PPL + query: | + source=otel-v1-apm-span-* + | WHERE NOT isnull(`attributes.gen_ai.operation.name`) + | stats count() as operations by serviceName, `attributes.gen_ai.operation.name` diff --git a/charts/observability-stack/templates/init-dashboards-configmap.yaml b/charts/observability-stack/templates/init-dashboards-configmap.yaml index 7682b755..4f4e98bd 100644 --- a/charts/observability-stack/templates/init-dashboards-configmap.yaml +++ b/charts/observability-stack/templates/init-dashboards-configmap.yaml @@ -12,4 +12,10 @@ metadata: data: init-opensearch-dashboards.py: | {{ .Files.Get "files/init-opensearch-dashboards.py" | indent 4 }} + saved-queries-traces.yaml: | +{{ .Files.Get "files/saved-queries-traces.yaml" | indent 4 }} + saved-queries-metrics.yaml: | +{{ .Files.Get "files/saved-queries-metrics.yaml" | indent 4 }} +binaryData: + architecture.png: {{ .Files.Get "files/architecture.png" | b64enc }} {{- end }} diff --git a/charts/observability-stack/templates/init-dashboards-job.yaml b/charts/observability-stack/templates/init-dashboards-job.yaml index c0ace5e4..3c957106 100644 --- a/charts/observability-stack/templates/init-dashboards-job.yaml +++ b/charts/observability-stack/templates/init-dashboards-job.yaml @@ -47,6 +47,8 @@ spec: volumeMounts: - name: init-script mountPath: /scripts + - name: init-script + mountPath: /config volumes: - name: init-script configMap: From 32bf22eb9ee852b5221554175a512e48457333a1 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Wed, 18 Mar 2026 16:08:11 -0700 Subject: [PATCH 16/64] feat: add Terraform AWS EKS deployment (VPC, EKS, ALB, IRSA, helm release) --- terraform/aws/.gitignore | 6 + terraform/aws/README.md | 97 +++++++++++++ terraform/aws/addons.tf | 67 +++++++++ terraform/aws/main.tf | 189 +++++++++++++++++++++++++ terraform/aws/observability-stack.tf | 168 ++++++++++++++++++++++ terraform/aws/outputs.tf | 33 +++++ terraform/aws/terraform.tfvars.example | 13 ++ terraform/aws/values-eks.yaml | 23 +++ terraform/aws/variables.tf | 89 ++++++++++++ 9 files changed, 685 insertions(+) create mode 100644 terraform/aws/.gitignore create mode 100644 terraform/aws/README.md create mode 100644 terraform/aws/addons.tf create mode 100644 terraform/aws/main.tf create mode 100644 terraform/aws/observability-stack.tf create mode 100644 terraform/aws/outputs.tf create mode 100644 terraform/aws/terraform.tfvars.example create mode 100644 terraform/aws/values-eks.yaml create mode 100644 terraform/aws/variables.tf diff --git a/terraform/aws/.gitignore b/terraform/aws/.gitignore new file mode 100644 index 00000000..ca032f95 --- /dev/null +++ b/terraform/aws/.gitignore @@ -0,0 +1,6 @@ +# Terraform state and providers +.terraform/ +.terraform.lock.hcl +*.tfstate +*.tfstate.backup +terraform.tfvars diff --git a/terraform/aws/README.md b/terraform/aws/README.md new file mode 100644 index 00000000..2502d947 --- /dev/null +++ b/terraform/aws/README.md @@ -0,0 +1,97 @@ +# Observability Stack — AWS EKS Deployment + +Deploy the full observability stack to EKS in one command. Start with plain HTTP, add TLS/DNS when ready. + +## What You Get + +- EKS cluster with managed node group +- OpenSearch + Dashboards + Data Prepper + OTel Collector + Prometheus +- Internet-facing ALB (HTTP by default, HTTPS when domain is configured) +- Auto-generated OpenSearch credentials + +## Prerequisites + +- [Terraform](https://developer.hashicorp.com/terraform/install) >= 1.5 +- [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) configured with credentials +- ~15 minutes for initial deployment + +## Quick Start + +```bash +cd terraform/aws +terraform init +terraform apply +``` + +That's it. No config file needed. When complete (~15 min): + +```bash +# Configure kubectl +eval $(terraform output -raw kubeconfig_command) + +# Get the ALB URL +kubectl get ingress -n observability-stack + +# Get the password +terraform output -raw opensearch_password +``` + +Open the ALB DNS name in your browser. Login: `admin` / ``. + +## Add TLS + Custom Domain + +When the HTTP smoke test passes, add your domain: + +```bash +cp terraform.tfvars.example terraform.tfvars +``` + +Uncomment and set: +```hcl +domain = "obs.example.com" +route53_zone_id = "Z0123456789ABCDEFGHIJ" +``` + +```bash +terraform apply +``` + +This adds: +- ACM certificate (auto-validated via DNS) +- HTTPS listener with SSL redirect +- Route53 DNS record via external-dns + +## Add WAF Rate Limiting + +```hcl +enable_waf = true +``` + +Adds a WAFv2 WebACL: 2000 requests per 5 minutes per IP, 429 response when exceeded. + +## Send Telemetry + +```bash +kubectl port-forward -n observability-stack svc/obs-stack-opentelemetry-collector 4317:4317 4318:4318 & +export OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4317" +``` + +## Destroy + +```bash +terraform destroy +``` + +## All Variables + +| Variable | Default | Description | +|---|---|---| +| `region` | `us-west-2` | AWS region | +| `cluster_name` | `observability-stack` | EKS cluster name | +| `node_instance_type` | `m5.xlarge` | Worker node type (4 vCPU, 16GB) | +| `node_count` | `2` | Number of workers | +| `domain` | `""` (disabled) | Custom domain — enables HTTPS | +| `route53_zone_id` | `""` (disabled) | Route53 zone — required with domain | +| `enable_waf` | `false` | WAF rate limiting | +| `anonymous_auth` | `false` | Public read-only access | +| `enable_examples` | `false` | Example agent services | diff --git a/terraform/aws/addons.tf b/terraform/aws/addons.tf new file mode 100644 index 00000000..ace06ee8 --- /dev/null +++ b/terraform/aws/addons.tf @@ -0,0 +1,67 @@ +# ============================================================================ +# EKS Add-ons — AWS Load Balancer Controller, external-dns (conditional) +# ============================================================================ + +# --- AWS Load Balancer Controller --- + +resource "helm_release" "aws_lb_controller" { + name = "aws-load-balancer-controller" + repository = "https://aws.github.io/eks-charts" + chart = "aws-load-balancer-controller" + namespace = "kube-system" + version = "1.12.0" + + set { + name = "clusterName" + value = module.eks.cluster_name + } + set { + name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" + value = module.lb_controller_irsa.iam_role_arn + } + set { + name = "vpcId" + value = module.vpc.vpc_id + } + set { + name = "region" + value = var.region + } + + depends_on = [module.eks] +} + +# --- external-dns (only when domain is configured) --- + +resource "helm_release" "external_dns" { + count = local.enable_tls ? 1 : 0 + + name = "external-dns" + repository = "https://kubernetes-sigs.github.io/external-dns" + chart = "external-dns" + namespace = "kube-system" + version = "1.16.1" + + set { + name = "provider.name" + value = "aws" + } + set { + name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" + value = module.external_dns_irsa.iam_role_arn + } + set { + name = "domainFilters[0]" + value = var.domain + } + set { + name = "policy" + value = "sync" + } + set { + name = "txtOwnerId" + value = var.cluster_name + } + + depends_on = [module.eks] +} diff --git a/terraform/aws/main.tf b/terraform/aws/main.tf new file mode 100644 index 00000000..937d07c3 --- /dev/null +++ b/terraform/aws/main.tf @@ -0,0 +1,189 @@ +# ============================================================================ +# Observability Stack — AWS EKS Deployment +# +# Deploys a complete observability stack to EKS: +# VPC → EKS → Add-ons (LB Controller, external-dns) → Helm chart +# +# Usage: +# cp terraform.tfvars.example terraform.tfvars +# # Edit: domain, route53_zone_id +# terraform init +# terraform apply +# ============================================================================ + +terraform { + required_version = ">= 1.5" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + helm = { + source = "hashicorp/helm" + version = "~> 2.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.0" + } + } +} + +provider "aws" { + region = var.region + default_tags { + tags = var.tags + } +} + +data "aws_availability_zones" "available" { + state = "available" +} + +# ============================================================================ +# VPC +# ============================================================================ + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 5.0" + + name = var.cluster_name + cidr = "10.0.0.0/16" + + # 3 AZs for resilience + azs = slice(data.aws_availability_zones.available.names, 0, 3) + # /19 subnets = 8,190 IPs each — room for pod networking at scale + private_subnets = ["10.0.0.0/19", "10.0.32.0/19", "10.0.64.0/19"] + public_subnets = ["10.0.96.0/19", "10.0.128.0/19", "10.0.160.0/19"] + + enable_nat_gateway = true + single_nat_gateway = true # Cost optimization — one NAT vs one per AZ + enable_dns_hostnames = true + enable_dns_support = true + + # Tags required by AWS Load Balancer Controller for auto-discovery + public_subnet_tags = { + "kubernetes.io/role/elb" = 1 + } + private_subnet_tags = { + "kubernetes.io/role/internal-elb" = 1 + } +} + +# ============================================================================ +# EKS Cluster +# ============================================================================ + +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 20.0" + + cluster_name = var.cluster_name + cluster_version = var.kubernetes_version + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + # Public endpoint for kubectl access + cluster_endpoint_public_access = true + + # EKS add-ons + cluster_addons = { + coredns = {} + kube-proxy = {} + vpc-cni = {} + aws-ebs-csi-driver = { service_account_role_arn = module.ebs_csi_irsa.iam_role_arn } + eks-pod-identity-agent = {} + } + + eks_managed_node_groups = { + default = { + instance_types = [var.node_instance_type] + subnet_ids = module.vpc.private_subnets + min_size = var.node_count + max_size = var.node_count + 1 + desired_size = var.node_count + } + } + + # Allow current caller full admin access + enable_cluster_creator_admin_permissions = true +} + +# ============================================================================ +# IRSA — IAM Roles for Service Accounts +# ============================================================================ + +module "ebs_csi_irsa" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 5.0" + + role_name = "${var.cluster_name}-ebs-csi" + attach_ebs_csi_policy = true + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:ebs-csi-controller-sa"] + } + } +} + +module "lb_controller_irsa" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 5.0" + + role_name = "${var.cluster_name}-lb-controller" + attach_load_balancer_controller_policy = true + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:aws-load-balancer-controller"] + } + } +} + +module "external_dns_irsa" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 5.0" + + role_name = "${var.cluster_name}-external-dns" + attach_external_dns_policy = true + + external_dns_hosted_zone_arns = ["arn:aws:route53:::hostedzone/${var.route53_zone_id}"] + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:external-dns"] + } + } +} + +# ============================================================================ +# Kubernetes & Helm providers (configured after EKS is created) +# ============================================================================ + +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name, "--region", var.region] + } +} + +provider "helm" { + kubernetes { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name, "--region", var.region] + } + } +} diff --git a/terraform/aws/observability-stack.tf b/terraform/aws/observability-stack.tf new file mode 100644 index 00000000..2b48609d --- /dev/null +++ b/terraform/aws/observability-stack.tf @@ -0,0 +1,168 @@ +# ============================================================================ +# TLS — ACM certificate (only when domain is configured) +# ============================================================================ + +resource "aws_acm_certificate" "dashboards" { + count = local.enable_tls ? 1 : 0 + + domain_name = var.domain + validation_method = "DNS" + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_route53_record" "cert_validation" { + for_each = local.enable_tls ? { + for dvo in aws_acm_certificate.dashboards[0].domain_validation_options : dvo.domain_name => { + name = dvo.resource_record_name + record = dvo.resource_record_value + type = dvo.resource_record_type + } + } : {} + + zone_id = var.route53_zone_id + name = each.value.name + type = each.value.type + records = [each.value.record] + ttl = 60 + + allow_overwrite = true +} + +resource "aws_acm_certificate_validation" "dashboards" { + count = local.enable_tls ? 1 : 0 + + certificate_arn = aws_acm_certificate.dashboards[0].arn + validation_record_fqdns = [for record in aws_route53_record.cert_validation : record.fqdn] +} + +# ============================================================================ +# WAF — Rate limiting (opt-in) +# ============================================================================ + +resource "aws_wafv2_web_acl" "rate_limit" { + count = var.enable_waf ? 1 : 0 + + name = "${var.cluster_name}-rate-limit" + scope = "REGIONAL" + + default_action { + allow {} + } + + rule { + name = "rate-limit" + priority = 1 + + action { + block { + custom_response { + response_code = 429 + custom_response_body_key = "rate-limited" + } + } + } + + statement { + rate_based_statement { + limit = 2000 + aggregate_key_type = "IP" + } + } + + visibility_config { + sampled_requests_enabled = true + cloudwatch_metrics_enabled = true + metric_name = "${var.cluster_name}-rate-limit" + } + } + + custom_response_body { + key = "rate-limited" + content = "{\"error\": \"Rate limit exceeded. Try again in 5 minutes.\"}" + content_type = "APPLICATION_JSON" + } + + visibility_config { + sampled_requests_enabled = true + cloudwatch_metrics_enabled = true + metric_name = "${var.cluster_name}-waf" + } +} + +# ============================================================================ +# Observability Stack — Helm release +# ============================================================================ + +resource "kubernetes_namespace" "observability" { + metadata { + name = "observability-stack" + } + + depends_on = [module.eks] +} + +resource "helm_release" "observability_stack" { + name = "obs-stack" + chart = "${path.module}/../../charts/observability-stack" + namespace = kubernetes_namespace.observability.metadata[0].name + + timeout = 900 + wait = true + wait_for_jobs = true + cleanup_on_fail = true + + values = [file("${path.module}/values-eks.yaml")] + + # --- TLS / Domain (conditional) --- + dynamic "set" { + for_each = local.enable_tls ? [1] : [] + content { + name = "opensearch-dashboards.ingress.hosts[0].host" + value = var.domain + } + } + dynamic "set" { + for_each = local.enable_tls ? [1] : [] + content { + name = "opensearch-dashboards.ingress.annotations.alb\\.ingress\\.kubernetes\\.io/listen-ports" + value = "[{\"HTTPS\":443}]" + } + } + dynamic "set" { + for_each = local.enable_tls ? [1] : [] + content { + name = "opensearch-dashboards.ingress.annotations.alb\\.ingress\\.kubernetes\\.io/certificate-arn" + value = aws_acm_certificate.dashboards[0].arn + } + } + dynamic "set" { + for_each = local.enable_tls ? [1] : [] + content { + name = "opensearch-dashboards.ingress.annotations.alb\\.ingress\\.kubernetes\\.io/ssl-redirect" + value = "443" + } + } + dynamic "set" { + for_each = local.enable_tls ? [1] : [] + content { + name = "opensearch-dashboards.ingress.annotations.external-dns\\.alpha\\.kubernetes\\.io/hostname" + value = var.domain + } + } + + # --- WAF (conditional) --- + dynamic "set" { + for_each = var.enable_waf ? [1] : [] + content { + name = "opensearch-dashboards.ingress.annotations.alb\\.ingress\\.kubernetes\\.io/wafv2-acl-arn" + value = aws_wafv2_web_acl.rate_limit[0].arn + } + } + + depends_on = [ + helm_release.aws_lb_controller, + ] +} diff --git a/terraform/aws/outputs.tf b/terraform/aws/outputs.tf new file mode 100644 index 00000000..b68a71b3 --- /dev/null +++ b/terraform/aws/outputs.tf @@ -0,0 +1,33 @@ +# ============================================================================ +# Outputs +# ============================================================================ + +output "dashboards_url" { + description = "OpenSearch Dashboards URL" + value = local.enable_tls ? "https://${var.domain}" : "http:// — run: kubectl get ingress -n observability-stack" +} + +output "credentials" { + description = "OpenSearch Dashboards login" + value = "Username: admin | Password: My_password_123!@#" +} + +output "kubeconfig_command" { + description = "Command to configure kubectl" + value = "aws eks update-kubeconfig --name ${module.eks.cluster_name} --region ${var.region}" +} + +output "cluster_name" { + description = "EKS cluster name" + value = module.eks.cluster_name +} + +output "otlp_endpoint" { + description = "OTLP endpoint (port-forward required)" + value = "kubectl port-forward -n observability-stack svc/obs-stack-opentelemetry-collector 4317:4317 4318:4318" +} + +output "next_steps" { + description = "To add TLS and custom domain" + value = local.enable_tls ? "✅ TLS enabled at https://${var.domain}" : "Add domain and route53_zone_id to terraform.tfvars, then terraform apply" +} diff --git a/terraform/aws/terraform.tfvars.example b/terraform/aws/terraform.tfvars.example new file mode 100644 index 00000000..5f5ea787 --- /dev/null +++ b/terraform/aws/terraform.tfvars.example @@ -0,0 +1,13 @@ +# All optional. Just `terraform apply` for a working stack on plain HTTP. +# Uncomment domain + route53_zone_id to enable HTTPS. + +# domain = "obs.example.com" +# route53_zone_id = "Z0123456789ABCDEFGHIJ" + +# region = "us-west-2" +# cluster_name = "observability-stack" +# node_instance_type = "m5.xlarge" +# node_count = 2 +# enable_waf = false +# anonymous_auth = false +# enable_examples = false diff --git a/terraform/aws/values-eks.yaml b/terraform/aws/values-eks.yaml new file mode 100644 index 00000000..50066d15 --- /dev/null +++ b/terraform/aws/values-eks.yaml @@ -0,0 +1,23 @@ +opensearch: + persistence: + storageClass: gp2 + +opensearch-dashboards: + ingress: + enabled: true + ingressClassName: alb + annotations: + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/target-type: ip + alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80}]' + hosts: + - paths: + - path: / + backend: + servicePort: 5601 + +gateway: + enabled: false + +examples: + enabled: false diff --git a/terraform/aws/variables.tf b/terraform/aws/variables.tf new file mode 100644 index 00000000..e015e040 --- /dev/null +++ b/terraform/aws/variables.tf @@ -0,0 +1,89 @@ +# ============================================================================ +# No required variables. Just `terraform apply` for a working stack. +# Add domain + route53_zone_id when ready for TLS/DNS. +# ============================================================================ + +variable "region" { + description = "AWS region" + type = string + default = "us-west-2" +} + +variable "cluster_name" { + description = "EKS cluster name" + type = string + default = "observability-stack" +} + +variable "kubernetes_version" { + description = "EKS Kubernetes version" + type = string + default = "1.32" +} + +variable "node_instance_type" { + description = "EC2 instance type for EKS nodes" + type = string + default = "m5.xlarge" +} + +variable "node_count" { + description = "Number of EKS worker nodes" + type = number + default = 2 +} + +# ============================================================================ +# TLS / DNS — optional. Set both to enable HTTPS + custom domain. +# ============================================================================ + +variable "domain" { + description = "Domain name for OpenSearch Dashboards (e.g. obs.example.com). Leave empty for plain HTTP on ALB." + type = string + default = "" +} + +variable "route53_zone_id" { + description = "Route53 hosted zone ID. Required when domain is set." + type = string + default = "" +} + +# ============================================================================ +# Security — off by default for initial smoke test +# ============================================================================ + +variable "enable_waf" { + description = "Enable WAF rate limiting on the ALB (2000 req/5min/IP)" + type = bool + default = false +} + +variable "anonymous_auth" { + description = "Enable anonymous read-only access to OpenSearch Dashboards (for public demos)" + type = bool + default = false +} + +variable "enable_examples" { + description = "Deploy example agent services (weather-agent, travel-planner, canary)" + type = bool + default = false +} + +variable "tags" { + description = "Tags applied to all resources" + type = map(string) + default = { + Project = "observability-stack" + ManagedBy = "terraform" + } +} + +# ============================================================================ +# Derived +# ============================================================================ + +locals { + enable_tls = var.domain != "" && var.route53_zone_id != "" +} From b4f74f5bb916e2a026195c78abbf7639a3cda32f Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Wed, 18 Mar 2026 16:08:25 -0700 Subject: [PATCH 17/64] feat: add Prometheus OTLP config, GenAI semconv attrs, node-exporter, kube-state-metrics --- charts/observability-stack/values.yaml | 74 +++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 2 deletions(-) diff --git a/charts/observability-stack/values.yaml b/charts/observability-stack/values.yaml index 5ca03112..81b38a03 100644 --- a/charts/observability-stack/values.yaml +++ b/charts/observability-stack/values.yaml @@ -256,6 +256,11 @@ opentelemetry-collector: containerPort: 4318 servicePort: 4318 protocol: TCP + metrics: + enabled: true + containerPort: 8888 + servicePort: 8888 + protocol: TCP # -- Prometheus prometheus: @@ -266,12 +271,77 @@ prometheus: extraFlags: - "web.enable-remote-write-receiver" - "web.enable-otlp-receiver" + global: + scrape_interval: 60s + scrape_timeout: 10s + evaluation_interval: 60s + external_labels: + cluster: 'observability-stack' + environment: 'development' + # Disable default K8s scrape configs — they add ~60k series from API server internals. + # We keep node-exporter and kube-state-metrics (scraped via service endpoints below), + # and add explicit scrape for OTel Collector and Prometheus self-monitoring. + scrapeConfigs: + # Disable noisy defaults + kubernetes-api-servers: { enabled: false } + kubernetes-nodes: { enabled: false } + kubernetes-nodes-cadvisor: { enabled: false } + kubernetes-pods: { enabled: false } + kubernetes-pods-slow: { enabled: false } + kubernetes-service-endpoints: { enabled: false } + kubernetes-service-endpoints-slow: { enabled: false } + kubernetes-services: { enabled: false } + prometheus-pushgateway: { enabled: false } + # Keep self-monitoring + OTel Collector + prometheus: + static_configs: + - targets: ['localhost:9090'] + otel-collector: + static_configs: + - targets: ['obs-stack-opentelemetry-collector:8888'] + scrape_interval: 10s + node-exporter: + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - source_labels: [__meta_kubernetes_service_name] + regex: obs-stack-prometheus-node-exporter + action: keep + kube-state-metrics: + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - source_labels: [__meta_kubernetes_service_name] + regex: obs-stack-kube-state-metrics + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + regex: http + action: keep + scrapeConfigFiles: [] + serverFiles: + prometheus.yml: + otlp: + keep_identifying_resource_attributes: true + promote_resource_attributes: + - service.instance.id + - service.name + - service.namespace + - service.version + - deployment.environment.name + - gen_ai.agent.id + - gen_ai.agent.name + - gen_ai.provider.name + - gen_ai.request.model + - gen_ai.response.model + storage: + tsdb: + out_of_order_time_window: 30m alertmanager: enabled: false kube-state-metrics: - enabled: false + enabled: true prometheus-node-exporter: - enabled: false + enabled: true prometheus-pushgateway: enabled: false From 7c9e36d9daf65a977d526d960fad3222be2af7fa Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Wed, 18 Mar 2026 16:08:40 -0700 Subject: [PATCH 18/64] feat: add K8s Cluster Health and Observability Pipeline Health dashboards --- .../files/dashboard-k8s-cluster-health.yaml | 37 ++++ .../files/dashboard-pipeline-health.yaml | 47 +++++ .../files/init-opensearch-dashboards.py | 173 ++++++++++++++++++ .../templates/init-dashboards-configmap.yaml | 4 + 4 files changed, 261 insertions(+) create mode 100644 charts/observability-stack/files/dashboard-k8s-cluster-health.yaml create mode 100644 charts/observability-stack/files/dashboard-pipeline-health.yaml diff --git a/charts/observability-stack/files/dashboard-k8s-cluster-health.yaml b/charts/observability-stack/files/dashboard-k8s-cluster-health.yaml new file mode 100644 index 00000000..767b990c --- /dev/null +++ b/charts/observability-stack/files/dashboard-k8s-cluster-health.yaml @@ -0,0 +1,37 @@ +# Kubernetes Cluster Health Dashboard — Node and pod infrastructure metrics + +dashboard: + id: k8s-cluster-health-dashboard + title: Kubernetes Cluster Health + description: Node resources, pod health, and container status + +panels: + - id: k8s-node-count + title: "Node Count" + query: "count(kube_node_info)" + chartType: line + + - id: k8s-pod-restarts + title: "Container Restarts (1h)" + query: "increase(kube_pod_container_status_restarts_total{namespace=\"observability-stack\"}[1h])" + chartType: line + + - id: k8s-node-cpu-usage + title: "Node CPU Usage %" + query: "100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)" + chartType: line + + - id: k8s-node-memory-usage + title: "Node Memory Usage %" + query: "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)" + chartType: line + + - id: k8s-node-disk-usage + title: "Node Disk Usage %" + query: "100 - (node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"} * 100)" + chartType: line + + - id: k8s-node-network-receive + title: "Node Network Receive (bytes/sec)" + query: "rate(node_network_receive_bytes_total{device!=\"lo\"}[5m])" + chartType: line diff --git a/charts/observability-stack/files/dashboard-pipeline-health.yaml b/charts/observability-stack/files/dashboard-pipeline-health.yaml new file mode 100644 index 00000000..8360aa87 --- /dev/null +++ b/charts/observability-stack/files/dashboard-pipeline-health.yaml @@ -0,0 +1,47 @@ +# Observability Pipeline Health Dashboard — OTel Collector and Prometheus self-monitoring + +dashboard: + id: observability-pipeline-health-dashboard + title: Observability Pipeline Health + description: OTel Collector throughput, Prometheus ingestion, and pipeline health + +panels: + - id: pipeline-otel-spans-received + title: "OTel Spans Received/sec" + query: "rate(otelcol_receiver_accepted_spans_total[5m])" + chartType: line + + - id: pipeline-otel-spans-exported + title: "OTel Spans Exported/sec" + query: "rate(otelcol_exporter_sent_spans_total[5m])" + chartType: line + + - id: pipeline-otel-metrics-received + title: "OTel Metrics Received/sec" + query: "rate(otelcol_receiver_accepted_metric_points_total[5m])" + chartType: line + + - id: pipeline-otel-queue-size + title: "OTel Exporter Queue Size" + query: "otelcol_exporter_queue_size" + chartType: line + + - id: pipeline-prometheus-ingestion + title: "Prometheus Ingestion Rate (samples/sec)" + query: "rate(prometheus_tsdb_head_samples_appended_total[5m])" + chartType: line + + - id: pipeline-prometheus-active-series + title: "Prometheus Active Time Series" + query: "prometheus_tsdb_head_series" + chartType: line + + - id: pipeline-prometheus-storage + title: "Prometheus Storage Size (bytes)" + query: "prometheus_tsdb_storage_blocks_bytes" + chartType: line + + - id: pipeline-otel-collector-memory + title: "OTel Collector Memory (bytes)" + query: "otelcol_process_memory_rss_bytes" + chartType: line diff --git a/charts/observability-stack/files/init-opensearch-dashboards.py b/charts/observability-stack/files/init-opensearch-dashboards.py index 5afb09bd..b51c7b36 100644 --- a/charts/observability-stack/files/init-opensearch-dashboards.py +++ b/charts/observability-stack/files/init-opensearch-dashboards.py @@ -945,6 +945,173 @@ def create_chart_visualization(workspace_id, vis_id, title, vis_type, field, ind return None +def create_promql_dashboard_from_yaml(workspace_id, config_path, prometheus_datasource_title="ObservabilityStack_Prometheus"): + """Create a dashboard with PromQL explore panels from a YAML config file""" + import json + + try: + with open(config_path, "r") as f: + config = yaml.safe_load(f) + except (FileNotFoundError, yaml.YAMLError) as e: + print(f"⚠️ Skipping dashboard from {config_path}: {e}") + return None + + dashboard_config = config.get("dashboard", {}) + panel_defs = config.get("panels", []) + dashboard_id = dashboard_config.get("id", "k8s-cluster-health-dashboard") + + print(f"📊 Creating {dashboard_config.get('title', 'K8s Cluster Health')} dashboard ({len(panel_defs)} panels)...") + + # Visualization template for explore panels + viz_template = json.dumps({ + "title": "", + "chartType": "line", + "params": { + "addLegend": True, "addTimeMarker": False, "legendPosition": "bottom", + "legendTitle": "", "lineMode": "straight", "lineStyle": "line", "lineWidth": 2, + "showFullTimeRange": False, "standardAxes": [], + "thresholdOptions": {"baseColor": "#00BD6B", "thresholds": [], "thresholdStyle": "off"}, + "titleOptions": {"show": False, "titleName": ""}, + "tooltipOptions": {"mode": "all"} + }, + "axesMapping": {"color": "Series", "x": "Time", "y": "Value"} + }) + + dataset = { + "id": prometheus_datasource_title, + "title": prometheus_datasource_title, + "type": "PROMETHEUS", + "language": "PROMQL", + "timeFieldName": "Time", + "dataSource": {}, + "signalType": "metrics" + } + + created_ids = [] + for panel_def in panel_defs: + panel_id = panel_def["id"] + search_source = json.dumps({ + "query": { + "query": panel_def["query"], + "language": "PROMQL", + "dataset": dataset + }, + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index" + }) + + payload = { + "attributes": { + "title": panel_def["title"], + "description": "", + "hits": 0, + "columns": ["_source"], + "sort": [], + "version": 1, + "type": "metrics", + "visualization": viz_template, + "uiState": json.dumps({"activeTab": "explore_visualization_tab"}), + "kibanaSavedObjectMeta": { + "searchSourceJSON": search_source + } + }, + "references": [{ + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": prometheus_datasource_title + }] + } + + if workspace_id and workspace_id != "default": + payload["workspaces"] = [workspace_id] + url = f"{BASE_URL}/w/{workspace_id}/api/saved_objects/explore/{panel_id}" + else: + url = f"{BASE_URL}/api/saved_objects/explore/{panel_id}" + + try: + response = requests.post( + url, auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + json=payload, verify=False, timeout=10, + ) + if response.status_code == 200: + created_ids.append(panel_id) + print(f" ✅ {panel_def['title']}") + elif response.status_code == 409: + # Update existing + requests.put( + url, auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + json={"attributes": payload["attributes"], "references": payload["references"]}, + verify=False, timeout=10, + ) + created_ids.append(panel_id) + print(f" 🔄 {panel_def['title']} (updated)") + else: + print(f" ⚠️ {panel_def['title']}: {response.status_code} {response.text[:100]}") + except requests.exceptions.RequestException as e: + print(f" ⚠️ {panel_def['title']}: {e}") + + if not created_ids: + print("⚠️ No panels created, skipping dashboard") + return None + + # Assemble dashboard — 2 panels per row, 24 units wide each + panels = [] + references = [] + for i, pid in enumerate(created_ids): + panels.append({ + "version": "3.6.0", + "panelIndex": pid, + "gridData": {"i": pid, "x": (i % 2) * 24, "y": (i // 2) * 15, "w": 24, "h": 15}, + "panelRefName": f"panel_{i}" + }) + references.append({"name": f"panel_{i}", "type": "explore", "id": pid}) + + dashboard_payload = { + "attributes": { + "title": dashboard_config.get("title", "Kubernetes Cluster Health"), + "description": dashboard_config.get("description", ""), + "panelsJSON": json.dumps(panels), + "optionsJSON": json.dumps({"useMargins": True, "hidePanelTitles": False}), + "timeRestore": False, + "kibanaSavedObjectMeta": { + "searchSourceJSON": json.dumps({}) + } + }, + "references": references + } + + if workspace_id and workspace_id != "default": + dashboard_payload["workspaces"] = [workspace_id] + url = f"{BASE_URL}/w/{workspace_id}/api/saved_objects/dashboard/{dashboard_id}" + else: + url = f"{BASE_URL}/api/saved_objects/dashboard/{dashboard_id}" + + try: + response = requests.post( + url, auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + json=dashboard_payload, verify=False, timeout=10, + ) + if response.status_code in (200, 409): + if response.status_code == 409: + requests.put( + url, auth=(USERNAME, PASSWORD), + headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + json={"attributes": dashboard_payload["attributes"], "references": references}, + verify=False, timeout=10, + ) + print(f"✅ Created {dashboard_config['title']} dashboard ({len(created_ids)} panels)") + return dashboard_id + else: + print(f"⚠️ Dashboard creation failed: {response.text[:200]}") + return None + except requests.exceptions.RequestException as e: + print(f"⚠️ Error creating dashboard: {e}") + return None + + def create_overview_dashboard(workspace_id): """Create an overview landing dashboard with markdown links to all observability features""" import json @@ -1166,6 +1333,12 @@ def main(): # Create overview landing dashboard (becomes the new default) create_overview_dashboard(workspace_id) + # Create K8s cluster health dashboard (PromQL explore panels) + create_promql_dashboard_from_yaml(workspace_id, "/config/dashboard-k8s-cluster-health.yaml") + + # Create observability pipeline health dashboard + create_promql_dashboard_from_yaml(workspace_id, "/config/dashboard-pipeline-health.yaml") + # Create saved queries for common agent observability patterns create_default_saved_queries(workspace_id) diff --git a/charts/observability-stack/templates/init-dashboards-configmap.yaml b/charts/observability-stack/templates/init-dashboards-configmap.yaml index 4f4e98bd..90becb32 100644 --- a/charts/observability-stack/templates/init-dashboards-configmap.yaml +++ b/charts/observability-stack/templates/init-dashboards-configmap.yaml @@ -16,6 +16,10 @@ data: {{ .Files.Get "files/saved-queries-traces.yaml" | indent 4 }} saved-queries-metrics.yaml: | {{ .Files.Get "files/saved-queries-metrics.yaml" | indent 4 }} + dashboard-k8s-cluster-health.yaml: | +{{ .Files.Get "files/dashboard-k8s-cluster-health.yaml" | indent 4 }} + dashboard-pipeline-health.yaml: | +{{ .Files.Get "files/dashboard-pipeline-health.yaml" | indent 4 }} binaryData: architecture.png: {{ .Files.Get "files/architecture.png" | b64enc }} {{- end }} From 5a6a9da03788e3365f5943626c24023ea9b0e7e4 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Wed, 18 Mar 2026 16:15:57 -0700 Subject: [PATCH 19/64] feat: expand self-monitoring dashboards (pods by status, memory vs request, OTel CPU, spans dropped, Prometheus query latency) --- .../files/dashboard-k8s-cluster-health.yaml | 20 ++++++++++-- .../files/dashboard-pipeline-health.yaml | 32 +++++++++++++++++-- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/charts/observability-stack/files/dashboard-k8s-cluster-health.yaml b/charts/observability-stack/files/dashboard-k8s-cluster-health.yaml index 767b990c..f8f9f050 100644 --- a/charts/observability-stack/files/dashboard-k8s-cluster-health.yaml +++ b/charts/observability-stack/files/dashboard-k8s-cluster-health.yaml @@ -6,16 +6,18 @@ dashboard: description: Node resources, pod health, and container status panels: + # --- Row 1: Cluster Overview --- - id: k8s-node-count title: "Node Count" query: "count(kube_node_info)" chartType: line - - id: k8s-pod-restarts - title: "Container Restarts (1h)" - query: "increase(kube_pod_container_status_restarts_total{namespace=\"observability-stack\"}[1h])" + - id: k8s-pods-by-status + title: "Pods by Status" + query: "sum by (phase) (kube_pod_status_phase{namespace=\"observability-stack\"})" chartType: line + # --- Row 2: Node CPU & Memory --- - id: k8s-node-cpu-usage title: "Node CPU Usage %" query: "100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)" @@ -26,6 +28,7 @@ panels: query: "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)" chartType: line + # --- Row 3: Disk & Network --- - id: k8s-node-disk-usage title: "Node Disk Usage %" query: "100 - (node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"} * 100)" @@ -35,3 +38,14 @@ panels: title: "Node Network Receive (bytes/sec)" query: "rate(node_network_receive_bytes_total{device!=\"lo\"}[5m])" chartType: line + + # --- Row 4: Pod Health --- + - id: k8s-pod-restarts + title: "Container Restarts (1h)" + query: "increase(kube_pod_container_status_restarts_total{namespace=\"observability-stack\"}[1h])" + chartType: line + + - id: k8s-pod-memory-vs-request + title: "Pod Memory Usage vs Request %" + query: "100 * sum by (pod) (container_memory_working_set_bytes{namespace=\"observability-stack\", container!=\"\"}) / sum by (pod) (kube_pod_container_resource_requests{namespace=\"observability-stack\", resource=\"memory\"})" + chartType: line diff --git a/charts/observability-stack/files/dashboard-pipeline-health.yaml b/charts/observability-stack/files/dashboard-pipeline-health.yaml index 8360aa87..97e350fa 100644 --- a/charts/observability-stack/files/dashboard-pipeline-health.yaml +++ b/charts/observability-stack/files/dashboard-pipeline-health.yaml @@ -6,6 +6,7 @@ dashboard: description: OTel Collector throughput, Prometheus ingestion, and pipeline health panels: + # --- Row 1: OTel Collector Throughput --- - id: pipeline-otel-spans-received title: "OTel Spans Received/sec" query: "rate(otelcol_receiver_accepted_spans_total[5m])" @@ -16,16 +17,40 @@ panels: query: "rate(otelcol_exporter_sent_spans_total[5m])" chartType: line + # --- Row 2: OTel Metrics & Failures --- - id: pipeline-otel-metrics-received title: "OTel Metrics Received/sec" query: "rate(otelcol_receiver_accepted_metric_points_total[5m])" chartType: line + - id: pipeline-otel-spans-dropped + title: "OTel Spans Dropped/sec" + query: "rate(otelcol_exporter_send_failed_spans_total[5m])" + chartType: line + + # --- Row 3: OTel Collector Resources --- - id: pipeline-otel-queue-size title: "OTel Exporter Queue Size" query: "otelcol_exporter_queue_size" chartType: line + - id: pipeline-otel-collector-memory + title: "OTel Collector Memory (bytes)" + query: "otelcol_process_memory_rss_bytes" + chartType: line + + # --- Row 4: OTel Collector CPU & Uptime --- + - id: pipeline-otel-collector-cpu + title: "OTel Collector CPU Usage" + query: "rate(otelcol_process_cpu_seconds_total[5m])" + chartType: line + + - id: pipeline-otel-batch-cardinality + title: "OTel Batch Metadata Cardinality" + query: "otelcol_processor_batch_metadata_cardinality" + chartType: line + + # --- Row 5: Prometheus Health --- - id: pipeline-prometheus-ingestion title: "Prometheus Ingestion Rate (samples/sec)" query: "rate(prometheus_tsdb_head_samples_appended_total[5m])" @@ -36,12 +61,13 @@ panels: query: "prometheus_tsdb_head_series" chartType: line + # --- Row 6: Prometheus Storage --- - id: pipeline-prometheus-storage title: "Prometheus Storage Size (bytes)" query: "prometheus_tsdb_storage_blocks_bytes" chartType: line - - id: pipeline-otel-collector-memory - title: "OTel Collector Memory (bytes)" - query: "otelcol_process_memory_rss_bytes" + - id: pipeline-prometheus-query-latency + title: "Prometheus Query Latency P99 (sec)" + query: "histogram_quantile(0.99, rate(prometheus_http_request_duration_seconds_bucket{handler=\"/api/v1/query\"}[5m]))" chartType: line From c12f2022be96a4c81b1edc8b3e99bd13cdc001c4 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Wed, 18 Mar 2026 16:31:34 -0700 Subject: [PATCH 20/64] feat: add OpenSearch cluster health dashboard with prometheus-opensearch-exporter --- .../files/dashboard-opensearch-health.yaml | 62 ++++++++++++++++++ .../files/init-opensearch-dashboards.py | 3 + .../templates/init-dashboards-configmap.yaml | 2 + .../templates/opensearch-exporter.yaml | 65 +++++++++++++++++++ charts/observability-stack/values.yaml | 11 ++++ 5 files changed, 143 insertions(+) create mode 100644 charts/observability-stack/files/dashboard-opensearch-health.yaml create mode 100644 charts/observability-stack/templates/opensearch-exporter.yaml diff --git a/charts/observability-stack/files/dashboard-opensearch-health.yaml b/charts/observability-stack/files/dashboard-opensearch-health.yaml new file mode 100644 index 00000000..cc507285 --- /dev/null +++ b/charts/observability-stack/files/dashboard-opensearch-health.yaml @@ -0,0 +1,62 @@ +# OpenSearch Cluster Health Dashboard — Cluster, index, JVM, and search metrics + +dashboard: + id: opensearch-cluster-health-dashboard + title: OpenSearch Cluster Health + description: Cluster status, index stats, JVM health, and indexing performance + +panels: + # --- Row 1: Cluster Status --- + - id: os-cluster-status + title: "Cluster Status (0=green, 1=yellow/red)" + query: "elasticsearch_cluster_health_status{color=\"yellow\"} or elasticsearch_cluster_health_status{color=\"red\"}" + chartType: line + + - id: os-active-shards + title: "Active Shards" + query: "elasticsearch_cluster_health_active_shards" + chartType: line + + # --- Row 2: Index Stats --- + - id: os-unassigned-shards + title: "Unassigned Shards" + query: "elasticsearch_cluster_health_unassigned_shards" + chartType: line + + - id: os-docs-count + title: "Total Documents" + query: "sum(elasticsearch_indices_docs)" + chartType: line + + # --- Row 3: Indexing & Storage --- + - id: os-indexing-rate + title: "Indexing Rate (docs/sec)" + query: "rate(elasticsearch_indices_indexing_index_total[5m])" + chartType: line + + - id: os-store-size + title: "Store Size (bytes)" + query: "sum(elasticsearch_indices_store_size_bytes_total)" + chartType: line + + # --- Row 4: JVM Health --- + - id: os-jvm-heap-used-pct + title: "JVM Heap Used %" + query: "100 * elasticsearch_jvm_memory_used_bytes{area=\"heap\"} / elasticsearch_jvm_memory_max_bytes{area=\"heap\"}" + chartType: line + + - id: os-jvm-gc-rate + title: "JVM GC Collection Rate (sec/sec)" + query: "rate(elasticsearch_jvm_gc_collection_seconds_sum[5m])" + chartType: line + + # --- Row 5: Search & CPU --- + - id: os-search-rate + title: "Search Rate (queries/sec)" + query: "rate(elasticsearch_indices_search_query_total[5m])" + chartType: line + + - id: os-cpu-percent + title: "OpenSearch CPU %" + query: "elasticsearch_os_cpu_percent" + chartType: line diff --git a/charts/observability-stack/files/init-opensearch-dashboards.py b/charts/observability-stack/files/init-opensearch-dashboards.py index b51c7b36..f7cb9f4b 100644 --- a/charts/observability-stack/files/init-opensearch-dashboards.py +++ b/charts/observability-stack/files/init-opensearch-dashboards.py @@ -1339,6 +1339,9 @@ def main(): # Create observability pipeline health dashboard create_promql_dashboard_from_yaml(workspace_id, "/config/dashboard-pipeline-health.yaml") + # Create OpenSearch cluster health dashboard + create_promql_dashboard_from_yaml(workspace_id, "/config/dashboard-opensearch-health.yaml") + # Create saved queries for common agent observability patterns create_default_saved_queries(workspace_id) diff --git a/charts/observability-stack/templates/init-dashboards-configmap.yaml b/charts/observability-stack/templates/init-dashboards-configmap.yaml index 90becb32..2496dff0 100644 --- a/charts/observability-stack/templates/init-dashboards-configmap.yaml +++ b/charts/observability-stack/templates/init-dashboards-configmap.yaml @@ -20,6 +20,8 @@ data: {{ .Files.Get "files/dashboard-k8s-cluster-health.yaml" | indent 4 }} dashboard-pipeline-health.yaml: | {{ .Files.Get "files/dashboard-pipeline-health.yaml" | indent 4 }} + dashboard-opensearch-health.yaml: | +{{ .Files.Get "files/dashboard-opensearch-health.yaml" | indent 4 }} binaryData: architecture.png: {{ .Files.Get "files/architecture.png" | b64enc }} {{- end }} diff --git a/charts/observability-stack/templates/opensearch-exporter.yaml b/charts/observability-stack/templates/opensearch-exporter.yaml new file mode 100644 index 00000000..6343180a --- /dev/null +++ b/charts/observability-stack/templates/opensearch-exporter.yaml @@ -0,0 +1,65 @@ +{{- if .Values.opensearchExporter.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "observability-stack.fullname" . }}-opensearch-exporter + labels: + {{- include "observability-stack.labels" . | nindent 4 }} + app.kubernetes.io/component: opensearch-exporter +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: opensearch-exporter + template: + metadata: + labels: + app.kubernetes.io/component: opensearch-exporter + spec: + containers: + - name: opensearch-exporter + image: "{{ .Values.opensearchExporter.image.repository }}:{{ .Values.opensearchExporter.image.tag }}" + args: + - --es.uri=https://opensearch-cluster-master:9200 + - --es.ssl-skip-verify + - --es.all + - --es.indices + - --es.shards + env: + - name: ES_USERNAME + value: {{ .Values.opensearchUsername | default "admin" | quote }} + - name: ES_PASSWORD + value: {{ .Values.opensearchPassword | default "My_password_123!@#" | quote }} + ports: + - containerPort: 9114 + name: metrics + resources: + limits: + memory: "128Mi" + cpu: "100m" + livenessProbe: + httpGet: + path: /healthz + port: 9114 + initialDelaySeconds: 10 + readinessProbe: + httpGet: + path: /healthz + port: 9114 + initialDelaySeconds: 10 +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "observability-stack.fullname" . }}-opensearch-exporter + labels: + {{- include "observability-stack.labels" . | nindent 4 }} + app.kubernetes.io/component: opensearch-exporter +spec: + selector: + app.kubernetes.io/component: opensearch-exporter + ports: + - port: 9114 + targetPort: 9114 + name: metrics +{{- end }} diff --git a/charts/observability-stack/values.yaml b/charts/observability-stack/values.yaml index 81b38a03..c502395e 100644 --- a/charts/observability-stack/values.yaml +++ b/charts/observability-stack/values.yaml @@ -300,6 +300,10 @@ prometheus: static_configs: - targets: ['obs-stack-opentelemetry-collector:8888'] scrape_interval: 10s + opensearch: + static_configs: + - targets: ['obs-stack-observability-stack-opensearch-exporter:9114'] + scrape_interval: 30s node-exporter: kubernetes_sd_configs: - role: endpoints @@ -353,6 +357,13 @@ prometheus: opensearchUsername: "admin" opensearchPassword: "My_password_123!@#" +# -- OpenSearch Prometheus Exporter +opensearchExporter: + enabled: true + image: + repository: prometheuscommunity/elasticsearch-exporter + tag: v1.10.0 + # -- Example agents (matches docker-compose.examples.yml) examples: enabled: true From e604eefbe5e4e7e79814d601f7a9fc6162f393b4 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Wed, 18 Mar 2026 16:45:54 -0700 Subject: [PATCH 21/64] feat: add self-monitoring to docker-compose (opensearch-exporter, pipeline + OpenSearch health dashboards) --- .../dashboard-pipeline-health.yaml | 75 ++----------------- 1 file changed, 5 insertions(+), 70 deletions(-) diff --git a/docker-compose/opensearch-dashboards/dashboard-pipeline-health.yaml b/docker-compose/opensearch-dashboards/dashboard-pipeline-health.yaml index 4ef8020d..97e350fa 100644 --- a/docker-compose/opensearch-dashboards/dashboard-pipeline-health.yaml +++ b/docker-compose/opensearch-dashboards/dashboard-pipeline-health.yaml @@ -52,8 +52,8 @@ panels: # --- Row 5: Prometheus Health --- - id: pipeline-prometheus-ingestion - title: "Prometheus Ingestion Rate (chunks/sec)" - query: "rate(prometheus_tsdb_head_chunks_created_total[5m])" + title: "Prometheus Ingestion Rate (samples/sec)" + query: "rate(prometheus_tsdb_head_samples_appended_total[5m])" chartType: line - id: pipeline-prometheus-active-series @@ -62,77 +62,12 @@ panels: chartType: line # --- Row 6: Prometheus Storage --- - - id: pipeline-prometheus-wal-size - title: "Prometheus WAL Size (bytes)" - query: "prometheus_tsdb_wal_storage_size_bytes" - chartType: line - - - id: pipeline-prometheus-head-chunks - title: "Prometheus Head Chunks Size (bytes)" - query: "prometheus_tsdb_head_chunks_storage_size_bytes" + - id: pipeline-prometheus-storage + title: "Prometheus Storage Size (bytes)" + query: "prometheus_tsdb_storage_blocks_bytes" chartType: line - id: pipeline-prometheus-query-latency title: "Prometheus Query Latency P99 (sec)" query: "histogram_quantile(0.99, rate(prometheus_http_request_duration_seconds_bucket{handler=\"/api/v1/query\"}[5m]))" chartType: line - - # --- Row 7: Data Prepper — Logs Pipeline --- - - id: pipeline-dp-logs-processed - title: "DP Logs Processed/sec" - query: "rate(otel_logs_pipeline_recordsProcessed_total[5m])" - chartType: line - - - id: pipeline-dp-logs-latency - title: "DP Logs Pipeline Latency (avg sec)" - query: "rate(otel_logs_pipeline_opensearch_PipelineLatency_seconds_sum[5m]) / rate(otel_logs_pipeline_opensearch_PipelineLatency_seconds_count[5m])" - chartType: line - - # --- Row 8: Data Prepper — Traces Pipeline --- - - id: pipeline-dp-traces-processed - title: "DP Traces Processed/sec" - query: "rate(otel_traces_pipeline_recordsProcessed_total[5m])" - chartType: line - - - id: pipeline-dp-traces-latency - title: "DP Traces Pipeline Latency (avg sec)" - query: "rate(traces_raw_pipeline_opensearch_PipelineLatency_seconds_sum[5m]) / rate(traces_raw_pipeline_opensearch_PipelineLatency_seconds_count[5m])" - chartType: line - - # --- Row 9: Data Prepper — Metrics Pipeline --- - - id: pipeline-dp-metrics-received - title: "DP Metrics Received/sec" - query: "rate(otlp_metrics_requestsReceived_total[5m])" - chartType: line - - - id: pipeline-dp-otlp-requests - title: "DP OTLP Requests Received/sec (all)" - query: "rate(otlp_traces_requestsReceived_total[5m]) + rate(otlp_logs_requestsReceived_total[5m]) + rate(otlp_metrics_requestsReceived_total[5m])" - chartType: line - - # --- Row 10: Data Prepper — Writes & Errors --- - - id: pipeline-dp-logs-docs-written - title: "DP Logs Docs Written/sec" - query: "rate(otel_logs_pipeline_opensearch_documentsSuccess_total[5m])" - chartType: line - - - id: pipeline-dp-traces-docs-written - title: "DP Traces Docs Written/sec" - query: "rate(traces_raw_pipeline_opensearch_documentsSuccess_total[5m])" - chartType: line - - # --- Row 11: Data Prepper — Errors & Buffer --- - - id: pipeline-dp-bulk-errors - title: "DP Bulk Request Errors" - query: "sum(rate(otel_logs_pipeline_opensearch_bulkRequestErrors_total[5m])) + sum(rate(traces_raw_pipeline_opensearch_bulkRequestErrors_total[5m]))" - chartType: line - - - id: pipeline-dp-buffer-usage - title: "DP Buffer Writes/sec" - query: "rate(otel_logs_pipeline_BlockingBuffer_recordsWritten_total[5m]) + rate(otel_traces_pipeline_BlockingBuffer_recordsWritten_total[5m])" - chartType: line - - - id: pipeline-dp-buffer-capacity - title: "DP Buffer Capacity Used %" - query: "otlp_pipeline_BlockingBuffer_capacityUsed + otel_logs_pipeline_BlockingBuffer_capacityUsed + otel_traces_pipeline_BlockingBuffer_capacityUsed" - chartType: line From c56383ebf0cd9d70fd8972c51f7057bbb5e77d46 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Wed, 18 Mar 2026 17:37:57 -0700 Subject: [PATCH 22/64] fix: prometheus ingestion rate filter + showFullTimeRange for dashboard panels --- charts/observability-stack/files/dashboard-pipeline-health.yaml | 2 +- charts/observability-stack/files/init-opensearch-dashboards.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/observability-stack/files/dashboard-pipeline-health.yaml b/charts/observability-stack/files/dashboard-pipeline-health.yaml index 97e350fa..0292f306 100644 --- a/charts/observability-stack/files/dashboard-pipeline-health.yaml +++ b/charts/observability-stack/files/dashboard-pipeline-health.yaml @@ -53,7 +53,7 @@ panels: # --- Row 5: Prometheus Health --- - id: pipeline-prometheus-ingestion title: "Prometheus Ingestion Rate (samples/sec)" - query: "rate(prometheus_tsdb_head_samples_appended_total[5m])" + query: "rate(prometheus_tsdb_head_samples_appended_total{type=\"float\"}[5m])" chartType: line - id: pipeline-prometheus-active-series diff --git a/charts/observability-stack/files/init-opensearch-dashboards.py b/charts/observability-stack/files/init-opensearch-dashboards.py index f7cb9f4b..94faaecd 100644 --- a/charts/observability-stack/files/init-opensearch-dashboards.py +++ b/charts/observability-stack/files/init-opensearch-dashboards.py @@ -969,7 +969,7 @@ def create_promql_dashboard_from_yaml(workspace_id, config_path, prometheus_data "params": { "addLegend": True, "addTimeMarker": False, "legendPosition": "bottom", "legendTitle": "", "lineMode": "straight", "lineStyle": "line", "lineWidth": 2, - "showFullTimeRange": False, "standardAxes": [], + "showFullTimeRange": True, "standardAxes": [], "thresholdOptions": {"baseColor": "#00BD6B", "thresholds": [], "thresholdStyle": "off"}, "titleOptions": {"show": False, "titleName": ""}, "tooltipOptions": {"mode": "all"} From 642d54868bd047556b6b6e87e10538349601a432 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Wed, 18 Mar 2026 18:31:58 -0700 Subject: [PATCH 23/64] sync: copy init script + dashboard YAMLs from docker-compose (source of truth) to helm, preserve K8s dashboard call --- .../files/dashboard-pipeline-health.yaml | 2 +- .../files/init-opensearch-dashboards.py | 124 +++++------------- 2 files changed, 37 insertions(+), 89 deletions(-) diff --git a/charts/observability-stack/files/dashboard-pipeline-health.yaml b/charts/observability-stack/files/dashboard-pipeline-health.yaml index 0292f306..97e350fa 100644 --- a/charts/observability-stack/files/dashboard-pipeline-health.yaml +++ b/charts/observability-stack/files/dashboard-pipeline-health.yaml @@ -53,7 +53,7 @@ panels: # --- Row 5: Prometheus Health --- - id: pipeline-prometheus-ingestion title: "Prometheus Ingestion Rate (samples/sec)" - query: "rate(prometheus_tsdb_head_samples_appended_total{type=\"float\"}[5m])" + query: "rate(prometheus_tsdb_head_samples_appended_total[5m])" chartType: line - id: pipeline-prometheus-active-series diff --git a/charts/observability-stack/files/init-opensearch-dashboards.py b/charts/observability-stack/files/init-opensearch-dashboards.py index 94faaecd..cbdb6f82 100644 --- a/charts/observability-stack/files/init-opensearch-dashboards.py +++ b/charts/observability-stack/files/init-opensearch-dashboards.py @@ -5,11 +5,16 @@ import requests import yaml -BASE_URL = os.getenv("BASE_URL", "http://opensearch-dashboards:5601") +_dashboards_host = os.getenv("OPENSEARCH_DASHBOARDS_HOST", "opensearch-dashboards") +_dashboards_port = os.getenv("OPENSEARCH_DASHBOARDS_PORT", "5601") +_dashboards_protocol = os.getenv("OPENSEARCH_DASHBOARDS_PROTOCOL", "http") +BASE_URL = f"{_dashboards_protocol}://{_dashboards_host}:{_dashboards_port}" USERNAME = os.getenv("OPENSEARCH_USER", "admin") PASSWORD = os.getenv("OPENSEARCH_PASSWORD", "My_password_123!@#") PROMETHEUS_HOST = os.getenv("PROMETHEUS_HOST", "prometheus") PROMETHEUS_PORT = os.getenv("PROMETHEUS_PORT", "9090") +_opensearch_protocol = os.getenv("OPENSEARCH_PROTOCOL", "https") +OPENSEARCH_ENDPOINT = f"{_opensearch_protocol}://{os.getenv('OPENSEARCH_HOST', 'opensearch')}:{os.getenv('OPENSEARCH_PORT', '9200')}" def wait_for_dashboards(): """Wait for OpenSearch Dashboards to be ready""" @@ -18,7 +23,7 @@ def wait_for_dashboards(): while True: try: response = requests.get( - f"{BASE_URL}/api/status", auth=(USERNAME, PASSWORD), timeout=5 + f"{BASE_URL}/api/status", auth=(USERNAME, PASSWORD), timeout=5, verify=False ) if response.status_code == 200: break @@ -376,7 +381,7 @@ def create_opensearch_datasource(workspace_id): print("🔧 Creating OpenSearch datasource...") - opensearch_endpoint = os.environ.get("OPENSEARCH_ENDPOINT", "https://opensearch:9200") + opensearch_endpoint = OPENSEARCH_ENDPOINT payload = { "attributes": { @@ -958,18 +963,16 @@ def create_promql_dashboard_from_yaml(workspace_id, config_path, prometheus_data dashboard_config = config.get("dashboard", {}) panel_defs = config.get("panels", []) - dashboard_id = dashboard_config.get("id", "k8s-cluster-health-dashboard") + dashboard_id = dashboard_config.get("id", "promql-dashboard") - print(f"📊 Creating {dashboard_config.get('title', 'K8s Cluster Health')} dashboard ({len(panel_defs)} panels)...") + print(f"📊 Creating {dashboard_config.get('title', 'PromQL Dashboard')} dashboard ({len(panel_defs)} panels)...") - # Visualization template for explore panels viz_template = json.dumps({ - "title": "", - "chartType": "line", + "title": "", "chartType": "line", "params": { "addLegend": True, "addTimeMarker": False, "legendPosition": "bottom", "legendTitle": "", "lineMode": "straight", "lineStyle": "line", "lineWidth": 2, - "showFullTimeRange": True, "standardAxes": [], + "showFullTimeRange": False, "standardAxes": [], "thresholdOptions": {"baseColor": "#00BD6B", "thresholds": [], "thresholdStyle": "off"}, "titleOptions": {"show": False, "titleName": ""}, "tooltipOptions": {"mode": "all"} @@ -978,73 +981,40 @@ def create_promql_dashboard_from_yaml(workspace_id, config_path, prometheus_data }) dataset = { - "id": prometheus_datasource_title, - "title": prometheus_datasource_title, - "type": "PROMETHEUS", - "language": "PROMQL", - "timeFieldName": "Time", - "dataSource": {}, - "signalType": "metrics" + "id": prometheus_datasource_title, "title": prometheus_datasource_title, + "type": "PROMETHEUS", "language": "PROMQL", "timeFieldName": "Time", + "dataSource": {}, "signalType": "metrics" } created_ids = [] for panel_def in panel_defs: panel_id = panel_def["id"] search_source = json.dumps({ - "query": { - "query": panel_def["query"], - "language": "PROMQL", - "dataset": dataset - }, - "filter": [], - "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index" + "query": {"query": panel_def["query"], "language": "PROMQL", "dataset": dataset}, + "filter": [], "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index" }) - payload = { "attributes": { - "title": panel_def["title"], - "description": "", - "hits": 0, - "columns": ["_source"], - "sort": [], - "version": 1, - "type": "metrics", + "title": panel_def["title"], "description": "", "hits": 0, + "columns": ["_source"], "sort": [], "version": 1, "type": "metrics", "visualization": viz_template, "uiState": json.dumps({"activeTab": "explore_visualization_tab"}), - "kibanaSavedObjectMeta": { - "searchSourceJSON": search_source - } + "kibanaSavedObjectMeta": {"searchSourceJSON": search_source} }, - "references": [{ - "name": "kibanaSavedObjectMeta.searchSourceJSON.index", - "type": "index-pattern", - "id": prometheus_datasource_title - }] + "references": [{"name": "kibanaSavedObjectMeta.searchSourceJSON.index", "type": "index-pattern", "id": prometheus_datasource_title}] } - if workspace_id and workspace_id != "default": payload["workspaces"] = [workspace_id] url = f"{BASE_URL}/w/{workspace_id}/api/saved_objects/explore/{panel_id}" else: url = f"{BASE_URL}/api/saved_objects/explore/{panel_id}" - try: - response = requests.post( - url, auth=(USERNAME, PASSWORD), - headers={"Content-Type": "application/json", "osd-xsrf": "true"}, - json=payload, verify=False, timeout=10, - ) + response = requests.post(url, auth=(USERNAME, PASSWORD), headers={"Content-Type": "application/json", "osd-xsrf": "true"}, json=payload, verify=False, timeout=10) if response.status_code == 200: created_ids.append(panel_id) print(f" ✅ {panel_def['title']}") elif response.status_code == 409: - # Update existing - requests.put( - url, auth=(USERNAME, PASSWORD), - headers={"Content-Type": "application/json", "osd-xsrf": "true"}, - json={"attributes": payload["attributes"], "references": payload["references"]}, - verify=False, timeout=10, - ) + requests.put(url, auth=(USERNAME, PASSWORD), headers={"Content-Type": "application/json", "osd-xsrf": "true"}, json={"attributes": payload["attributes"], "references": payload["references"]}, verify=False, timeout=10) created_ids.append(panel_id) print(f" 🔄 {panel_def['title']} (updated)") else: @@ -1056,52 +1026,33 @@ def create_promql_dashboard_from_yaml(workspace_id, config_path, prometheus_data print("⚠️ No panels created, skipping dashboard") return None - # Assemble dashboard — 2 panels per row, 24 units wide each panels = [] references = [] for i, pid in enumerate(created_ids): - panels.append({ - "version": "3.6.0", - "panelIndex": pid, - "gridData": {"i": pid, "x": (i % 2) * 24, "y": (i // 2) * 15, "w": 24, "h": 15}, - "panelRefName": f"panel_{i}" - }) + panels.append({"version": "3.6.0", "panelIndex": pid, "gridData": {"i": pid, "x": (i % 2) * 24, "y": (i // 2) * 15, "w": 24, "h": 15}, "panelRefName": f"panel_{i}"}) references.append({"name": f"panel_{i}", "type": "explore", "id": pid}) dashboard_payload = { "attributes": { - "title": dashboard_config.get("title", "Kubernetes Cluster Health"), + "title": dashboard_config.get("title", "PromQL Dashboard"), "description": dashboard_config.get("description", ""), "panelsJSON": json.dumps(panels), "optionsJSON": json.dumps({"useMargins": True, "hidePanelTitles": False}), "timeRestore": False, - "kibanaSavedObjectMeta": { - "searchSourceJSON": json.dumps({}) - } + "kibanaSavedObjectMeta": {"searchSourceJSON": json.dumps({})} }, "references": references } - if workspace_id and workspace_id != "default": dashboard_payload["workspaces"] = [workspace_id] url = f"{BASE_URL}/w/{workspace_id}/api/saved_objects/dashboard/{dashboard_id}" else: url = f"{BASE_URL}/api/saved_objects/dashboard/{dashboard_id}" - try: - response = requests.post( - url, auth=(USERNAME, PASSWORD), - headers={"Content-Type": "application/json", "osd-xsrf": "true"}, - json=dashboard_payload, verify=False, timeout=10, - ) + response = requests.post(url, auth=(USERNAME, PASSWORD), headers={"Content-Type": "application/json", "osd-xsrf": "true"}, json=dashboard_payload, verify=False, timeout=10) if response.status_code in (200, 409): if response.status_code == 409: - requests.put( - url, auth=(USERNAME, PASSWORD), - headers={"Content-Type": "application/json", "osd-xsrf": "true"}, - json={"attributes": dashboard_payload["attributes"], "references": references}, - verify=False, timeout=10, - ) + requests.put(url, auth=(USERNAME, PASSWORD), headers={"Content-Type": "application/json", "osd-xsrf": "true"}, json={"attributes": dashboard_payload["attributes"], "references": references}, verify=False, timeout=10) print(f"✅ Created {dashboard_config['title']} dashboard ({len(created_ids)} panels)") return dashboard_id else: @@ -1121,11 +1072,12 @@ def create_overview_dashboard(workspace_id): dashboard_id = "observability-overview-dashboard" # Check if dashboard already exists - existing = get_existing_dashboard(workspace_id, dashboard_id) - if existing: - print("🔄 Overview dashboard exists, updating...") - else: - print("📊 Creating Observability Stack overview dashboard...") + if get_existing_dashboard(workspace_id, dashboard_id): + print("✅ Overview dashboard already exists") + set_default_dashboard(workspace_id, dashboard_id) + return dashboard_id + + print("📊 Creating Observability Stack overview dashboard...") # Load architecture image as base64 data URI arch_img_tag = "" @@ -1216,7 +1168,7 @@ def create_overview_dashboard(workspace_id): try: response = requests.post( - vis_url + "?overwrite=true", + vis_url, auth=(USERNAME, PASSWORD), headers={"Content-Type": "application/json", "osd-xsrf": "true"}, json=vis_payload, @@ -1333,13 +1285,9 @@ def main(): # Create overview landing dashboard (becomes the new default) create_overview_dashboard(workspace_id) - # Create K8s cluster health dashboard (PromQL explore panels) + # Create self-monitoring dashboards (PromQL explore panels) create_promql_dashboard_from_yaml(workspace_id, "/config/dashboard-k8s-cluster-health.yaml") - - # Create observability pipeline health dashboard create_promql_dashboard_from_yaml(workspace_id, "/config/dashboard-pipeline-health.yaml") - - # Create OpenSearch cluster health dashboard create_promql_dashboard_from_yaml(workspace_id, "/config/dashboard-opensearch-health.yaml") # Create saved queries for common agent observability patterns From 1aef13d63b82dd0161bc1761a668fd7876327703 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Wed, 18 Mar 2026 19:29:15 -0700 Subject: [PATCH 24/64] fix: respect BASE_URL env var in init script for helm compatibility --- charts/observability-stack/files/init-opensearch-dashboards.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/observability-stack/files/init-opensearch-dashboards.py b/charts/observability-stack/files/init-opensearch-dashboards.py index cbdb6f82..6f32b98d 100644 --- a/charts/observability-stack/files/init-opensearch-dashboards.py +++ b/charts/observability-stack/files/init-opensearch-dashboards.py @@ -8,7 +8,7 @@ _dashboards_host = os.getenv("OPENSEARCH_DASHBOARDS_HOST", "opensearch-dashboards") _dashboards_port = os.getenv("OPENSEARCH_DASHBOARDS_PORT", "5601") _dashboards_protocol = os.getenv("OPENSEARCH_DASHBOARDS_PROTOCOL", "http") -BASE_URL = f"{_dashboards_protocol}://{_dashboards_host}:{_dashboards_port}" +BASE_URL = os.getenv("BASE_URL", f"{_dashboards_protocol}://{_dashboards_host}:{_dashboards_port}") USERNAME = os.getenv("OPENSEARCH_USER", "admin") PASSWORD = os.getenv("OPENSEARCH_PASSWORD", "My_password_123!@#") PROMETHEUS_HOST = os.getenv("PROMETHEUS_HOST", "prometheus") From 1b8eda70138e99d5c86fadd00d88a52d746d50bb Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Wed, 18 Mar 2026 21:05:38 -0700 Subject: [PATCH 25/64] sync: update pipeline dashboard YAML from upstream PR #107 (adds Data Prepper panels, fixes prometheus panels) --- .../files/dashboard-pipeline-health.yaml | 76 ++++++++++++++----- .../dashboard-pipeline-health.yaml | 76 ++++++++++++++----- 2 files changed, 116 insertions(+), 36 deletions(-) diff --git a/charts/observability-stack/files/dashboard-pipeline-health.yaml b/charts/observability-stack/files/dashboard-pipeline-health.yaml index 97e350fa..1fdd2ea3 100644 --- a/charts/observability-stack/files/dashboard-pipeline-health.yaml +++ b/charts/observability-stack/files/dashboard-pipeline-health.yaml @@ -1,73 +1,113 @@ # Observability Pipeline Health Dashboard — OTel Collector and Prometheus self-monitoring - dashboard: id: observability-pipeline-health-dashboard title: Observability Pipeline Health description: OTel Collector throughput, Prometheus ingestion, and pipeline health - panels: # --- Row 1: OTel Collector Throughput --- - id: pipeline-otel-spans-received title: "OTel Spans Received/sec" query: "rate(otelcol_receiver_accepted_spans_total[5m])" chartType: line - - id: pipeline-otel-spans-exported title: "OTel Spans Exported/sec" query: "rate(otelcol_exporter_sent_spans_total[5m])" chartType: line - # --- Row 2: OTel Metrics & Failures --- - id: pipeline-otel-metrics-received title: "OTel Metrics Received/sec" query: "rate(otelcol_receiver_accepted_metric_points_total[5m])" chartType: line - - id: pipeline-otel-spans-dropped title: "OTel Spans Dropped/sec" query: "rate(otelcol_exporter_send_failed_spans_total[5m])" chartType: line - # --- Row 3: OTel Collector Resources --- - id: pipeline-otel-queue-size title: "OTel Exporter Queue Size" query: "otelcol_exporter_queue_size" chartType: line - - id: pipeline-otel-collector-memory title: "OTel Collector Memory (bytes)" query: "otelcol_process_memory_rss_bytes" chartType: line - # --- Row 4: OTel Collector CPU & Uptime --- - id: pipeline-otel-collector-cpu title: "OTel Collector CPU Usage" query: "rate(otelcol_process_cpu_seconds_total[5m])" chartType: line - - id: pipeline-otel-batch-cardinality title: "OTel Batch Metadata Cardinality" query: "otelcol_processor_batch_metadata_cardinality" chartType: line - # --- Row 5: Prometheus Health --- - id: pipeline-prometheus-ingestion - title: "Prometheus Ingestion Rate (samples/sec)" - query: "rate(prometheus_tsdb_head_samples_appended_total[5m])" + title: "Prometheus Ingestion Rate (chunks/sec)" + query: "rate(prometheus_tsdb_head_chunks_created_total[5m])" chartType: line - - id: pipeline-prometheus-active-series title: "Prometheus Active Time Series" query: "prometheus_tsdb_head_series" chartType: line - # --- Row 6: Prometheus Storage --- - - id: pipeline-prometheus-storage - title: "Prometheus Storage Size (bytes)" - query: "prometheus_tsdb_storage_blocks_bytes" + - id: pipeline-prometheus-wal-size + title: "Prometheus WAL Size (bytes)" + query: "prometheus_tsdb_wal_storage_size_bytes" + chartType: line + - id: pipeline-prometheus-head-chunks + title: "Prometheus Head Chunks Size (bytes)" + query: "prometheus_tsdb_head_chunks_storage_size_bytes" chartType: line - - id: pipeline-prometheus-query-latency title: "Prometheus Query Latency P99 (sec)" query: "histogram_quantile(0.99, rate(prometheus_http_request_duration_seconds_bucket{handler=\"/api/v1/query\"}[5m]))" chartType: line + # --- Row 7: Data Prepper — Logs Pipeline --- + - id: pipeline-dp-logs-processed + title: "DP Logs Processed/sec" + query: "rate(otel_logs_pipeline_recordsProcessed_total[5m])" + chartType: line + - id: pipeline-dp-logs-latency + title: "DP Logs Pipeline Latency (avg sec)" + query: "rate(otel_logs_pipeline_opensearch_PipelineLatency_seconds_sum[5m]) / rate(otel_logs_pipeline_opensearch_PipelineLatency_seconds_count[5m])" + chartType: line + # --- Row 8: Data Prepper — Traces Pipeline --- + - id: pipeline-dp-traces-processed + title: "DP Traces Processed/sec" + query: "rate(otel_traces_pipeline_recordsProcessed_total[5m])" + chartType: line + - id: pipeline-dp-traces-latency + title: "DP Traces Pipeline Latency (avg sec)" + query: "rate(traces_raw_pipeline_opensearch_PipelineLatency_seconds_sum[5m]) / rate(traces_raw_pipeline_opensearch_PipelineLatency_seconds_count[5m])" + chartType: line + # --- Row 9: Data Prepper — Metrics Pipeline --- + - id: pipeline-dp-metrics-received + title: "DP Metrics Received/sec" + query: "rate(otlp_metrics_requestsReceived_total[5m])" + chartType: line + - id: pipeline-dp-otlp-requests + title: "DP OTLP Requests Received/sec (all)" + query: "rate(otlp_traces_requestsReceived_total[5m]) + rate(otlp_logs_requestsReceived_total[5m]) + rate(otlp_metrics_requestsReceived_total[5m])" + chartType: line + # --- Row 10: Data Prepper — Writes & Errors --- + - id: pipeline-dp-logs-docs-written + title: "DP Logs Docs Written/sec" + query: "rate(otel_logs_pipeline_opensearch_documentsSuccess_total[5m])" + chartType: line + - id: pipeline-dp-traces-docs-written + title: "DP Traces Docs Written/sec" + query: "rate(traces_raw_pipeline_opensearch_documentsSuccess_total[5m])" + chartType: line + # --- Row 11: Data Prepper — Errors & Buffer --- + - id: pipeline-dp-bulk-errors + title: "DP Bulk Request Errors" + query: "sum(rate(otel_logs_pipeline_opensearch_bulkRequestErrors_total[5m])) + sum(rate(traces_raw_pipeline_opensearch_bulkRequestErrors_total[5m]))" + chartType: line + - id: pipeline-dp-buffer-usage + title: "DP Buffer Writes/sec" + query: "rate(otel_logs_pipeline_BlockingBuffer_recordsWritten_total[5m]) + rate(otel_traces_pipeline_BlockingBuffer_recordsWritten_total[5m])" + chartType: line + - id: pipeline-dp-buffer-capacity + title: "DP Buffer Capacity Used %" + query: "otlp_pipeline_BlockingBuffer_capacityUsed + otel_logs_pipeline_BlockingBuffer_capacityUsed + otel_traces_pipeline_BlockingBuffer_capacityUsed" + chartType: line diff --git a/docker-compose/opensearch-dashboards/dashboard-pipeline-health.yaml b/docker-compose/opensearch-dashboards/dashboard-pipeline-health.yaml index 97e350fa..1fdd2ea3 100644 --- a/docker-compose/opensearch-dashboards/dashboard-pipeline-health.yaml +++ b/docker-compose/opensearch-dashboards/dashboard-pipeline-health.yaml @@ -1,73 +1,113 @@ # Observability Pipeline Health Dashboard — OTel Collector and Prometheus self-monitoring - dashboard: id: observability-pipeline-health-dashboard title: Observability Pipeline Health description: OTel Collector throughput, Prometheus ingestion, and pipeline health - panels: # --- Row 1: OTel Collector Throughput --- - id: pipeline-otel-spans-received title: "OTel Spans Received/sec" query: "rate(otelcol_receiver_accepted_spans_total[5m])" chartType: line - - id: pipeline-otel-spans-exported title: "OTel Spans Exported/sec" query: "rate(otelcol_exporter_sent_spans_total[5m])" chartType: line - # --- Row 2: OTel Metrics & Failures --- - id: pipeline-otel-metrics-received title: "OTel Metrics Received/sec" query: "rate(otelcol_receiver_accepted_metric_points_total[5m])" chartType: line - - id: pipeline-otel-spans-dropped title: "OTel Spans Dropped/sec" query: "rate(otelcol_exporter_send_failed_spans_total[5m])" chartType: line - # --- Row 3: OTel Collector Resources --- - id: pipeline-otel-queue-size title: "OTel Exporter Queue Size" query: "otelcol_exporter_queue_size" chartType: line - - id: pipeline-otel-collector-memory title: "OTel Collector Memory (bytes)" query: "otelcol_process_memory_rss_bytes" chartType: line - # --- Row 4: OTel Collector CPU & Uptime --- - id: pipeline-otel-collector-cpu title: "OTel Collector CPU Usage" query: "rate(otelcol_process_cpu_seconds_total[5m])" chartType: line - - id: pipeline-otel-batch-cardinality title: "OTel Batch Metadata Cardinality" query: "otelcol_processor_batch_metadata_cardinality" chartType: line - # --- Row 5: Prometheus Health --- - id: pipeline-prometheus-ingestion - title: "Prometheus Ingestion Rate (samples/sec)" - query: "rate(prometheus_tsdb_head_samples_appended_total[5m])" + title: "Prometheus Ingestion Rate (chunks/sec)" + query: "rate(prometheus_tsdb_head_chunks_created_total[5m])" chartType: line - - id: pipeline-prometheus-active-series title: "Prometheus Active Time Series" query: "prometheus_tsdb_head_series" chartType: line - # --- Row 6: Prometheus Storage --- - - id: pipeline-prometheus-storage - title: "Prometheus Storage Size (bytes)" - query: "prometheus_tsdb_storage_blocks_bytes" + - id: pipeline-prometheus-wal-size + title: "Prometheus WAL Size (bytes)" + query: "prometheus_tsdb_wal_storage_size_bytes" + chartType: line + - id: pipeline-prometheus-head-chunks + title: "Prometheus Head Chunks Size (bytes)" + query: "prometheus_tsdb_head_chunks_storage_size_bytes" chartType: line - - id: pipeline-prometheus-query-latency title: "Prometheus Query Latency P99 (sec)" query: "histogram_quantile(0.99, rate(prometheus_http_request_duration_seconds_bucket{handler=\"/api/v1/query\"}[5m]))" chartType: line + # --- Row 7: Data Prepper — Logs Pipeline --- + - id: pipeline-dp-logs-processed + title: "DP Logs Processed/sec" + query: "rate(otel_logs_pipeline_recordsProcessed_total[5m])" + chartType: line + - id: pipeline-dp-logs-latency + title: "DP Logs Pipeline Latency (avg sec)" + query: "rate(otel_logs_pipeline_opensearch_PipelineLatency_seconds_sum[5m]) / rate(otel_logs_pipeline_opensearch_PipelineLatency_seconds_count[5m])" + chartType: line + # --- Row 8: Data Prepper — Traces Pipeline --- + - id: pipeline-dp-traces-processed + title: "DP Traces Processed/sec" + query: "rate(otel_traces_pipeline_recordsProcessed_total[5m])" + chartType: line + - id: pipeline-dp-traces-latency + title: "DP Traces Pipeline Latency (avg sec)" + query: "rate(traces_raw_pipeline_opensearch_PipelineLatency_seconds_sum[5m]) / rate(traces_raw_pipeline_opensearch_PipelineLatency_seconds_count[5m])" + chartType: line + # --- Row 9: Data Prepper — Metrics Pipeline --- + - id: pipeline-dp-metrics-received + title: "DP Metrics Received/sec" + query: "rate(otlp_metrics_requestsReceived_total[5m])" + chartType: line + - id: pipeline-dp-otlp-requests + title: "DP OTLP Requests Received/sec (all)" + query: "rate(otlp_traces_requestsReceived_total[5m]) + rate(otlp_logs_requestsReceived_total[5m]) + rate(otlp_metrics_requestsReceived_total[5m])" + chartType: line + # --- Row 10: Data Prepper — Writes & Errors --- + - id: pipeline-dp-logs-docs-written + title: "DP Logs Docs Written/sec" + query: "rate(otel_logs_pipeline_opensearch_documentsSuccess_total[5m])" + chartType: line + - id: pipeline-dp-traces-docs-written + title: "DP Traces Docs Written/sec" + query: "rate(traces_raw_pipeline_opensearch_documentsSuccess_total[5m])" + chartType: line + # --- Row 11: Data Prepper — Errors & Buffer --- + - id: pipeline-dp-bulk-errors + title: "DP Bulk Request Errors" + query: "sum(rate(otel_logs_pipeline_opensearch_bulkRequestErrors_total[5m])) + sum(rate(traces_raw_pipeline_opensearch_bulkRequestErrors_total[5m]))" + chartType: line + - id: pipeline-dp-buffer-usage + title: "DP Buffer Writes/sec" + query: "rate(otel_logs_pipeline_BlockingBuffer_recordsWritten_total[5m]) + rate(otel_traces_pipeline_BlockingBuffer_recordsWritten_total[5m])" + chartType: line + - id: pipeline-dp-buffer-capacity + title: "DP Buffer Capacity Used %" + query: "otlp_pipeline_BlockingBuffer_capacityUsed + otel_logs_pipeline_BlockingBuffer_capacityUsed + otel_traces_pipeline_BlockingBuffer_capacityUsed" + chartType: line From f2e758ef35ab588b50b5050534626d5e6449a6a0 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Wed, 18 Mar 2026 21:35:38 -0700 Subject: [PATCH 26/64] sync: copy latest AGENTS.md, docker-compose.yml, .gitignore from main --- .gitignore | 5 +++-- AGENTS.md | 30 ++++++++++++++++++++++++++++++ docker-compose.yml | 1 + 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 24473f48..a449f2a5 100644 --- a/.gitignore +++ b/.gitignore @@ -9,5 +9,6 @@ node_modules dist build .DS_Store -charts/observability-stack/charts/ -charts/observability-stack/Chart.lock + +# Agent worktrees +.worktrees/ diff --git a/AGENTS.md b/AGENTS.md index 9c15f30d..4cf9aa28 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -905,6 +905,36 @@ When creating examples or documentation, always reference the OpenTelemetry Gen- 6. **Commit**: Use descriptive commit messages 7. **Submit PR**: Follow CONTRIBUTING.md guidelines + +## Multi-Agent Development with Worktrees + +When multiple agents or sessions work on this repo simultaneously, each feature branch gets its own worktree for isolation. + +### Structure + +``` +observability-stack/ +├── .worktrees/ # gitignored — one per feature branch +│ ├── feat-self-monitoring/ +│ ├── feat-helm-charts/ +│ └── fix-docs/ +└── ... # main branch +``` + +### Usage + +```bash +# Create +mkdir -p .worktrees +git worktree add .worktrees/ + +# REQUIRED: Clean up after PR merge +git worktree remove .worktrees/ +git branch -d +``` + +**You MUST remove worktrees after their PR is merged.** Stale worktrees waste disk space and cause confusion about what work is active. + ## Common Pitfalls to Avoid - ❌ Using `latest` image tags (use specific versions) diff --git a/docker-compose.yml b/docker-compose.yml index 23a268d3..cb26d289 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,7 @@ # Observability Stack # Docker Compose configuration for local development # WARNING: This configuration is for development/testing only - not production-ready +name: observability-stack include: - path: ${INCLUDE_COMPOSE_EXAMPLES:-docker-compose/util/docker-compose.empty.yml} From 0736ad78216176ee05d73a7ef527a8b9070adb0d Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Wed, 18 Mar 2026 21:36:39 -0700 Subject: [PATCH 27/64] feat: add Data Prepper Prometheus scrape config to helm chart --- charts/observability-stack/values.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/charts/observability-stack/values.yaml b/charts/observability-stack/values.yaml index c502395e..75898d78 100644 --- a/charts/observability-stack/values.yaml +++ b/charts/observability-stack/values.yaml @@ -304,6 +304,11 @@ prometheus: static_configs: - targets: ['obs-stack-observability-stack-opensearch-exporter:9114'] scrape_interval: 30s + data-prepper: + metrics_path: '/metrics/prometheus' + static_configs: + - targets: ['obs-stack-data-prepper:4900'] + scrape_interval: 30s node-exporter: kubernetes_sd_configs: - role: endpoints From 840f5a4fe26eed7868383e8709960b7fed3a41b2 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Wed, 18 Mar 2026 21:41:02 -0700 Subject: [PATCH 28/64] fix: expose Data Prepper metrics port 4900 on service for Prometheus scraping --- charts/observability-stack/values.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/charts/observability-stack/values.yaml b/charts/observability-stack/values.yaml index 75898d78..7b2939e1 100644 --- a/charts/observability-stack/values.yaml +++ b/charts/observability-stack/values.yaml @@ -64,6 +64,17 @@ data-prepper: image: repository: "sgguruda62324/opensearch-data-prepper" tag: "2.15.0-SNAPSHOT" + ports: + - name: http-source + port: 2021 + - name: otel-traces + port: 21890 + - name: otel-metrics + port: 21891 + - name: otel-logs + port: 21892 + - name: metrics + port: 4900 config: data-prepper-config.yaml: | ssl: false From bf71657ac9b315420e13c24a5bc23c678814685f Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Wed, 18 Mar 2026 22:06:16 -0700 Subject: [PATCH 29/64] fix: force ssl-redirect annotation as string type to avoid helm decode error --- terraform/aws/observability-stack.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/aws/observability-stack.tf b/terraform/aws/observability-stack.tf index 2b48609d..cc4e20fc 100644 --- a/terraform/aws/observability-stack.tf +++ b/terraform/aws/observability-stack.tf @@ -143,6 +143,7 @@ resource "helm_release" "observability_stack" { content { name = "opensearch-dashboards.ingress.annotations.alb\\.ingress\\.kubernetes\\.io/ssl-redirect" value = "443" + type = "string" } } dynamic "set" { From d429a4329707f500c9b2902b64fe88bd9522da64 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Wed, 18 Mar 2026 22:26:29 -0700 Subject: [PATCH 30/64] docs: add helm chart README and values-eks comments --- charts/observability-stack/README.md | 125 +++++++++++++++++++++++++++ terraform/aws/values-eks.yaml | 21 ++++- 2 files changed, 142 insertions(+), 4 deletions(-) create mode 100644 charts/observability-stack/README.md diff --git a/charts/observability-stack/README.md b/charts/observability-stack/README.md new file mode 100644 index 00000000..8bf4d57d --- /dev/null +++ b/charts/observability-stack/README.md @@ -0,0 +1,125 @@ +# Observability Stack Helm Chart + +Umbrella Helm chart that deploys the full observability stack to Kubernetes. Wraps community subcharts (OpenSearch, Prometheus, OTel Collector, Data Prepper) with opinionated defaults and adds self-monitoring dashboards. + +## Components + +| Subchart | Source | Purpose | +|----------|--------|---------| +| `opensearch` | opensearch-project/helm-charts | Log and trace storage | +| `opensearch-dashboards` | opensearch-project/helm-charts | Web UI | +| `data-prepper` | opensearch-project/helm-charts | OTLP → OpenSearch pipeline | +| `opentelemetry-collector` | open-telemetry/helm-charts | Telemetry receiver and router | +| `prometheus` | prometheus-community/helm-charts | Metrics storage (OTLP + scrape) | + +Additional templates (not subcharts): +- `opensearch-exporter` — Prometheus exporter for OpenSearch cluster metrics +- `init-dashboards-job` — Post-install hook that creates index patterns, dashboards, saved queries +- `opensearch-credentials-secret` — Shared credentials secret + +## Install + +```bash +cd charts/observability-stack +helm dependency build +helm install obs-stack . -n observability-stack --create-namespace +``` + +For EKS with ALB ingress, use the values override: +```bash +helm install obs-stack . -n observability-stack --create-namespace -f ../../terraform/aws/values-eks.yaml +``` + +Or use Terraform (recommended) — see `terraform/aws/README.md`. + +## Upgrading + +The init job (dashboard/index pattern setup) runs as a post-install/post-upgrade hook. It installs pip packages and takes 3-5 minutes, which often exceeds helm's default timeout. + +**Recommended upgrade workflow:** +```bash +# 1. Deploy chart changes (skip hooks to avoid timeout) +helm upgrade obs-stack . -n observability-stack -f ../../terraform/aws/values-eks.yaml --no-hooks + +# 2. If dashboard or init script changed, trigger the job manually: +kubectl delete job obs-stack-observability-stack-init-dashboards -n observability-stack 2>/dev/null +helm get hooks obs-stack -n observability-stack | kubectl apply -n observability-stack -f - +kubectl wait --for=condition=complete job/obs-stack-observability-stack-init-dashboards -n observability-stack --timeout=10m +kubectl logs -n observability-stack job/obs-stack-observability-stack-init-dashboards --tail=30 +``` + +If only `values.yaml` scrape configs changed (no dashboard changes), step 2 is not needed — but you may need to restart Prometheus to pick up the new configmap: +```bash +kubectl rollout restart deployment obs-stack-prometheus-server -n observability-stack +``` + +## Self-Monitoring Dashboards + +Three dashboards are auto-created by the init job from YAML config files in `files/`: + +| Dashboard | Panels | File | +|-----------|--------|------| +| Kubernetes Cluster Health | 8 | `files/dashboard-k8s-cluster-health.yaml` | +| Observability Pipeline Health | 24 | `files/dashboard-pipeline-health.yaml` | +| OpenSearch Cluster Health | 10 | `files/dashboard-opensearch-health.yaml` | + +**Adding a new dashboard:** +1. Create `files/dashboard-my-thing.yaml` (see existing files for format) +2. Add it to `templates/init-dashboards-configmap.yaml` +3. Add one line to `main()` in `files/init-opensearch-dashboards.py`: + ```python + create_promql_dashboard_from_yaml(workspace_id, "/config/dashboard-my-thing.yaml") + ``` + +**Dashboard YAML format:** +```yaml +dashboard: + id: my-dashboard-id + title: My Dashboard + description: What this monitors + +panels: + - id: panel-unique-id + title: "Panel Title" + query: "rate(some_metric_total[5m])" + chartType: line +``` + +**Syncing with docker-compose:** The docker-compose init script and dashboard YAMLs (`docker-compose/opensearch-dashboards/`) are the source of truth. The helm versions in `files/` should be kept in sync. The only helm-specific addition is the K8s Cluster Health dashboard (not applicable to docker-compose) and the `BASE_URL` env var override in the init script (line 11). + +## Prometheus Scrape Targets + +Configured via `scrapeConfigs` in `values.yaml`. Default K8s scrape jobs are disabled (saves ~60k series). Active targets: + +| Job | Target | Interval | +|-----|--------|----------| +| `prometheus` | localhost:9090 | 60s | +| `otel-collector` | ``-opentelemetry-collector:8888 | 10s | +| `opensearch` | ``-observability-stack-opensearch-exporter:9114 | 30s | +| `data-prepper` | ``-data-prepper:4900 | 30s | +| `node-exporter` | auto-discovered via kubernetes_sd | 60s | +| `kube-state-metrics` | auto-discovered via kubernetes_sd | 60s | + +> **Note:** Targets use the helm release name as prefix. The values in `values.yaml` are hardcoded to `obs-stack-*` — update them if you change the release name. + +## Key Values + +See `values.yaml` for all options. Notable settings: + +```yaml +# Credentials (update opensearchPassword before any real deployment) +opensearchUsername: "admin" +opensearchPassword: "My_password_123!@#" + +# Data Prepper metrics port (must be in ports list for Prometheus to scrape) +data-prepper: + ports: + - name: metrics + port: 4900 + +# Disable noisy K8s scrape defaults +prometheus: + scrapeConfigs: + kubernetes-api-servers: { enabled: false } + # ... etc +``` diff --git a/terraform/aws/values-eks.yaml b/terraform/aws/values-eks.yaml index 50066d15..d594fe25 100644 --- a/terraform/aws/values-eks.yaml +++ b/terraform/aws/values-eks.yaml @@ -1,3 +1,12 @@ +# EKS-specific overrides for the observability-stack helm chart. +# Used by: helm upgrade obs-stack ... -f values-eks.yaml +# Used by: terraform/aws/observability-stack.tf (via values = [file(...)]) +# +# TLS annotations (certificate-arn, listen-ports, ssl-redirect, hostname) +# are injected conditionally by Terraform via dynamic set blocks — not here. + +# gp2 is the default EBS StorageClass on EKS. Without this, OpenSearch PVCs +# stay Pending because the chart default ("") doesn't match any provisioner. opensearch: persistence: storageClass: gp2 @@ -9,15 +18,19 @@ opensearch-dashboards: annotations: alb.ingress.kubernetes.io/scheme: internet-facing alb.ingress.kubernetes.io/target-type: ip - alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80}]' + alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80}]' # Terraform overrides to HTTPS:443 when domain is set hosts: - - paths: + - host: "" + paths: - path: / backend: servicePort: 5601 -gateway: +# Examples disabled — images use imagePullPolicy: Never which doesn't work on EKS. +# Enable after pushing images to a registry (ECR/GHCR). +examples: enabled: false -examples: +# Gateway not needed — ALB handles ingress directly. +gateway: enabled: false From c25a193cafb399ac87a75a7a8bf5b95439de7567 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Wed, 18 Mar 2026 22:28:14 -0700 Subject: [PATCH 31/64] sync: update .gitignore and AGENTS.md from main --- .gitignore | 7 +++++++ AGENTS.md | 2 ++ 2 files changed, 9 insertions(+) diff --git a/.gitignore b/.gitignore index a449f2a5..ab734f73 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,10 @@ build # Agent worktrees .worktrees/ + +# Terraform +*.tfstate +*.tfstate.backup +.terraform/ +.terraform.lock.hcl +.terraform.tfstate.lock.info diff --git a/AGENTS.md b/AGENTS.md index 4cf9aa28..eec07c4a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -935,6 +935,8 @@ git branch -d **You MUST remove worktrees after their PR is merged.** Stale worktrees waste disk space and cause confusion about what work is active. +**Terraform limitation:** Terraform state is local and lives in the main repo's `terraform/aws/` directory. It is NOT shared across worktrees. Only run `terraform plan/apply` from the main repo, never from a worktree. + ## Common Pitfalls to Avoid - ❌ Using `latest` image tags (use specific versions) From 08d337aa0bd91122129b3c741fc9b743e0dd5d47 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Wed, 18 Mar 2026 22:28:52 -0700 Subject: [PATCH 32/64] chore: gitignore helm dependency artifacts and terraform plan files --- .gitignore | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index ab734f73..ee10f6a9 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,10 @@ build .terraform/ .terraform.lock.hcl .terraform.tfstate.lock.info + +# Helm dependency artifacts (regenerated by `helm dependency build`) +charts/*/Chart.lock +charts/*/charts/*.tgz + +# Terraform plan files +*.plan From 364c5fa186af10225e25afa1effaa18799e9b92d Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Wed, 18 Mar 2026 22:34:38 -0700 Subject: [PATCH 33/64] ci: add GHCR image publish workflow (fork-only, skips on upstream) --- .github/workflows/publish-images.yml | 44 ++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/publish-images.yml diff --git a/.github/workflows/publish-images.yml b/.github/workflows/publish-images.yml new file mode 100644 index 00000000..5ae79b6d --- /dev/null +++ b/.github/workflows/publish-images.yml @@ -0,0 +1,44 @@ +name: Build and Push Example Images + +on: + push: + branches: [main] + paths: + - 'examples/**' + - 'docker-compose/canary/**' + - '.github/workflows/publish-images.yml' + workflow_dispatch: + +# Only runs on the fork — skips silently on opensearch-project/observability-stack +jobs: + build: + if: github.repository == 'kylehounslow/observability-stack' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + matrix: + include: + - name: weather-agent + context: examples/plain-agents/weather-agent + - name: travel-planner + context: examples/plain-agents/multi-agent-planner/orchestrator + - name: events-agent + context: examples/plain-agents/multi-agent-planner/events-agent + - name: mcp-server + context: examples/plain-agents/multi-agent-planner/mcp-server + - name: canary + context: docker-compose/canary + steps: + - uses: actions/checkout@v4 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: docker/build-push-action@v6 + with: + context: ${{ matrix.context }} + push: true + tags: ghcr.io/${{ github.repository }}/${{ matrix.name }}:latest From 34d20aad6d7d60bb94598347ca42ed57e4d36a09 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Wed, 18 Mar 2026 22:48:04 -0700 Subject: [PATCH 34/64] feat: enable example agents on EKS with GHCR images - Update image refs to ghcr.io/kylehounslow/observability-stack/* - Make imagePullPolicy configurable (default: IfNotPresent) - Enable examples in values-eks.yaml Verified on EKS: all 5 pods running, 69 trace spans in OpenSearch, OTel Collector receiving 0.2 spans/sec, Data Prepper writing to OS. --- charts/observability-stack/templates/examples.yaml | 10 +++++----- charts/observability-stack/values.yaml | 11 ++++++----- terraform/aws/values-eks.yaml | 5 ++--- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/charts/observability-stack/templates/examples.yaml b/charts/observability-stack/templates/examples.yaml index a3ab9c08..69c4e207 100644 --- a/charts/observability-stack/templates/examples.yaml +++ b/charts/observability-stack/templates/examples.yaml @@ -19,7 +19,7 @@ spec: containers: - name: mcp-server image: {{ .Values.examples.mcpServer.image }} - imagePullPolicy: Never + imagePullPolicy: {{ .Values.examples.imagePullPolicy | default "IfNotPresent" }} ports: - containerPort: 8003 env: @@ -60,7 +60,7 @@ spec: containers: - name: weather-agent image: {{ .Values.examples.weatherAgent.image }} - imagePullPolicy: Never + imagePullPolicy: {{ .Values.examples.imagePullPolicy | default "IfNotPresent" }} ports: - containerPort: 8000 env: @@ -103,7 +103,7 @@ spec: containers: - name: events-agent image: {{ .Values.examples.eventsAgent.image }} - imagePullPolicy: Never + imagePullPolicy: {{ .Values.examples.imagePullPolicy | default "IfNotPresent" }} ports: - containerPort: 8002 env: @@ -146,7 +146,7 @@ spec: containers: - name: travel-planner image: {{ .Values.examples.travelPlanner.image }} - imagePullPolicy: Never + imagePullPolicy: {{ .Values.examples.imagePullPolicy | default "IfNotPresent" }} ports: - containerPort: 8000 env: @@ -193,7 +193,7 @@ spec: containers: - name: canary image: {{ .Values.examples.canary.image }} - imagePullPolicy: Never + imagePullPolicy: {{ .Values.examples.imagePullPolicy | default "IfNotPresent" }} env: - name: TRAVEL_PLANNER_URL value: "http://travel-planner:8000" diff --git a/charts/observability-stack/values.yaml b/charts/observability-stack/values.yaml index 7b2939e1..e593f3a8 100644 --- a/charts/observability-stack/values.yaml +++ b/charts/observability-stack/values.yaml @@ -383,16 +383,17 @@ opensearchExporter: # -- Example agents (matches docker-compose.examples.yml) examples: enabled: true + imagePullPolicy: IfNotPresent weatherAgent: - image: "example-weather-agent:latest" + image: "ghcr.io/kylehounslow/observability-stack/weather-agent:latest" eventsAgent: - image: "example-events-agent:latest" + image: "ghcr.io/kylehounslow/observability-stack/events-agent:latest" travelPlanner: - image: "example-travel-planner:latest" + image: "ghcr.io/kylehounslow/observability-stack/travel-planner:latest" mcpServer: - image: "example-mcp-server:latest" + image: "ghcr.io/kylehounslow/observability-stack/mcp-server:latest" canary: - image: "example-canary:latest" + image: "ghcr.io/kylehounslow/observability-stack/canary:latest" interval: "120" # -- Gateway API for OpenSearch Dashboards diff --git a/terraform/aws/values-eks.yaml b/terraform/aws/values-eks.yaml index d594fe25..4febb834 100644 --- a/terraform/aws/values-eks.yaml +++ b/terraform/aws/values-eks.yaml @@ -26,10 +26,9 @@ opensearch-dashboards: backend: servicePort: 5601 -# Examples disabled — images use imagePullPolicy: Never which doesn't work on EKS. -# Enable after pushing images to a registry (ECR/GHCR). +# Examples — pulls from GHCR (public images built by .github/workflows/publish-images.yml) examples: - enabled: false + enabled: true # Gateway not needed — ALB handles ingress directly. gateway: From 18b7e569292c402dbbd62f1e93bfcbcbbd23f06e Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Wed, 18 Mar 2026 23:04:58 -0700 Subject: [PATCH 35/64] feat: add OpenTelemetry Demo as optional subchart (disabled by default) Adds opentelemetry-demo chart as a dependency, disabled by default. When enabled (opentelemetry-demo.enabled=true), deploys the full OTel Demo microservices app pointing at our OTel Collector. All bundled backends (jaeger, grafana, prometheus, opensearch) are disabled. Verified on EKS: 27 demo services running, 8311 spans from 20+ services (frontend, checkout, cart, payment, etc.) flowing through our pipeline to OpenSearch. --- charts/observability-stack/Chart.yaml | 5 +++++ charts/observability-stack/values.yaml | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/charts/observability-stack/Chart.yaml b/charts/observability-stack/Chart.yaml index 4be4cf42..aab8b244 100644 --- a/charts/observability-stack/Chart.yaml +++ b/charts/observability-stack/Chart.yaml @@ -45,3 +45,8 @@ dependencies: version: "28.13.0" repository: "https://prometheus-community.github.io/helm-charts" condition: prometheus.enabled + + - name: opentelemetry-demo + version: "0.40.5" + repository: "https://open-telemetry.github.io/opentelemetry-helm-charts" + condition: opentelemetry-demo.enabled diff --git a/charts/observability-stack/values.yaml b/charts/observability-stack/values.yaml index e593f3a8..1955294f 100644 --- a/charts/observability-stack/values.yaml +++ b/charts/observability-stack/values.yaml @@ -396,6 +396,29 @@ examples: image: "ghcr.io/kylehounslow/observability-stack/canary:latest" interval: "120" +# -- OpenTelemetry Demo (optional) +# Full microservices e-commerce app that generates realistic telemetry. +# Disabled by default — adds ~2GB memory. Enable with opentelemetry-demo.enabled=true. +# All bundled backends (jaeger, grafana, prometheus, opensearch, collector) are disabled — +# demo services send telemetry to our OTel Collector instead. +opentelemetry-demo: + enabled: false + default: + envOverrides: + - name: OTEL_COLLECTOR_NAME + value: obs-stack-opentelemetry-collector + # Disable all bundled backends — we use our own stack + opentelemetry-collector: + enabled: false + jaeger: + enabled: false + prometheus: + enabled: false + grafana: + enabled: false + opensearch: + enabled: false + # -- Gateway API for OpenSearch Dashboards # Requires: Gateway API CRDs + a gateway controller installed on the cluster. # Supported providers: envoy (Envoy Gateway), aws (AWS Gateway API Controller). From 628e651d53097aae978142f710f2ebfa17357f7e Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Wed, 18 Mar 2026 23:27:53 -0700 Subject: [PATCH 36/64] docs: add OTel Demo and GHCR images to READMEs --- charts/observability-stack/README.md | 19 +++++++++++++++++++ terraform/aws/README.md | 12 ++++++++++++ 2 files changed, 31 insertions(+) diff --git a/charts/observability-stack/README.md b/charts/observability-stack/README.md index 8bf4d57d..bcf8f47d 100644 --- a/charts/observability-stack/README.md +++ b/charts/observability-stack/README.md @@ -123,3 +123,22 @@ prometheus: kubernetes-api-servers: { enabled: false } # ... etc ``` + +## OpenTelemetry Demo (Optional) + +The [OpenTelemetry Demo](https://opentelemetry.io/docs/demo/) is available as an optional subchart. It deploys a full microservices e-commerce app (20+ services) that generates realistic telemetry — useful for load testing and showcasing the stack. + +Disabled by default (~2GB additional memory required). + +**Enable:** +```bash +helm upgrade obs-stack . -n observability-stack -f ../../terraform/aws/values-eks.yaml \ + --set opentelemetry-demo.enabled=true --no-hooks +``` + +**Disable:** +```bash +helm upgrade obs-stack . -n observability-stack -f ../../terraform/aws/values-eks.yaml --no-hooks +``` + +All bundled backends (Jaeger, Grafana, Prometheus, OpenSearch) in the demo chart are disabled — demo services send telemetry to our OTel Collector. No duplicate infrastructure. diff --git a/terraform/aws/README.md b/terraform/aws/README.md index 2502d947..ae83b28c 100644 --- a/terraform/aws/README.md +++ b/terraform/aws/README.md @@ -76,6 +76,18 @@ kubectl port-forward -n observability-stack svc/obs-stack-opentelemetry-collecto export OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4317" ``` +## Add OpenTelemetry Demo + +To deploy the [OTel Demo](https://opentelemetry.io/docs/demo/) microservices app alongside the stack: + +```bash +cd terraform/aws +helm upgrade obs-stack ../../charts/observability-stack -n observability-stack -f values-eks.yaml \ + --set opentelemetry-demo.enabled=true --no-hooks +``` + +Adds 20+ services generating realistic e-commerce telemetry (~2GB additional memory). All demo telemetry flows through the stack's OTel Collector — no duplicate backends. + ## Destroy ```bash From 7daac6737f84c3cb601822416b7d2ffde1b57b46 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Thu, 19 Mar 2026 13:13:35 -0700 Subject: [PATCH 37/64] feat: add enterprise sizing knobs and sizing guide - OpenSearch: persistence.size, OPENSEARCH_JAVA_OPTS, storageClass - Data Prepper: number_of_shards/number_of_replicas on opensearch sinks - Prometheus: server.retention (15d default), persistentVolume options - README: sizing guide with storage formula, shard rules, quick-reference profiles (dev/small team/enterprise) --- charts/observability-stack/README.md | 55 ++++++++++++++++++++++++++ charts/observability-stack/values.yaml | 24 ++++++++++- 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/charts/observability-stack/README.md b/charts/observability-stack/README.md index bcf8f47d..d46c830c 100644 --- a/charts/observability-stack/README.md +++ b/charts/observability-stack/README.md @@ -102,6 +102,61 @@ Configured via `scrapeConfigs` in `values.yaml`. Default K8s scrape jobs are dis > **Note:** Targets use the helm release name as prefix. The values in `values.yaml` are hardcoded to `obs-stack-*` — update them if you change the release name. +## Sizing Guide + +The default values are tuned for development/demo (single-node OpenSearch, minimal resources). For production or enterprise-scale deployments, adjust the following knobs. + +### OpenSearch Cluster + +| Knob | Default | Production Guidance | +|------|---------|---------------------| +| `opensearch.replicas` | `1` | 3+ data nodes minimum for HA | +| `opensearch.singleNode` | `true` | Set `false` for multi-node | +| `opensearch.resources.requests.memory` | `2Gi` | 8–64Gi per node (JVM gets 50%) | +| `opensearch.persistence.size` | `8Gi` | Size per formula below | +| `opensearch.extraEnvs[OPENSEARCH_JAVA_OPTS]` | `-Xms1g -Xmx1g` | 50% of node RAM, max 31g | + +**Storage formula:** +``` +storage_per_node = (daily_ingest_GB × 1.45 × (replicas + 1) × retention_days) / node_count +``` +The 1.45x multiplier accounts for indexing overhead (10%), OS reserved space for merges (20%), filesystem overhead (5%), and node failure buffer (10%). + +**Shard sizing:** +- Logs/traces (write-heavy): 30–50 GB per primary shard +- Search (latency-sensitive): 10–30 GB per primary shard +- Total shards should be a multiple of data node count +- Max 25 shards per GB of JVM heap + +Shard count is configurable per Data Prepper pipeline sink via `number_of_shards` and `number_of_replicas` (commented out in `values.yaml`). + +### Data Prepper Pipeline Tuning + +| Knob | Default | Description | +|------|---------|-------------| +| `data-prepper.pipelineConfig.config.otel-logs-pipeline.workers` | `5` | Parallel log processing threads | +| `...opensearch.number_of_shards` | (OS default: 1) | Primary shards per index | +| `...opensearch.number_of_replicas` | (OS default: 1) | Replica shards per primary | +| `...opensearch.bulk_size` | `5` (MiB) | Bulk request size to OpenSearch | + +### Prometheus + +| Knob | Default | Description | +|------|---------|-------------| +| `prometheus.server.retention` | `15d` | How long metrics are kept | +| `prometheus.server.persistentVolume.enabled` | `false` | Enable for production | +| `prometheus.server.persistentVolume.size` | `8Gi` | Disk for metrics TSDB | + +### Quick Reference: Sizing Profiles + +| Profile | OS Nodes | OS Memory | OS Disk | Prometheus Retention | +|---------|----------|-----------|---------|---------------------| +| **Dev/Demo** (default) | 1 | 2Gi | 8Gi | 15d | +| **Small team** (~10 GB/day) | 3 | 8Gi | 100Gi | 30d | +| **Enterprise** (~100 GB/day) | 6+ | 32Gi | 500Gi+ | 90d | + +Sources: [OpenSearch shard sizing](https://opensearch.org/blog/optimize-opensearch-index-shard-size/), [AWS sizing guide](https://docs.aws.amazon.com/prescriptive-guidance/latest/opensearch-service-migration/sizing.html), [AWS shard best practices](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/bp-sharding.html) + ## Key Values See `values.yaml` for all options. Notable settings: diff --git a/charts/observability-stack/values.yaml b/charts/observability-stack/values.yaml index 1955294f..090c58f8 100644 --- a/charts/observability-stack/values.yaml +++ b/charts/observability-stack/values.yaml @@ -2,6 +2,12 @@ # Mirrors the docker-compose setup for Kubernetes deployment # -- OpenSearch +# Sizing guide: +# Storage: daily_ingest_GB × 1.45 × (replicas + 1) × retention_days +# Shards: 30–50 GB per shard for logs/traces, 10–30 GB for search +# JVM: 50% of node RAM, max ~31 GB (set via OPENSEARCH_JAVA_OPTS) +# Nodes: minimum 3 for production, 1 for dev/demo +# Heap-to-shard ratio: max 25 shards per GB of JVM heap opensearch: enabled: true singleNode: true @@ -13,9 +19,16 @@ opensearch: requests: memory: "2Gi" cpu: "500m" + persistence: + enabled: true + size: 8Gi # Increase for production (e.g. 100Gi, 500Gi) + # storageClass: "gp3" # Uncomment for AWS gp3 (better IOPS/$ than gp2) extraEnvs: - name: OPENSEARCH_INITIAL_ADMIN_PASSWORD value: "My_password_123!@#" + # JVM heap — set to 50% of resources.requests.memory, max 31g + - name: OPENSEARCH_JAVA_OPTS + value: "-Xms1g -Xmx1g" config: opensearch.yml: | plugins.query.datasources.encryption.masterkey: "BTqK4Ytdz67La1kShIKV3Pu9" @@ -135,6 +148,10 @@ data-prepper: password: "My_password_123!@#" insecure: true index_type: log-analytics-plain + # Shard tuning — adjust for ingest volume: + # 1 shard handles ~30-50 GB for logs. Scale shards with data node count. + # number_of_shards: 1 + # number_of_replicas: 1 otel-traces-pipeline: delay: 100 @@ -160,6 +177,8 @@ data-prepper: password: "My_password_123!@#" insecure: true index_type: trace-analytics-plain-raw + # number_of_shards: 1 + # number_of_replicas: 1 service-map-pipeline: delay: 100 @@ -277,8 +296,11 @@ opentelemetry-collector: prometheus: enabled: true server: + # Retention — how long Prometheus keeps metrics. Increase for longer history. + retention: "15d" persistentVolume: - enabled: false + enabled: false # Enable for production (survives pod restarts) + # size: 50Gi extraFlags: - "web.enable-remote-write-receiver" - "web.enable-otlp-receiver" From 16864696ce5d0879c900e3df8584980b4651b6ec Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Thu, 19 Mar 2026 16:57:54 -0700 Subject: [PATCH 38/64] test: add helm-unittest suite for chart templates - 29 tests across 6 suites covering all custom templates - credentials, examples, gateway, init-dashboards, opensearch-exporter - Tests conditional rendering, custom values, labels, annotations - Wrapper script runs helm lint + helm unittest - Requires helm-unittest plugin --- .../tests/credentials_test.yaml | 41 ++++++ .../tests/examples_test.yaml | 69 ++++++++++ .../tests/gateway_test.yaml | 121 ++++++++++++++++++ .../tests/init_dashboards_configmap_test.yaml | 24 ++++ .../tests/init_dashboards_job_test.yaml | 35 +++++ .../tests/opensearch_exporter_test.yaml | 57 +++++++++ test/helm-test.sh | 14 ++ 7 files changed, 361 insertions(+) create mode 100644 charts/observability-stack/tests/credentials_test.yaml create mode 100644 charts/observability-stack/tests/examples_test.yaml create mode 100644 charts/observability-stack/tests/gateway_test.yaml create mode 100644 charts/observability-stack/tests/init_dashboards_configmap_test.yaml create mode 100644 charts/observability-stack/tests/init_dashboards_job_test.yaml create mode 100644 charts/observability-stack/tests/opensearch_exporter_test.yaml create mode 100755 test/helm-test.sh diff --git a/charts/observability-stack/tests/credentials_test.yaml b/charts/observability-stack/tests/credentials_test.yaml new file mode 100644 index 00000000..98ea1571 --- /dev/null +++ b/charts/observability-stack/tests/credentials_test.yaml @@ -0,0 +1,41 @@ +suite: opensearch credentials secret +templates: + - templates/opensearch-credentials-secret.yaml +tests: + - it: should render with default credentials + asserts: + - isKind: + of: Secret + - equal: + path: stringData.username + value: admin + - equal: + path: stringData.password + value: "My_password_123!@#" + + - it: should use custom credentials + set: + opensearchUsername: myuser + opensearchPassword: "S3cret!" + asserts: + - equal: + path: stringData.username + value: myuser + - equal: + path: stringData.password + value: "S3cret!" + + - it: should include standard labels + asserts: + - exists: + path: metadata.labels["helm.sh/chart"] + - exists: + path: metadata.labels["app.kubernetes.io/managed-by"] + + - it: should respect fullnameOverride + set: + fullnameOverride: custom + asserts: + - equal: + path: metadata.name + value: custom-opensearch-credentials diff --git a/charts/observability-stack/tests/examples_test.yaml b/charts/observability-stack/tests/examples_test.yaml new file mode 100644 index 00000000..0d69259f --- /dev/null +++ b/charts/observability-stack/tests/examples_test.yaml @@ -0,0 +1,69 @@ +suite: example agents +templates: + - templates/examples.yaml +tests: + - it: should render all example deployments and services by default + asserts: + - hasDocuments: + count: 9 # 4 services + 5 deployments + - containsDocument: + kind: Deployment + apiVersion: apps/v1 + name: example-weather-agent + any: true + - containsDocument: + kind: Deployment + apiVersion: apps/v1 + name: example-events-agent + any: true + - containsDocument: + kind: Deployment + apiVersion: apps/v1 + name: example-travel-planner + any: true + - containsDocument: + kind: Deployment + apiVersion: apps/v1 + name: example-mcp-server + any: true + - containsDocument: + kind: Deployment + apiVersion: apps/v1 + name: example-canary + any: true + - containsDocument: + kind: Service + apiVersion: v1 + name: weather-agent + any: true + + - it: should not render when disabled + set: + examples.enabled: false + asserts: + - hasDocuments: + count: 0 + + - it: should set OTEL_EXPORTER_OTLP_ENDPOINT on weather-agent + documentSelector: + path: metadata.name + value: example-weather-agent + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://RELEASE-NAME-opentelemetry-collector:4317" + + - it: should set canary interval from values + set: + examples.canary.interval: "60" + documentSelector: + path: metadata.name + value: example-canary + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: CANARY_INTERVAL + value: "60" diff --git a/charts/observability-stack/tests/gateway_test.yaml b/charts/observability-stack/tests/gateway_test.yaml new file mode 100644 index 00000000..1925cc1b --- /dev/null +++ b/charts/observability-stack/tests/gateway_test.yaml @@ -0,0 +1,121 @@ +suite: gateway +templates: + - templates/gateway.yaml +tests: + - it: should not render when disabled + asserts: + - hasDocuments: + count: 0 + + - it: should render Gateway and HTTPRoute when enabled + set: + gateway: + enabled: true + provider: envoy + className: eg + host: dashboards.example.com + tls: + secretName: my-tls + asserts: + - hasDocuments: + count: 2 + - containsDocument: + kind: Gateway + apiVersion: gateway.networking.k8s.io/v1 + any: true + - containsDocument: + kind: HTTPRoute + apiVersion: gateway.networking.k8s.io/v1 + any: true + + - it: should set gatewayClassName + set: + gateway: + enabled: true + provider: envoy + className: eg + documentIndex: 0 + asserts: + - equal: + path: spec.gatewayClassName + value: eg + + - it: should set hostname on Gateway and HTTPRoute + set: + gateway: + enabled: true + provider: envoy + className: eg + host: dashboards.example.com + asserts: + - equal: + path: spec.listeners[0].hostname + value: dashboards.example.com + documentIndex: 0 + - contains: + path: spec.hostnames + content: dashboards.example.com + documentIndex: 1 + + - it: should include certificateRefs for envoy with TLS + set: + gateway: + enabled: true + provider: envoy + className: eg + tls: + secretName: my-cert + documentIndex: 0 + asserts: + - equal: + path: spec.listeners[0].tls.certificateRefs[0].name + value: my-cert + + - it: should not include certificateRefs for aws provider + set: + gateway: + enabled: true + provider: aws + className: amazon-vpc-lattice + documentIndex: 0 + asserts: + - notExists: + path: spec.listeners[0].tls.certificateRefs + + - it: should set TLS terminate mode + set: + gateway: + enabled: true + provider: envoy + className: eg + documentIndex: 0 + asserts: + - equal: + path: spec.listeners[0].tls.mode + value: Terminate + + - it: should route to opensearch-dashboards on port 5601 + set: + gateway: + enabled: true + provider: envoy + className: eg + documentIndex: 1 + asserts: + - equal: + path: spec.rules[0].backendRefs[0].port + value: 5601 + + - it: should apply custom annotations + set: + gateway: + enabled: true + provider: aws + className: amazon-vpc-lattice + annotations: + application-networking.k8s.aws/certificate-arn: "arn:aws:acm:us-east-1:123:certificate/abc" + documentIndex: 0 + asserts: + - equal: + path: metadata.annotations["application-networking.k8s.aws/certificate-arn"] + value: "arn:aws:acm:us-east-1:123:certificate/abc" diff --git a/charts/observability-stack/tests/init_dashboards_configmap_test.yaml b/charts/observability-stack/tests/init_dashboards_configmap_test.yaml new file mode 100644 index 00000000..614ce69d --- /dev/null +++ b/charts/observability-stack/tests/init_dashboards_configmap_test.yaml @@ -0,0 +1,24 @@ +suite: init dashboards configmap +templates: + - templates/init-dashboards-configmap.yaml +tests: + - it: should render when dashboards enabled + asserts: + - isKind: + of: ConfigMap + - isAPIVersion: + of: v1 + + - it: should not render when dashboards disabled + set: + opensearch-dashboards: + enabled: false + asserts: + - hasDocuments: + count: 0 + + - it: should use post-install hook + asserts: + - equal: + path: metadata.annotations["helm.sh/hook"] + value: post-install,post-upgrade diff --git a/charts/observability-stack/tests/init_dashboards_job_test.yaml b/charts/observability-stack/tests/init_dashboards_job_test.yaml new file mode 100644 index 00000000..d8317da1 --- /dev/null +++ b/charts/observability-stack/tests/init_dashboards_job_test.yaml @@ -0,0 +1,35 @@ +suite: init dashboards job +templates: + - templates/init-dashboards-job.yaml +tests: + - it: should render when dashboards enabled + asserts: + - isKind: + of: Job + - isAPIVersion: + of: batch/v1 + + - it: should not render when dashboards disabled + set: + opensearch-dashboards: + enabled: false + asserts: + - hasDocuments: + count: 0 + + - it: should use post-install hook + asserts: + - equal: + path: metadata.annotations["helm.sh/hook"] + value: post-install,post-upgrade + + - it: should read credentials from secret + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSEARCH_USER + valueFrom: + secretKeyRef: + name: RELEASE-NAME-observability-stack-opensearch-credentials + key: username diff --git a/charts/observability-stack/tests/opensearch_exporter_test.yaml b/charts/observability-stack/tests/opensearch_exporter_test.yaml new file mode 100644 index 00000000..ccb8e91c --- /dev/null +++ b/charts/observability-stack/tests/opensearch_exporter_test.yaml @@ -0,0 +1,57 @@ +suite: opensearch exporter +templates: + - templates/opensearch-exporter.yaml +tests: + - it: should render deployment and service by default + asserts: + - hasDocuments: + count: 2 + - containsDocument: + kind: Deployment + apiVersion: apps/v1 + any: true + - containsDocument: + kind: Service + apiVersion: v1 + any: true + + - it: should not render when disabled + set: + opensearchExporter.enabled: false + asserts: + - hasDocuments: + count: 0 + + - it: should use configured image + documentIndex: 0 + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: "prometheuscommunity/elasticsearch-exporter:v1.10.0" + + - it: should use custom credentials in env + set: + opensearchUsername: myuser + opensearchPassword: "S3cret!" + documentIndex: 0 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: ES_USERNAME + value: myuser + - contains: + path: spec.template.spec.containers[0].env + content: + name: ES_PASSWORD + value: "S3cret!" + + - it: should expose metrics port 9114 + documentIndex: 1 + asserts: + - contains: + path: spec.ports + content: + port: 9114 + targetPort: 9114 + name: metrics diff --git a/test/helm-test.sh b/test/helm-test.sh new file mode 100755 index 00000000..455683c2 --- /dev/null +++ b/test/helm-test.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Helm lint + unit tests for the observability-stack chart. +# Requires: helm, helm-unittest plugin (helm plugin install https://github.com/helm-unittest/helm-unittest.git) +# Usage: ./test/helm-test.sh +set -euo pipefail + +CHART="charts/observability-stack" + +echo "==> helm lint" +helm lint "$CHART" + +echo "" +echo "==> helm unittest" +helm unittest "$CHART" From ee6cc63d19e9a2819a8cba7703e37560aa1571d3 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Thu, 19 Mar 2026 16:58:55 -0700 Subject: [PATCH 39/64] ci: add helm lint + unittest workflow Runs on push/PR to main when charts/ or test/helm-test.sh change. --- .github/workflows/helm-test.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/helm-test.yml diff --git a/.github/workflows/helm-test.yml b/.github/workflows/helm-test.yml new file mode 100644 index 00000000..10167a0f --- /dev/null +++ b/.github/workflows/helm-test.yml @@ -0,0 +1,30 @@ +name: Helm Tests + +on: + push: + branches: [main] + paths: + - 'charts/**' + - 'test/helm-test.sh' + pull_request: + branches: [main] + paths: + - 'charts/**' + - 'test/helm-test.sh' + workflow_dispatch: + +jobs: + helm-test: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + + - name: Set up Helm + uses: azure/setup-helm@v4 + + - name: Install helm-unittest plugin + run: helm plugin install https://github.com/helm-unittest/helm-unittest.git + + - name: Run helm lint + unittest + run: ./test/helm-test.sh From b2530cd16af2f01d6553f68f6574877e595fe90c Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 20 Mar 2026 10:56:00 -0700 Subject: [PATCH 40/64] feat: add load testing scripts (OSB, telemetrygen, k6 API + browser) --- load-testing/README.md | 243 ++++++++++++++++++ load-testing/k6/full-test.js | 192 ++++++++++++++ load-testing/k6/scenarios/api-queries.js | 137 ++++++++++ load-testing/k6/scenarios/browser-discover.js | 104 ++++++++ load-testing/k6/scenarios/browser-metrics.js | 103 ++++++++ load-testing/k6/scenarios/browser-traces.js | 100 +++++++ load-testing/osb/index-settings.json | 25 ++ load-testing/osb/run-osb.sh | 34 +++ load-testing/osb/workload.json | 99 +++++++ load-testing/pipeline/run-telemetrygen.sh | 67 +++++ 10 files changed, 1104 insertions(+) create mode 100644 load-testing/README.md create mode 100644 load-testing/k6/full-test.js create mode 100644 load-testing/k6/scenarios/api-queries.js create mode 100644 load-testing/k6/scenarios/browser-discover.js create mode 100644 load-testing/k6/scenarios/browser-metrics.js create mode 100644 load-testing/k6/scenarios/browser-traces.js create mode 100644 load-testing/osb/index-settings.json create mode 100755 load-testing/osb/run-osb.sh create mode 100644 load-testing/osb/workload.json create mode 100755 load-testing/pipeline/run-telemetrygen.sh diff --git a/load-testing/README.md b/load-testing/README.md new file mode 100644 index 00000000..e1944e3a --- /dev/null +++ b/load-testing/README.md @@ -0,0 +1,243 @@ +# Observability Stack Load Testing Plan + +## Goal + +Determine the breaking points and concurrent user capacity of the Observability Stack for a given Helm deployment. Testing is manual (not CI-automated) and covers both backend ingestion throughput and frontend dashboard responsiveness. + +## Test Environment + +- Helm release: `obs-stack` in namespace `observability-stack` +- Chart: `observability-stack-0.1.0` +- OpenTelemetry Demo: **enabled** — full microservices e-commerce app generating realistic telemetry via built-in load generator +- Prometheus: single pod (`obs-stack-prometheus-server`) +- OpenSearch: single node (`singleNode: true`) +- OTel demo services provide baseline ingestion load (traces, logs, metrics from ~20 microservices) + +## Architecture Under Test + +``` +telemetrygen / OSB + │ + ▼ +OTel Collector (4317/4318) + │ + ├──► Data Prepper ──► OpenSearch (logs, traces) + └──► Prometheus (metrics, single pod) + │ + ▼ + OpenSearch Dashboards + ├── Discover (queries OpenSearch) + ├── Trace Analytics (queries OpenSearch) + ├── Metric Panels (queries Prometheus) + └── PPL queries (queries OpenSearch) +``` + +## Tools + +| Tool | Purpose | Install | +|------|---------|---------| +| [OpenSearch Benchmark (OSB)](https://github.com/opensearch-project/OpenSearch-Benchmark) | Direct OpenSearch indexing/query load + redline testing | `pip install opensearch-benchmark` | +| [telemetrygen](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/cmd/telemetrygen) | OTLP trace/log/metric generation through the full pipeline | `go install github.com/open-telemetry/opentelemetry-collector-contrib/cmd/telemetrygen@latest` | +| [k6 + browser module](https://grafana.com/docs/k6/latest/using-k6-browser/) | HTTP API load + real browser UI simulation | `brew install k6` (browser module is built-in) | + +## Test Layers + +### Layer 1: OpenSearch Capacity (OSB Direct) + +Bypasses the pipeline to find OpenSearch's own ceiling. + +**What it tests:** Indexing throughput (docs/sec), query latency under load, heap pressure, thread pool rejections. + +**How:** +```bash +# Redline test — auto-ramps until the cluster breaks +opensearch-benchmark execute-test \ + --target-hosts=https://opensearch:9200 \ + --pipeline=benchmark-only \ + --workload=pmc \ + --client-options="use_ssl:true,verify_certs:false,basic_auth_user:admin,basic_auth_password:My_password_123!@#" \ + --redline-test +``` + +**Custom workload (TODO):** Create an OSB workload that uses trace/log document shapes matching our `otel-v1-apm-span-*` and `otel-v1-apm-log-*` indices for realistic testing. + +**Metrics to watch:** +- Indexing throughput (docs/sec) +- p50/p95/p99 query latency +- `_nodes/stats`: heap used %, thread pool rejected count, merge times +- Pod CPU/memory via `kubectl top pod` + +### Layer 2: Pipeline Throughput (telemetrygen → OTel Collector) + +Tests the full ingestion path: OTel Collector → Data Prepper → OpenSearch. + +**What it tests:** End-to-end pipeline capacity, backpressure behavior, which component saturates first. + +**How:** +```bash +# Traces — ramp up spans/sec until pipeline drops data +telemetrygen traces \ + --otlp-endpoint=otel-collector:4317 \ + --otlp-insecure \ + --rate=100 \ + --duration=5m \ + --service=load-test-agent \ + --otlp-attributes='gen_ai.agent.name="load-test"' + +# Logs +telemetrygen logs \ + --otlp-endpoint=otel-collector:4317 \ + --otlp-insecure \ + --rate=100 \ + --duration=5m \ + --service=load-test-agent + +# Metrics +telemetrygen metrics \ + --otlp-endpoint=otel-collector:4317 \ + --otlp-insecure \ + --rate=50 \ + --duration=5m \ + --service=load-test-agent +``` + +Increase `--rate` in increments (100 → 500 → 1000 → 5000) until failures appear. + +**Metrics to watch:** +- Collector: `otelcol_exporter_sent_spans` vs `otelcol_exporter_send_failed_spans` +- Collector: `otelcol_processor_batch_timeout_trigger_send` (batch pressure) +- Data Prepper: pipeline queue depth, processing latency +- OpenSearch: bulk indexing rejection rate +- End-to-end latency: time from telemetrygen send to document appearing in OpenSearch + +### Layer 3: Dashboard UI Under Load (k6) + +Simulates concurrent users interacting with OpenSearch Dashboards while the cluster is under ingestion load from Layers 1-2. + +**What it tests:** Dashboard responsiveness, PPL query performance, Prometheus query capacity (single pod), OSD server memory/CPU. + +#### Scenarios + +| # | Scenario | Target Backend | +|---|----------|----------------| +| 1 | **Dashboard viewer** — Open saved dashboard, wait for panels, change time range, refresh | OpenSearch + Prometheus | +| 2 | **Discover explorer** — Select log index, run PPL query, paginate results | OpenSearch | +| 3 | **Trace analytics** — View traces list, click into a trace, expand spans, view service map | OpenSearch | +| 4 | **Expensive PPL** — High-cardinality aggregations, long time ranges, `dedup`, `stats ... by` | OpenSearch | +| 5 | **Metrics explorer** — Open metric visualizations, PromQL queries, change time range to 7d | **Prometheus (single pod)** | +| 6 | **APM browser** — Services list, click service, view latency/error panels, drill into operations | OpenSearch + Prometheus | + +#### k6 Test Structure + +Two scenario types run concurrently in a single k6 test: + +**API-level VUs (high scale):** Replay the HTTP requests that OSD makes under the hood. Scales to hundreds of concurrent users. + +- `POST _plugins/_ppl` — PPL queries of varying complexity +- `POST _search` — Discover-style searches against trace/log indices +- `GET prometheus:9090/api/v1/query_range` — PromQL queries mirroring metric panels +- `GET _dashboards/api/saved_objects` — Dashboard/visualization loads + +**Browser VUs (low scale, high fidelity):** Real Chromium sessions clicking through OSD. 5-20 concurrent sessions. + +- Login → navigate to Traces → click trace → expand spans +- Login → Discover → run PPL → paginate +- Login → Dashboard → change time range → wait for render + +#### Ramp-Up Strategy (Finding Breaking Points) + +``` +Phase 1: API-only ramp + 0 → 50 → 200 → 500 VUs over 15 min + Find: p95 latency spike, first errors + +Phase 2: Browser users on top + Hold API at ~70% of Phase 1 breaking point + Add 5 → 10 → 15 → 20 browser VUs + Find: OSD pod OOM, page load > 10s + +Phase 3: Combined with ingestion + Run telemetrygen at ~70% of Layer 2 breaking point + Repeat Phase 1+2 + Find: degradation from concurrent read+write load +``` + +#### Prometheus-Specific Stress + +Since Prometheus is a single pod, dedicate specific API VUs to PromQL queries: + +- `rate(gen_ai_usage_input_tokens_total[5m])` — simple +- `sum by (service_name, agent_name, model) (rate(gen_ai_usage_input_tokens_total[5m]))` — high cardinality fan-out +- Same queries with `[7d]` range — memory-heavy +- Multiple concurrent `query_range` requests with overlapping time windows + +**Metrics to watch:** +- Prometheus pod CPU/memory (`kubectl top pod`) +- Prometheus query duration (`prometheus_engine_query_duration_seconds`) +- Prometheus query failures (`prometheus_engine_queries_concurrent_max`) +- OSD response times for metric panels + +### k6 Thresholds + +```javascript +thresholds: { + // API-level + http_req_duration: ['p(95)<3000'], // API queries under 3s + http_req_failed: ['rate<0.05'], // <5% error rate + + // Browser-level + browser_web_vital_lcp: ['p(95)<4000'], // Largest Contentful Paint under 4s + browser_web_vital_cls: ['p(95)<0.25'], // Cumulative Layout Shift +} +``` + +## Execution Order + +1. **Layer 1** — Run OSB redline test to establish OpenSearch ceiling (no other load) +2. **Layer 2** — Run telemetrygen ramp to find pipeline throughput limit (no UI load) +3. **Layer 3 Phase 1** — API-only k6 ramp (no ingestion load) to find query capacity +4. **Layer 3 Phase 2** — Add browser VUs to find OSD/Prometheus limits +5. **Layer 3 Phase 3** — Combine: telemetrygen at 70% + k6 at 70% to find real-world capacity +6. **Report** — Document breaking points, bottleneck component, and max concurrent users per capacity tier + +## Expected Bottleneck Order (Hypothesis) + +1. **Prometheus (single pod)** — likely first to degrade under concurrent metric queries with long time ranges +2. **Data Prepper** — pipeline queue saturation under high ingestion rates +3. **OpenSearch Dashboards** — Node.js server memory under many concurrent browser sessions +4. **OpenSearch** — heap pressure from concurrent expensive queries + indexing + +## Deliverables + +- [ ] OSB custom workload matching our trace/log document shapes +- [ ] telemetrygen wrapper script with incremental rate steps +- [ ] k6 test scripts for all 6 UI scenarios (API + browser) +- [ ] Results report: breaking points per component, max concurrent users, resource utilization graphs +- [ ] Capacity recommendations: pod sizing for N concurrent users + +## Directory Structure + +``` +load-testing/ +├── README.md +├── osb/ +│ └── workload.json # Custom OSB workload for trace/log shapes +├── pipeline/ +│ └── run-telemetrygen.sh # Incremental rate ramp script +└── k6/ + ├── scenarios/ + │ ├── api-queries.js # API-level PPL, search, PromQL + │ ├── browser-traces.js # Browser: trace analytics flow + │ ├── browser-discover.js# Browser: discover + PPL flow + │ └── browser-metrics.js # Browser: metric panels flow + └── full-test.js # Combined ramp-up test +``` + +## References + +- [OpenSearch Benchmark docs](https://opensearch.org/docs/latest/benchmark/) +- [OSB Redline Testing](https://opensearch.org/blog/redline-testing-now-available-in-opensearch-benchmark/) +- [telemetrygen](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/cmd/telemetrygen) +- [k6 Browser Module](https://grafana.com/docs/k6/latest/using-k6-browser/) +- [OpenSearch PPL API](https://docs.opensearch.org/latest/sql-and-ppl/sql-and-ppl-api/index/) +- [Prometheus Query API](https://prometheus.io/docs/prometheus/latest/querying/api/) diff --git a/load-testing/k6/full-test.js b/load-testing/k6/full-test.js new file mode 100644 index 00000000..2e7caa02 --- /dev/null +++ b/load-testing/k6/full-test.js @@ -0,0 +1,192 @@ +// k6 combined load test — runs API-level and browser scenarios together. +// This is the "Phase 3" test from the load testing plan: combined read+write load. +// +// Usage: +// # Port-forward all services: +// kubectl port-forward -n observability-stack svc/opensearch-cluster-master 9200:9200 & +// kubectl port-forward -n observability-stack svc/obs-stack-prometheus-server 9090:80 & +// kubectl port-forward -n observability-stack svc/obs-stack-opensearch-dashboards 5601:5601 & +// +// K6_BROWSER_ENABLED=true k6 run k6/full-test.js +// K6_BROWSER_ENABLED=true k6 run k6/full-test.js --env TARGET_VUS=300 --env BROWSER_VUS=10 + +import http from 'k6/http'; +import { browser } from 'k6/browser'; +import { check, sleep } from 'k6'; + +const TARGET_VUS = parseInt(__ENV.TARGET_VUS || '150'); +const BROWSER_VUS = parseInt(__ENV.BROWSER_VUS || '5'); + +export const options = { + scenarios: { + // --- API layer: OpenSearch queries --- + api_opensearch: { + executor: 'ramping-vus', + startVUs: 0, + stages: [ + { duration: '2m', target: Math.round(TARGET_VUS * 0.5) }, + { duration: '5m', target: TARGET_VUS }, + { duration: '5m', target: TARGET_VUS }, + { duration: '2m', target: 0 }, + ], + exec: 'apiOpensearch', + }, + // --- API layer: Prometheus queries --- + api_prometheus: { + executor: 'ramping-vus', + startVUs: 0, + stages: [ + { duration: '2m', target: Math.round(TARGET_VUS * 0.25) }, + { duration: '5m', target: Math.round(TARGET_VUS * 0.5) }, + { duration: '5m', target: Math.round(TARGET_VUS * 0.5) }, + { duration: '2m', target: 0 }, + ], + exec: 'apiPrometheus', + }, + // --- Browser: Trace Analytics --- + browser_traces: { + executor: 'constant-vus', + vus: Math.max(1, Math.round(BROWSER_VUS * 0.4)), + duration: '12m', + exec: 'browserTraces', + startTime: '2m', // start after API ramp begins + options: { browser: { type: 'chromium' } }, + }, + // --- Browser: Discover + PPL --- + browser_discover: { + executor: 'constant-vus', + vus: Math.max(1, Math.round(BROWSER_VUS * 0.3)), + duration: '12m', + exec: 'browserDiscover', + startTime: '2m', + options: { browser: { type: 'chromium' } }, + }, + // --- Browser: Metrics dashboards --- + browser_metrics: { + executor: 'constant-vus', + vus: Math.max(1, Math.round(BROWSER_VUS * 0.3)), + duration: '12m', + exec: 'browserMetrics', + startTime: '2m', + options: { browser: { type: 'chromium' } }, + }, + }, + thresholds: { + http_req_duration: ['p(95)<5000'], + http_req_failed: ['rate<0.05'], + browser_web_vital_lcp: ['p(95)<8000'], + }, +}; + +// --- Config --- +const OS_BASE = __ENV.OPENSEARCH_URL || 'https://localhost:9200'; +const PROM_BASE = __ENV.PROMETHEUS_URL || 'http://localhost:9090'; +const DASHBOARDS_URL = __ENV.DASHBOARDS_URL || 'http://localhost:5601'; +const USERNAME = __ENV.OSD_USER || 'admin'; +const PASSWORD = __ENV.OSD_PASSWORD || 'My_password_123!@#'; + +const osParams = { + headers: { 'Content-Type': 'application/json' }, + auth: 'basic', + username: USERNAME, + password: PASSWORD, + insecureSkipTLSVerify: true, +}; + +const pplQueries = [ + 'source=otel-v1-apm-span-000001 | head 50', + 'source=otel-v1-apm-span-000001 | stats count() by serviceName', + 'source=otel-v1-apm-span-000001 | stats count() by serviceName, name | sort - count()', + 'source=logs-otel-v1-000001 | stats count() by serviceName', +]; + +const promQueries = [ + 'up', + 'rate(otelcol_exporter_sent_spans_total[5m])', + 'sum by (service_name) (rate(otelcol_exporter_sent_spans_total[5m]))', + 'histogram_quantile(0.99, rate(prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query"}[5m]))', + 'prometheus_tsdb_head_series', +]; + +// --- API functions --- + +export function apiOpensearch() { + const q = pplQueries[Math.floor(Math.random() * pplQueries.length)]; + http.post(`${OS_BASE}/_plugins/_ppl`, JSON.stringify({ query: q }), osParams); + http.post(`${OS_BASE}/otel-v1-apm-span-*/_search`, + JSON.stringify({ size: 50, query: { match_all: {} }, sort: [{ startTime: 'desc' }] }), + osParams); + sleep(Math.random() * 2 + 1); +} + +export function apiPrometheus() { + const q = promQueries[Math.floor(Math.random() * promQueries.length)]; + const now = Math.floor(Date.now() / 1000); + http.get(`${PROM_BASE}/api/v1/query_range?query=${encodeURIComponent(q)}&start=${now - 3600}&end=${now}&step=60`); + sleep(Math.random() * 2 + 1); +} + +// --- Browser functions --- + +async function login(page) { + await page.goto(`${DASHBOARDS_URL}/app/home`); + const userField = await page.locator('[data-test-subj="user-name"]'); + if (await userField.isVisible()) { + await userField.fill(USERNAME); + await page.locator('[data-test-subj="password"]').fill(PASSWORD); + await page.locator('[data-test-subj="submit"]').click(); + await page.waitForNavigation(); + } +} + +export async function browserTraces() { + const ctx = await browser.newContext({ ignoreHTTPSErrors: true }); + const page = await ctx.newPage(); + try { + await login(page); + await page.goto(`${DASHBOARDS_URL}/app/observability-traces#/traces`); + await page.waitForTimeout(5000); + const row = await page.locator('table tbody tr').first(); + if (await row.isVisible()) { + await row.click(); + await page.waitForTimeout(5000); + } + sleep(3); + } finally { + await page.close(); + await ctx.close(); + } +} + +export async function browserDiscover() { + const ctx = await browser.newContext({ ignoreHTTPSErrors: true }); + const page = await ctx.newPage(); + try { + await login(page); + await page.goto(`${DASHBOARDS_URL}/app/data-explorer/discover`); + await page.waitForTimeout(5000); + sleep(3); + } finally { + await page.close(); + await ctx.close(); + } +} + +export async function browserMetrics() { + const ctx = await browser.newContext({ ignoreHTTPSErrors: true }); + const page = await ctx.newPage(); + try { + await login(page); + await page.goto(`${DASHBOARDS_URL}/app/dashboards`); + await page.waitForTimeout(3000); + const link = await page.locator('table tbody tr a').first(); + if (await link.isVisible()) { + await link.click(); + await page.waitForTimeout(8000); + } + sleep(3); + } finally { + await page.close(); + await ctx.close(); + } +} diff --git a/load-testing/k6/scenarios/api-queries.js b/load-testing/k6/scenarios/api-queries.js new file mode 100644 index 00000000..a61fe90a --- /dev/null +++ b/load-testing/k6/scenarios/api-queries.js @@ -0,0 +1,137 @@ +// k6 API-level load test — replays the HTTP queries that OpenSearch Dashboards +// makes under the hood: PPL, _search, PromQL, saved objects. +// +// Usage: +// # Port-forward first: +// kubectl port-forward -n observability-stack svc/opensearch-cluster-master 9200:9200 & +// kubectl port-forward -n observability-stack svc/obs-stack-prometheus-server 9090:80 & +// kubectl port-forward -n observability-stack svc/obs-stack-opensearch-dashboards 5601:5601 & +// +// k6 run scenarios/api-queries.js +// k6 run scenarios/api-queries.js --env TARGET_VUS=500 # override peak + +import http from 'k6/http'; +import { check, sleep } from 'k6'; +import { Rate, Trend } from 'k6/metrics'; + +const TARGET_VUS = parseInt(__ENV.TARGET_VUS || '200'); + +export const options = { + scenarios: { + opensearch_queries: { + executor: 'ramping-vus', + startVUs: 0, + stages: [ + { duration: '2m', target: Math.round(TARGET_VUS * 0.25) }, + { duration: '3m', target: Math.round(TARGET_VUS * 0.5) }, + { duration: '5m', target: TARGET_VUS }, + { duration: '3m', target: TARGET_VUS }, // hold at peak + { duration: '2m', target: 0 }, + ], + exec: 'opensearchLoad', + }, + prometheus_queries: { + executor: 'ramping-vus', + startVUs: 0, + stages: [ + { duration: '2m', target: Math.round(TARGET_VUS * 0.1) }, + { duration: '3m', target: Math.round(TARGET_VUS * 0.25) }, + { duration: '5m', target: Math.round(TARGET_VUS * 0.5) }, + { duration: '3m', target: Math.round(TARGET_VUS * 0.5) }, + { duration: '2m', target: 0 }, + ], + exec: 'prometheusLoad', + }, + }, + thresholds: { + http_req_duration: ['p(95)<5000'], + http_req_failed: ['rate<0.05'], + }, +}; + +const OS_BASE = __ENV.OPENSEARCH_URL || 'https://localhost:9200'; +const PROM_BASE = __ENV.PROMETHEUS_URL || 'http://localhost:9090'; +const OS_AUTH = { username: 'admin', password: 'My_password_123!@#' }; + +const osParams = { + headers: { 'Content-Type': 'application/json' }, + auth: 'basic', + ...OS_AUTH, + insecureSkipTLSVerify: true, +}; + +// --- PPL queries (light → heavy) --- +const pplQueries = [ + 'source=otel-v1-apm-span-000001 | head 50', + 'source=otel-v1-apm-span-000001 | stats count() by serviceName', + 'source=otel-v1-apm-span-000001 | where serviceName="frontend" | stats avg(durationInNanos) by name', + 'source=otel-v1-apm-span-000001 | stats count() by serviceName, name | sort - count()', + 'source=otel-v1-apm-span-000001 | where durationInNanos > 1000000000 | stats count() by serviceName | sort - count()', + 'source=logs-otel-v1-000001 | head 50', + 'source=logs-otel-v1-000001 | stats count() by serviceName', +]; + +// --- PromQL queries (light → heavy) --- +const promQueries = [ + 'up', + 'rate(otelcol_exporter_sent_spans_total[5m])', + 'sum by (service_name) (rate(otelcol_exporter_sent_spans_total[5m]))', + 'histogram_quantile(0.99, rate(prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query"}[5m]))', + 'rate(prometheus_tsdb_head_samples_appended_total[5m])', + 'prometheus_tsdb_head_series', + 'sum by (job) (rate(otelcol_receiver_accepted_spans_total[5m]))', +]; + +export function opensearchLoad() { + const queryIdx = Math.floor(Math.random() * pplQueries.length); + + // PPL query + const pplRes = http.post( + `${OS_BASE}/_plugins/_ppl`, + JSON.stringify({ query: pplQueries[queryIdx] }), + osParams, + ); + check(pplRes, { 'PPL 2xx': (r) => r.status >= 200 && r.status < 300 }); + + // Discover-style _search + const searchRes = http.post( + `${OS_BASE}/otel-v1-apm-span-*/_search`, + JSON.stringify({ + size: 50, + query: { match_all: {} }, + sort: [{ startTime: 'desc' }], + }), + osParams, + ); + check(searchRes, { 'Search 2xx': (r) => r.status >= 200 && r.status < 300 }); + + // Service map + const smRes = http.post( + `${OS_BASE}/otel-v2-apm-service-map-*/_search`, + JSON.stringify({ size: 200, query: { match_all: {} } }), + osParams, + ); + check(smRes, { 'ServiceMap 2xx': (r) => r.status >= 200 && r.status < 300 }); + + sleep(Math.random() * 2 + 1); // 1-3s think time +} + +export function prometheusLoad() { + const queryIdx = Math.floor(Math.random() * promQueries.length); + const now = Math.floor(Date.now() / 1000); + const oneHourAgo = now - 3600; + + // Instant query + const instantRes = http.get( + `${PROM_BASE}/api/v1/query?query=${encodeURIComponent(promQueries[queryIdx])}`, + ); + check(instantRes, { 'PromQL instant 2xx': (r) => r.status === 200 }); + + // Range query (1h window, 60s step) + const rangeRes = http.get( + `${PROM_BASE}/api/v1/query_range?query=${encodeURIComponent(promQueries[queryIdx])}&start=${oneHourAgo}&end=${now}&step=60`, + ); + check(rangeRes, { 'PromQL range 2xx': (r) => r.status === 200 }); + + sleep(Math.random() * 2 + 1); +} diff --git a/load-testing/k6/scenarios/browser-discover.js b/load-testing/k6/scenarios/browser-discover.js new file mode 100644 index 00000000..a0eddaff --- /dev/null +++ b/load-testing/k6/scenarios/browser-discover.js @@ -0,0 +1,104 @@ +// k6 browser test — Discover + PPL flow in OpenSearch Dashboards. +// Simulates a user opening Discover, selecting an index, running PPL queries. +// +// Usage: +// kubectl port-forward -n observability-stack svc/obs-stack-opensearch-dashboards 5601:5601 & +// K6_BROWSER_ENABLED=true k6 run scenarios/browser-discover.js + +import { browser } from 'k6/browser'; +import { check, sleep } from 'k6'; + +const TARGET_BROWSER_VUS = parseInt(__ENV.BROWSER_VUS || '5'); + +export const options = { + scenarios: { + discover_flow: { + executor: 'ramping-vus', + startVUs: 0, + stages: [ + { duration: '1m', target: 2 }, + { duration: '3m', target: TARGET_BROWSER_VUS }, + { duration: '3m', target: TARGET_BROWSER_VUS }, + { duration: '1m', target: 0 }, + ], + exec: 'discoverFlow', + options: { browser: { type: 'chromium' } }, + }, + }, + thresholds: { + browser_web_vital_lcp: ['p(95)<8000'], + }, +}; + +const DASHBOARDS_URL = __ENV.DASHBOARDS_URL || 'http://localhost:5601'; +const USERNAME = __ENV.OSD_USER || 'admin'; +const PASSWORD = __ENV.OSD_PASSWORD || 'My_password_123!@#'; + +async function login(page) { + await page.goto(`${DASHBOARDS_URL}/app/home`); + const userField = await page.locator('[data-test-subj="user-name"]'); + if (await userField.isVisible()) { + await userField.fill(USERNAME); + await page.locator('[data-test-subj="password"]').fill(PASSWORD); + await page.locator('[data-test-subj="submit"]').click(); + await page.waitForNavigation(); + } +} + +const pplQueries = [ + 'source=otel-v1-apm-span-000001 | head 50', + 'source=otel-v1-apm-span-000001 | stats count() by serviceName', + 'source=otel-v1-apm-span-000001 | where durationInNanos > 1000000000 | stats count() by serviceName', + 'source=logs-otel-v1-000001 | stats count() by serviceName', +]; + +export async function discoverFlow() { + const context = await browser.newContext({ ignoreHTTPSErrors: true }); + const page = await context.newPage(); + + try { + await login(page); + + // Navigate to Discover + await page.goto(`${DASHBOARDS_URL}/app/data-explorer/discover`); + await page.waitForTimeout(3000); + + check(page, { + 'Discover page loaded': () => true, + }); + + // Run a PPL query via the query bar + const query = pplQueries[Math.floor(Math.random() * pplQueries.length)]; + const queryInput = await page.locator('[data-test-subj="queryInput"]'); + if (await queryInput.isVisible()) { + await queryInput.fill(query); + // Submit query + const submitBtn = await page.locator('[data-test-subj="querySubmitButton"]'); + if (await submitBtn.isVisible()) { + await submitBtn.click(); + } + await page.waitForTimeout(5000); // wait for results + } + + check(page, { + 'Query executed': () => true, + }); + + // Change time range to last 24h + const timePicker = await page.locator('[data-test-subj="superDatePickerToggleQuickMenuButton"]'); + if (await timePicker.isVisible()) { + await timePicker.click(); + await page.waitForTimeout(1000); + const last24h = await page.locator('text=Last 24 hours'); + if (await last24h.isVisible()) { + await last24h.click(); + await page.waitForTimeout(3000); + } + } + + sleep(2); + } finally { + await page.close(); + await context.close(); + } +} diff --git a/load-testing/k6/scenarios/browser-metrics.js b/load-testing/k6/scenarios/browser-metrics.js new file mode 100644 index 00000000..2ef1af71 --- /dev/null +++ b/load-testing/k6/scenarios/browser-metrics.js @@ -0,0 +1,103 @@ +// k6 browser test — Metrics visualization flow in OpenSearch Dashboards. +// Simulates users viewing metric panels that query Prometheus (single pod). +// This is the scenario most likely to find the Prometheus breaking point. +// +// Usage: +// kubectl port-forward -n observability-stack svc/obs-stack-opensearch-dashboards 5601:5601 & +// K6_BROWSER_ENABLED=true k6 run scenarios/browser-metrics.js + +import { browser } from 'k6/browser'; +import { check, sleep } from 'k6'; + +const TARGET_BROWSER_VUS = parseInt(__ENV.BROWSER_VUS || '5'); + +export const options = { + scenarios: { + metrics_flow: { + executor: 'ramping-vus', + startVUs: 0, + stages: [ + { duration: '1m', target: 2 }, + { duration: '3m', target: TARGET_BROWSER_VUS }, + { duration: '3m', target: TARGET_BROWSER_VUS }, + { duration: '1m', target: 0 }, + ], + exec: 'metricsFlow', + options: { browser: { type: 'chromium' } }, + }, + }, + thresholds: { + browser_web_vital_lcp: ['p(95)<8000'], + }, +}; + +const DASHBOARDS_URL = __ENV.DASHBOARDS_URL || 'http://localhost:5601'; +const USERNAME = __ENV.OSD_USER || 'admin'; +const PASSWORD = __ENV.OSD_PASSWORD || 'My_password_123!@#'; + +async function login(page) { + await page.goto(`${DASHBOARDS_URL}/app/home`); + const userField = await page.locator('[data-test-subj="user-name"]'); + if (await userField.isVisible()) { + await userField.fill(USERNAME); + await page.locator('[data-test-subj="password"]').fill(PASSWORD); + await page.locator('[data-test-subj="submit"]').click(); + await page.waitForNavigation(); + } +} + +export async function metricsFlow() { + const context = await browser.newContext({ ignoreHTTPSErrors: true }); + const page = await context.newPage(); + + try { + await login(page); + + // Navigate to Observability > Metrics + await page.goto(`${DASHBOARDS_URL}/app/observability-metrics`); + await page.waitForTimeout(5000); + + check(page, { + 'Metrics page loaded': () => true, + }); + + // Try opening a saved dashboard (Pipeline Health or similar) + await page.goto(`${DASHBOARDS_URL}/app/dashboards`); + await page.waitForTimeout(3000); + + // Click first available dashboard + const dashboardLink = await page.locator('table tbody tr a').first(); + if (await dashboardLink.isVisible()) { + await dashboardLink.click(); + await page.waitForTimeout(8000); // dashboards with many panels take time + + check(page, { + 'Dashboard loaded': () => true, + }); + + // Change time range to 7d (stresses Prometheus) + const timePicker = await page.locator('[data-test-subj="superDatePickerToggleQuickMenuButton"]'); + if (await timePicker.isVisible()) { + await timePicker.click(); + await page.waitForTimeout(1000); + const last7d = await page.locator('text=Last 7 days'); + if (await last7d.isVisible()) { + await last7d.click(); + await page.waitForTimeout(10000); // 7d queries are expensive + } + } + + // Refresh the dashboard + const refreshBtn = await page.locator('[data-test-subj="querySubmitButton"]'); + if (await refreshBtn.isVisible()) { + await refreshBtn.click(); + await page.waitForTimeout(8000); + } + } + + sleep(2); + } finally { + await page.close(); + await context.close(); + } +} diff --git a/load-testing/k6/scenarios/browser-traces.js b/load-testing/k6/scenarios/browser-traces.js new file mode 100644 index 00000000..3d1c2566 --- /dev/null +++ b/load-testing/k6/scenarios/browser-traces.js @@ -0,0 +1,100 @@ +// k6 browser test — Trace Analytics flow in OpenSearch Dashboards. +// Simulates a user navigating to Traces, clicking into a trace, viewing spans. +// +// Usage: +// kubectl port-forward -n observability-stack svc/obs-stack-opensearch-dashboards 5601:5601 & +// K6_BROWSER_ENABLED=true k6 run scenarios/browser-traces.js + +import { browser } from 'k6/browser'; +import { check, sleep } from 'k6'; + +const TARGET_BROWSER_VUS = parseInt(__ENV.BROWSER_VUS || '5'); + +export const options = { + scenarios: { + traces_flow: { + executor: 'ramping-vus', + startVUs: 0, + stages: [ + { duration: '1m', target: 2 }, + { duration: '3m', target: TARGET_BROWSER_VUS }, + { duration: '3m', target: TARGET_BROWSER_VUS }, + { duration: '1m', target: 0 }, + ], + exec: 'tracesFlow', + options: { browser: { type: 'chromium' } }, + }, + }, + thresholds: { + browser_web_vital_lcp: ['p(95)<8000'], + browser_web_vital_cls: ['p(95)<0.25'], + }, +}; + +const DASHBOARDS_URL = __ENV.DASHBOARDS_URL || 'http://localhost:5601'; +const USERNAME = __ENV.OSD_USER || 'admin'; +const PASSWORD = __ENV.OSD_PASSWORD || 'My_password_123!@#'; + +async function login(page) { + await page.goto(`${DASHBOARDS_URL}/app/home`); + // Check if login page appears + const userField = await page.locator('[data-test-subj="user-name"]'); + if (await userField.isVisible()) { + await userField.fill(USERNAME); + await page.locator('[data-test-subj="password"]').fill(PASSWORD); + await page.locator('[data-test-subj="submit"]').click(); + await page.waitForNavigation(); + } +} + +export async function tracesFlow() { + const context = await browser.newContext({ ignoreHTTPSErrors: true }); + const page = await context.newPage(); + + try { + await login(page); + + // Navigate to Trace Analytics + await page.goto(`${DASHBOARDS_URL}/app/observability-traces#/traces`); + await page.waitForTimeout(3000); + + // Wait for trace table to load + const table = await page.locator('table'); + await table.waitFor({ state: 'visible', timeout: 15000 }); + + check(page, { + 'Traces page loaded': () => true, + }); + + // Click first trace row if available + const firstRow = await page.locator('table tbody tr').first(); + if (await firstRow.isVisible()) { + await firstRow.click(); + await page.waitForTimeout(5000); // wait for span waterfall + + check(page, { + 'Trace detail loaded': () => true, + }); + } + + // Navigate to Services + await page.goto(`${DASHBOARDS_URL}/app/observability-traces#/services`); + await page.waitForTimeout(3000); + + check(page, { + 'Services page loaded': () => true, + }); + + // Click first service if available + const serviceRow = await page.locator('table tbody tr').first(); + if (await serviceRow.isVisible()) { + await serviceRow.click(); + await page.waitForTimeout(5000); + } + + sleep(2); + } finally { + await page.close(); + await context.close(); + } +} diff --git a/load-testing/osb/index-settings.json b/load-testing/osb/index-settings.json new file mode 100644 index 00000000..46c218c5 --- /dev/null +++ b/load-testing/osb/index-settings.json @@ -0,0 +1,25 @@ +{ + "settings": { + "index": { + "number_of_shards": 1, + "number_of_replicas": 0 + } + }, + "mappings": { + "properties": { + "traceId": { "type": "keyword" }, + "spanId": { "type": "keyword" }, + "parentSpanId": { "type": "keyword" }, + "serviceName": { "type": "keyword" }, + "name": { "type": "keyword" }, + "kind": { "type": "keyword" }, + "startTime": { "type": "date_nanos" }, + "endTime": { "type": "date_nanos" }, + "durationInNanos": { "type": "long" }, + "status.code": { "type": "integer" }, + "resource.attributes.service@name": { "type": "keyword" }, + "resource.attributes.gen_ai@agent@name": { "type": "keyword" }, + "resource.attributes.gen_ai@operation@name": { "type": "keyword" } + } + } +} diff --git a/load-testing/osb/run-osb.sh b/load-testing/osb/run-osb.sh new file mode 100755 index 00000000..80f9e8a3 --- /dev/null +++ b/load-testing/osb/run-osb.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Run OpenSearch Benchmark against the cluster. +# Prerequisites: pip install opensearch-benchmark +# +# Usage: +# # Port-forward first: +# kubectl port-forward -n observability-stack svc/opensearch-cluster-master 9200:9200 +# +# ./run-osb.sh # standard benchmark +# ./run-osb.sh --redline-test # find breaking point automatically + +set -euo pipefail + +HOST="${OPENSEARCH_HOST:-https://localhost:9200}" +USER="${OPENSEARCH_USER:-admin}" +PASS="${OPENSEARCH_PASSWORD:-My_password_123!@#}" +WORKLOAD_DIR="$(cd "$(dirname "$0")" && pwd)" + +if [[ "${1:-}" == "--redline-test" ]]; then + echo "🔴 Running redline test (auto-ramp to breaking point)..." + opensearch-benchmark execute-test \ + --target-hosts="$HOST" \ + --pipeline=benchmark-only \ + --workload-path="$WORKLOAD_DIR" \ + --client-options="use_ssl:true,verify_certs:false,basic_auth_user:${USER},basic_auth_password:${PASS}" \ + --redline-test +else + echo "📊 Running standard benchmark..." + opensearch-benchmark execute-test \ + --target-hosts="$HOST" \ + --pipeline=benchmark-only \ + --workload-path="$WORKLOAD_DIR" \ + --client-options="use_ssl:true,verify_certs:false,basic_auth_user:${USER},basic_auth_password:${PASS}" +fi diff --git a/load-testing/osb/workload.json b/load-testing/osb/workload.json new file mode 100644 index 00000000..58eaa1a5 --- /dev/null +++ b/load-testing/osb/workload.json @@ -0,0 +1,99 @@ +{ + "short_description": "Observability Stack trace/log workload matching otel-v1-apm-span and logs-otel-v1 index shapes", + "description": "Custom workload for OpenSearch Benchmark that generates documents matching the OTLP trace and log index patterns used by the observability stack.", + "indices": [ + { + "name": "otel-v1-apm-span-loadtest", + "body": "index-settings.json" + } + ], + "operations": [ + { + "name": "bulk-index-traces", + "operation-type": "bulk", + "bulk-size": 500, + "index": "otel-v1-apm-span-loadtest" + }, + { + "name": "search-by-service", + "operation-type": "search", + "index": "otel-v1-apm-span-*", + "body": { + "size": 50, + "query": { + "term": { "serviceName": "load-test-traces" } + }, + "sort": [{ "startTime": "desc" }] + } + }, + { + "name": "agg-by-service", + "operation-type": "search", + "index": "otel-v1-apm-span-*", + "body": { + "size": 0, + "aggs": { + "services": { + "terms": { "field": "serviceName", "size": 50 }, + "aggs": { + "avg_duration": { "avg": { "field": "durationInNanos" } }, + "error_count": { + "filter": { "term": { "status.code": 2 } } + } + } + } + } + } + }, + { + "name": "service-map-query", + "operation-type": "search", + "index": "otel-v2-apm-service-map-*", + "body": { + "size": 200, + "query": { "match_all": {} } + } + }, + { + "name": "search-logs", + "operation-type": "search", + "index": "logs-otel-v1-*", + "body": { + "size": 50, + "query": { "match_all": {} }, + "sort": [{ "time": "desc" }] + } + } + ], + "schedule": [ + { + "operation": "bulk-index-traces", + "warmup-time-period": 60, + "clients": 4 + }, + { + "operation": "search-by-service", + "warmup-iterations": 50, + "iterations": 200, + "clients": 8 + }, + { + "operation": "agg-by-service", + "warmup-iterations": 20, + "iterations": 100, + "clients": 8 + }, + { + "operation": "service-map-query", + "warmup-iterations": 20, + "iterations": 100, + "clients": 4 + }, + { + "operation": "search-logs", + "warmup-iterations": 50, + "iterations": 200, + "clients": 8 + } + ] +} diff --git a/load-testing/pipeline/run-telemetrygen.sh b/load-testing/pipeline/run-telemetrygen.sh new file mode 100755 index 00000000..463d09b1 --- /dev/null +++ b/load-testing/pipeline/run-telemetrygen.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Pipeline throughput test — ramps telemetrygen rate to find breaking point. +# Prerequisites: telemetrygen installed +# go install github.com/open-telemetry/opentelemetry-collector-contrib/cmd/telemetrygen@latest +# +# Usage: +# # Port-forward first: +# kubectl port-forward -n observability-stack svc/obs-stack-opentelemetry-collector 4317:4317 +# +# ./run-telemetrygen.sh # default: traces +# ./run-telemetrygen.sh logs # test log pipeline +# ./run-telemetrygen.sh metrics # test metrics pipeline +# ./run-telemetrygen.sh all # test all three + +set -euo pipefail + +ENDPOINT="${OTEL_ENDPOINT:-localhost:4317}" +DURATION="${DURATION:-3m}" +SIGNAL="${1:-traces}" +RATES=(50 100 250 500 1000 2500 5000) + +run_step() { + local signal="$1" rate="$2" + echo "" + echo "==========================================" + echo " ${signal} @ ${rate}/sec for ${DURATION}" + echo "==========================================" + telemetrygen "$signal" \ + --otlp-endpoint="$ENDPOINT" \ + --otlp-insecure \ + --rate="$rate" \ + --duration="$DURATION" \ + --service="load-test-${signal}" \ + --otlp-attributes='gen_ai.agent.name="load-test"' \ + 2>&1 | tail -5 + echo "--- Sleeping 30s before next step ---" + sleep 30 +} + +run_signal() { + local signal="$1" + echo "" + echo "############################################" + echo " Starting ${signal} ramp test" + echo " Endpoint: ${ENDPOINT}" + echo " Duration per step: ${DURATION}" + echo " Rates: ${RATES[*]}" + echo "############################################" + for rate in "${RATES[@]}"; do + run_step "$signal" "$rate" + done + echo "" + echo "✅ ${signal} ramp complete" +} + +case "$SIGNAL" in + all) + for s in traces logs metrics; do run_signal "$s"; done + ;; + traces|logs|metrics) + run_signal "$SIGNAL" + ;; + *) + echo "Usage: $0 [traces|logs|metrics|all]" + exit 1 + ;; +esac From b0ac3f66c85346f4a0766d4f671af77147e4965a Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 20 Mar 2026 12:40:22 -0700 Subject: [PATCH 41/64] feat: add weather/events agent URLs to canary for trace variety Matches upstream canary changes (shallow/normal/deep trace shapes). --- charts/observability-stack/templates/examples.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/charts/observability-stack/templates/examples.yaml b/charts/observability-stack/templates/examples.yaml index 69c4e207..9499dce1 100644 --- a/charts/observability-stack/templates/examples.yaml +++ b/charts/observability-stack/templates/examples.yaml @@ -197,6 +197,10 @@ spec: env: - name: TRAVEL_PLANNER_URL value: "http://travel-planner:8000" + - name: WEATHER_AGENT_URL + value: "http://weather-agent:8000" + - name: EVENTS_AGENT_URL + value: "http://events-agent:8002" - name: CANARY_INTERVAL value: {{ .Values.examples.canary.interval | default "120" | quote }} resources: From 43e2e981cf094d1da60be091cdd9bdbef18ec304 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 20 Mar 2026 12:53:28 -0700 Subject: [PATCH 42/64] feat: add anonymous authentication support to Helm chart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port anonymous auth from docker-compose to Helm/K8s deployment. Closes #5. Changes: - Add anonymousAuth.enabled toggle in values.yaml (default: false) - Create opensearch-security-config Secret with config.yml, roles.yml, roles_mapping.yml — anonymous_auth_enabled templated from values - Update OpenSearch Dashboards config with anonymous_auth_enabled and conditional savedObjects.permission.enabled via global values + tpl - Sync init script with docker-compose version (ANONYMOUS_AUTH_ENABLED env var, conditional anonymous role in workspace allowedRoles) - Pass OPENSEARCH_ANONYMOUS_AUTH_ENABLED env var to init-dashboards Job - Wire up Terraform anonymous_auth variable to Helm release - Add 6 helm-unittest tests covering both enabled/disabled states - Document usage in chart README Usage: helm install obs-stack charts/observability-stack \ --set anonymousAuth.enabled=true \ --set global.anonymousAuth.enabled=true Kiro/claude on behalf of @kylehounslow --- charts/observability-stack/README.md | 28 +++++ .../files/init-opensearch-dashboards.py | 3 +- .../templates/init-dashboards-job.yaml | 2 + .../templates/opensearch-security-config.yaml | 103 ++++++++++++++++++ .../tests/anonymous_auth_test.yaml | 58 ++++++++++ charts/observability-stack/values.yaml | 23 ++++ terraform/aws/observability-stack.tf | 16 +++ 7 files changed, 232 insertions(+), 1 deletion(-) create mode 100644 charts/observability-stack/templates/opensearch-security-config.yaml create mode 100644 charts/observability-stack/tests/anonymous_auth_test.yaml diff --git a/charts/observability-stack/README.md b/charts/observability-stack/README.md index d46c830c..c99fd644 100644 --- a/charts/observability-stack/README.md +++ b/charts/observability-stack/README.md @@ -162,6 +162,10 @@ Sources: [OpenSearch shard sizing](https://opensearch.org/blog/optimize-opensear See `values.yaml` for all options. Notable settings: ```yaml +# Anonymous auth — skip login page for demos/workshops +anonymousAuth: + enabled: false # Set true to allow access without credentials + # Credentials (update opensearchPassword before any real deployment) opensearchUsername: "admin" opensearchPassword: "My_password_123!@#" @@ -179,6 +183,30 @@ prometheus: # ... etc ``` +## Anonymous Authentication + +By default, OpenSearch Dashboards requires login. Enable anonymous auth to skip the login page — useful for demos, workshops, or shared dev environments. + +```bash +helm install obs-stack charts/observability-stack \ + --set anonymousAuth.enabled=true \ + --set global.anonymousAuth.enabled=true +``` + +> **Note:** Both `anonymousAuth.enabled` and `global.anonymousAuth.enabled` must be set. The `global` value is needed because the OpenSearch Dashboards subchart config uses Go templating and can only access global values. + +**What anonymous users can do:** +- Browse all data (logs, traces, metrics) +- View, create, and modify saved objects (visualizations, dashboards, saved queries) +- Explore traces and service maps +- Run queries and access the OpenSearch REST API + +**What anonymous users cannot do:** +- Delete existing saved objects +- Perform admin operations (user management, security config) + +**To disable:** Remove the `--set` flags (or set both to `false`) and redeploy. + ## OpenTelemetry Demo (Optional) The [OpenTelemetry Demo](https://opentelemetry.io/docs/demo/) is available as an optional subchart. It deploys a full microservices e-commerce app (20+ services) that generates realistic telemetry — useful for load testing and showcasing the stack. diff --git a/charts/observability-stack/files/init-opensearch-dashboards.py b/charts/observability-stack/files/init-opensearch-dashboards.py index 6f32b98d..ecefd6f5 100644 --- a/charts/observability-stack/files/init-opensearch-dashboards.py +++ b/charts/observability-stack/files/init-opensearch-dashboards.py @@ -15,6 +15,7 @@ PROMETHEUS_PORT = os.getenv("PROMETHEUS_PORT", "9090") _opensearch_protocol = os.getenv("OPENSEARCH_PROTOCOL", "https") OPENSEARCH_ENDPOINT = f"{_opensearch_protocol}://{os.getenv('OPENSEARCH_HOST', 'opensearch')}:{os.getenv('OPENSEARCH_PORT', '9200')}" +ANONYMOUS_AUTH_ENABLED = os.getenv("OPENSEARCH_ANONYMOUS_AUTH_ENABLED", "false").lower() == "true" def wait_for_dashboards(): """Wait for OpenSearch Dashboards to be ready""" @@ -232,7 +233,7 @@ def create_prometheus_datasource(workspace_id): payload = { "name": datasource_name, - "allowedRoles": [], + "allowedRoles": ["all_access", "opendistro_security_anonymous_role"] if ANONYMOUS_AUTH_ENABLED else ["all_access"], "connector": "prometheus", "properties": { "prometheus.uri": prometheus_endpoint, diff --git a/charts/observability-stack/templates/init-dashboards-job.yaml b/charts/observability-stack/templates/init-dashboards-job.yaml index 3c957106..6b646b3b 100644 --- a/charts/observability-stack/templates/init-dashboards-job.yaml +++ b/charts/observability-stack/templates/init-dashboards-job.yaml @@ -44,6 +44,8 @@ spec: value: "80" - name: OPENSEARCH_ENDPOINT value: "https://opensearch-cluster-master:9200" + - name: OPENSEARCH_ANONYMOUS_AUTH_ENABLED + value: {{ .Values.anonymousAuth.enabled | quote }} volumeMounts: - name: init-script mountPath: /scripts diff --git a/charts/observability-stack/templates/opensearch-security-config.yaml b/charts/observability-stack/templates/opensearch-security-config.yaml new file mode 100644 index 00000000..6c42faff --- /dev/null +++ b/charts/observability-stack/templates/opensearch-security-config.yaml @@ -0,0 +1,103 @@ +{{- if index .Values "opensearch" "enabled" }} +apiVersion: v1 +kind: Secret +metadata: + name: opensearch-security-config + labels: + {{- include "observability-stack.labels" . | nindent 4 }} +type: Opaque +stringData: + config.yml: | + _meta: + type: "config" + config_version: 2 + config: + dynamic: + http: + anonymous_auth_enabled: {{ .Values.anonymousAuth.enabled }} + xff: + enabled: false + internalProxies: "192\\.168\\.0\\.10|192\\.168\\.0\\.11" + authc: + basic_internal_auth_domain: + description: "Authenticate via HTTP Basic against internal users database" + http_enabled: true + transport_enabled: true + order: 4 + http_authenticator: + type: "basic" + challenge: true + authentication_backend: + type: "intern" + roles.yml: | + _meta: + type: "roles" + config_version: 2 + opendistro_security_anonymous_role: + reserved: true + cluster_permissions: + - "read" + - "cluster_monitor" + - "cluster_composite_ops" + - "indices:data/read/scroll*" + - "cluster:admin/opensearch/ppl" + - "cluster:admin/opensearch/sql" + - "cluster:admin/opensearch/ql/datasources/read" + - "cluster:admin/opensearch/ql/async_query/read" + - "cluster:admin/opensearch/direct_query/read/query" + index_permissions: + - index_patterns: + - ".kibana" + - ".kibana-6" + - ".kibana_*" + - ".opensearch_dashboards" + - ".opensearch_dashboards-6" + - ".opensearch_dashboards_*" + allowed_actions: + - "read" + - "indices:data/write/index*" + - "indices:data/write/update*" + - "indices:data/write/bulk*" + - index_patterns: + - ".tasks" + - ".management-beats" + - "*:.tasks" + - "*:.management-beats" + allowed_actions: + - "read" + - index_patterns: + - '*' + allowed_actions: + - "read" + - "indices:data/read/*" + - "indices:admin/get" + - "indices:admin/exists" + - "indices:admin/aliases/exists*" + - "indices:admin/aliases/get*" + - "indices:admin/mappings/get" + - "indices:admin/resolve/index" + - "indices:monitor/settings/get" + - "indices:monitor/stats" + tenant_permissions: + - tenant_patterns: + - '*' + allowed_actions: + - "kibana_all_write" + roles_mapping.yml: | + _meta: + type: "rolesmapping" + config_version: 2 + opendistro_security_anonymous_role: + backend_roles: + - "opendistro_security_anonymous_backendrole" + all_access: + reserved: true + backend_roles: + - "admin" + description: "Maps admin to all_access" + kibana_server: + reserved: true + users: + - "kibanaserver" + description: "Maps kibana_server role to kibanaserver user" +{{- end }} diff --git a/charts/observability-stack/tests/anonymous_auth_test.yaml b/charts/observability-stack/tests/anonymous_auth_test.yaml new file mode 100644 index 00000000..2c1e7f94 --- /dev/null +++ b/charts/observability-stack/tests/anonymous_auth_test.yaml @@ -0,0 +1,58 @@ +suite: anonymous authentication +templates: + - templates/opensearch-security-config.yaml + - templates/init-dashboards-job.yaml +tests: + # --- Security config Secret --- + - it: should set anonymous_auth_enabled to false by default + template: templates/opensearch-security-config.yaml + asserts: + - isKind: + of: Secret + - matchRegex: + path: stringData["config.yml"] + pattern: "anonymous_auth_enabled: false" + + - it: should set anonymous_auth_enabled to true when enabled + template: templates/opensearch-security-config.yaml + set: + anonymousAuth.enabled: true + asserts: + - matchRegex: + path: stringData["config.yml"] + pattern: "anonymous_auth_enabled: true" + + - it: should always include anonymous role definition + template: templates/opensearch-security-config.yaml + asserts: + - matchRegex: + path: stringData["roles.yml"] + pattern: "opendistro_security_anonymous_role" + + - it: should always include anonymous role mapping + template: templates/opensearch-security-config.yaml + asserts: + - matchRegex: + path: stringData["roles_mapping.yml"] + pattern: "opendistro_security_anonymous_backendrole" + + # --- Init job env var --- + - it: should pass OPENSEARCH_ANONYMOUS_AUTH_ENABLED=false by default + template: templates/init-dashboards-job.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSEARCH_ANONYMOUS_AUTH_ENABLED + value: "false" + + - it: should pass OPENSEARCH_ANONYMOUS_AUTH_ENABLED=true when enabled + template: templates/init-dashboards-job.yaml + set: + anonymousAuth.enabled: true + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSEARCH_ANONYMOUS_AUTH_ENABLED + value: "true" diff --git a/charts/observability-stack/values.yaml b/charts/observability-stack/values.yaml index 090c58f8..f9ca686c 100644 --- a/charts/observability-stack/values.yaml +++ b/charts/observability-stack/values.yaml @@ -1,6 +1,13 @@ # Default values for observability-stack umbrella chart # Mirrors the docker-compose setup for Kubernetes deployment +# -- Anonymous authentication (skip login page for demos/workshops) +# When enabled, users can access OpenSearch Dashboards without logging in. +# Anonymous users can browse data, create/modify saved objects, but cannot +# delete existing saved objects or perform admin operations. +anonymousAuth: + enabled: false + # -- OpenSearch # Sizing guide: # Storage: daily_ingest_GB × 1.45 × (replicas + 1) × retention_days @@ -32,6 +39,9 @@ opensearch: config: opensearch.yml: | plugins.query.datasources.encryption.masterkey: "BTqK4Ytdz67La1kShIKV3Pu9" + securityConfig: + config: + securityConfigSecret: "opensearch-security-config" # -- OpenSearch Dashboards opensearch-dashboards: @@ -53,6 +63,7 @@ opensearch-dashboards: opensearch.requestHeadersAllowlist: ["authorization", "securitytenant"] opensearch_security.multitenancy.enabled: false opensearch_security.readonly_mode.roles: ["kibana_read_only"] + opensearch_security.auth.anonymous_auth_enabled: {{ .Values.global.anonymousAuth.enabled }} console.enabled: true server.maxPayloadBytes: 1048576 savedObjects.maxImportPayloadBytes: 26214400 @@ -62,6 +73,11 @@ opensearch-dashboards: explore.discoverMetrics.enabled: true explore.agentTraces.enabled: true workspace.enabled: true + {{- if .Values.global.anonymousAuth.enabled }} + savedObjects.permission.enabled: false + {{- else }} + savedObjects.permission.enabled: true + {{- end }} data_source.enabled: true data_source.ssl.verificationMode: none datasetManagement.enabled: true @@ -471,3 +487,10 @@ gateway: # host: dashboards.example.com # annotations: # application-networking.k8s.aws/certificate-arn: arn:aws:acm:REGION:ACCOUNT:certificate/ID + +# -- Global values (accessible to all subcharts via .Values.global) +# Used to pass anonymousAuth.enabled to opensearch-dashboards subchart config +# which uses tpl() for Go template rendering. +global: + anonymousAuth: + enabled: false diff --git a/terraform/aws/observability-stack.tf b/terraform/aws/observability-stack.tf index cc4e20fc..236c15d9 100644 --- a/terraform/aws/observability-stack.tf +++ b/terraform/aws/observability-stack.tf @@ -163,6 +163,22 @@ resource "helm_release" "observability_stack" { } } + # --- Anonymous auth (conditional) --- + dynamic "set" { + for_each = var.anonymous_auth ? [1] : [] + content { + name = "anonymousAuth.enabled" + value = "true" + } + } + dynamic "set" { + for_each = var.anonymous_auth ? [1] : [] + content { + name = "global.anonymousAuth.enabled" + value = "true" + } + } + depends_on = [ helm_release.aws_lb_controller, ] From a59695de11735dc5b7ad25c3908bb3579c8b3b09 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 20 Mar 2026 13:13:42 -0700 Subject: [PATCH 43/64] =?UTF-8?q?feat:=20load=20test=20results=20=E2=80=94?= =?UTF-8?q?=20300=20VUs=20clean,=201500=20VUs=20shows=20p95=3D2.3s=20degra?= =?UTF-8?q?dation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fixed k6 auth (manual Base64 header) and PPL query syntax - Test 002: 300 VUs, 0% errors, p95=16ms — no stress - Test 003: 1500 VUs, 0% errors, p95=2.28s — saturated - Breaking point estimated between 500-700 VUs for good UX --- load-testing/RESULTS.md | 36 +++++++++++++ load-testing/k6/full-test.js | 2 +- load-testing/k6/scenarios/api-queries.js | 23 ++++---- .../results/001-api-queries-auth-bug.md | 48 +++++++++++++++++ load-testing/results/002-api-queries.md | 53 +++++++++++++++++++ .../results/003-api-queries-1500vu.md | 53 +++++++++++++++++++ 6 files changed, 205 insertions(+), 10 deletions(-) create mode 100644 load-testing/RESULTS.md create mode 100644 load-testing/results/001-api-queries-auth-bug.md create mode 100644 load-testing/results/002-api-queries.md create mode 100644 load-testing/results/003-api-queries-1500vu.md diff --git a/load-testing/RESULTS.md b/load-testing/RESULTS.md new file mode 100644 index 00000000..8780327c --- /dev/null +++ b/load-testing/RESULTS.md @@ -0,0 +1,36 @@ +# Load Test Results + +## Baseline Topology (2026-03-20) + +### Cluster +- EKS cluster: `observability-stack`, us-west-2 +- Kubernetes: 1.32 +- Nodes: 2x `m5.xlarge` (4 vCPU, 16 GB RAM each = 8 vCPU / 32 GB total) + +### Core Stack Pods +| Component | Replicas | CPU Req | Mem Req | Node | +|-----------|----------|---------|---------|------| +| OpenSearch | 1 (single-node) | 500m | 2Gi (JVM: 1g) | node-1 | +| OpenSearch Dashboards | 1 | 100m | 512M | node-2 | +| OTel Collector | 1 | none | none | node-2 | +| Data Prepper | 2 | none | none | node-1, node-2 | +| Prometheus | 1 (no PV) | none | none | node-2 | + +### OpenSearch State +- Cluster status: yellow (3 unassigned replica shards — expected with single node) +- Active shards: 14 primary +- Indices: `otel-v1-apm-span-000001` (111k docs, 57MB), `logs-otel-v1-000001` (7k docs, 12MB), `otel-v2-apm-service-map-000001` (14k docs, 2.8MB) + +### Background Load +- OTel Demo: ~20 microservices generating traces/logs/metrics via built-in load generator +- Example agents: travel-planner, weather-agent, events-agent, canary + +--- + +## Test Results + +| # | Test | Date | Status | Result File | +|---|------|------|--------|-------------| +| 1 | API Query Load (200 OS VUs + 100 Prom VUs) | 2026-03-20 12:04–12:19 | ⚠️ Auth bug | [001-api-queries-auth-bug.md](results/001-api-queries-auth-bug.md) | +| 2 | API Query Load (300 VUs, auth fixed) | 2026-03-20 12:42–12:57 | ✅ 0% errors, p95=16ms | [002-api-queries.md](results/002-api-queries.md) | +| 3 | API Query Load (1500 VUs) | 2026-03-20 12:57–13:12 | ⚠️ p95=2.28s, 0% errors | [003-api-queries-1500vu.md](results/003-api-queries-1500vu.md) | diff --git a/load-testing/k6/full-test.js b/load-testing/k6/full-test.js index 2e7caa02..d48d90ac 100644 --- a/load-testing/k6/full-test.js +++ b/load-testing/k6/full-test.js @@ -18,6 +18,7 @@ const TARGET_VUS = parseInt(__ENV.TARGET_VUS || '150'); const BROWSER_VUS = parseInt(__ENV.BROWSER_VUS || '5'); export const options = { + insecureSkipTLSVerify: true, scenarios: { // --- API layer: OpenSearch queries --- api_opensearch: { @@ -90,7 +91,6 @@ const osParams = { auth: 'basic', username: USERNAME, password: PASSWORD, - insecureSkipTLSVerify: true, }; const pplQueries = [ diff --git a/load-testing/k6/scenarios/api-queries.js b/load-testing/k6/scenarios/api-queries.js index a61fe90a..8ef49e4f 100644 --- a/load-testing/k6/scenarios/api-queries.js +++ b/load-testing/k6/scenarios/api-queries.js @@ -17,6 +17,7 @@ import { Rate, Trend } from 'k6/metrics'; const TARGET_VUS = parseInt(__ENV.TARGET_VUS || '200'); export const options = { + insecureSkipTLSVerify: true, scenarios: { opensearch_queries: { executor: 'ramping-vus', @@ -51,24 +52,28 @@ export const options = { const OS_BASE = __ENV.OPENSEARCH_URL || 'https://localhost:9200'; const PROM_BASE = __ENV.PROMETHEUS_URL || 'http://localhost:9090'; -const OS_AUTH = { username: 'admin', password: 'My_password_123!@#' }; + +import encoding from 'k6/encoding'; +const OS_USER = __ENV.OSD_USER || 'admin'; +const OS_PASS = __ENV.OSD_PASSWORD || 'My_password_123!@#'; +const OS_AUTH_HEADER = `Basic ${encoding.b64encode(`${OS_USER}:${OS_PASS}`)}`; const osParams = { - headers: { 'Content-Type': 'application/json' }, - auth: 'basic', - ...OS_AUTH, - insecureSkipTLSVerify: true, + headers: { + 'Content-Type': 'application/json', + 'Authorization': OS_AUTH_HEADER, + }, }; // --- PPL queries (light → heavy) --- const pplQueries = [ 'source=otel-v1-apm-span-000001 | head 50', 'source=otel-v1-apm-span-000001 | stats count() by serviceName', - 'source=otel-v1-apm-span-000001 | where serviceName="frontend" | stats avg(durationInNanos) by name', - 'source=otel-v1-apm-span-000001 | stats count() by serviceName, name | sort - count()', - 'source=otel-v1-apm-span-000001 | where durationInNanos > 1000000000 | stats count() by serviceName | sort - count()', + 'source=otel-v1-apm-span-000001 | where serviceName="frontend" | stats avg(durationInNanos)', + 'source=otel-v1-apm-span-000001 | stats count() by serviceName, kind', + 'source=otel-v1-apm-span-000001 | where durationInNanos > 1000000000 | stats count() by serviceName', 'source=logs-otel-v1-000001 | head 50', - 'source=logs-otel-v1-000001 | stats count() by serviceName', + 'source=logs-otel-v1-000001 | stats count() by severityText', ]; // --- PromQL queries (light → heavy) --- diff --git a/load-testing/results/001-api-queries-auth-bug.md b/load-testing/results/001-api-queries-auth-bug.md new file mode 100644 index 00000000..59524964 --- /dev/null +++ b/load-testing/results/001-api-queries-auth-bug.md @@ -0,0 +1,48 @@ +# Test 001: API Query Load (Auth Bug) + +**Status:** ⚠️ Invalid — OpenSearch auth failure +**Script:** `k6/scenarios/api-queries.js` +**Start:** 2026-03-20 12:04:00 PDT +**End:** 2026-03-20 12:19:01 PDT +**Duration:** 15m01s + +## Parameters +- OpenSearch VUs: 200 (ramping) +- Prometheus VUs: 100 (ramping) +- Total peak VUs: 300 +- Background: OTel Demo load generator active + +## Summary + +| Metric | Value | +|--------|-------| +| Total iterations | 81,422 | +| Total HTTP requests | 217,392 | +| Requests/sec | 241 | +| http_req_duration p(50) | 1.12ms | +| http_req_duration p(90) | 2.47ms | +| http_req_duration p(95) | 3.45ms | +| http_req_duration max | 139.92ms | +| http_req_failed | 75.27% | + +## Per-Check Breakdown + +| Check | Pass | Fail | Rate | Notes | +|-------|------|------|------|-------| +| PPL 2xx | 0 | 54,548 | 0% | Auth failure | +| Search 2xx | 0 | 54,548 | 0% | Auth failure | +| ServiceMap 2xx | 0 | 54,548 | 0% | Auth failure | +| PromQL instant 2xx | 26,874 | 0 | 100% | ✅ | +| PromQL range 2xx | 26,874 | 0 | 100% | ✅ | + +## Root Cause + +k6's `auth: 'basic'` parameter with spread `...OS_AUTH` does not work as expected. OpenSearch returned immediate rejections (avg 1.6ms response = 401/403, not timeout). All OpenSearch data points are invalid. + +## Valid Takeaway + +Prometheus (single pod, no resource limits) handled 100 concurrent VUs doing instant + range PromQL queries with 100% success and p95 of 3.45ms. Not stressed at this level. + +## Fix Applied + +Switched to manual `Authorization: Basic ` header using `k6/encoding` module. diff --git a/load-testing/results/002-api-queries.md b/load-testing/results/002-api-queries.md new file mode 100644 index 00000000..9af6724d --- /dev/null +++ b/load-testing/results/002-api-queries.md @@ -0,0 +1,53 @@ +# Test 002: API Query Load (Auth Fixed) + +**Status:** ✅ Passed — no failures +**Script:** `k6/scenarios/api-queries.js` +**Start:** 2026-03-20 12:42:14 PDT (19:42:14 UTC) +**End:** 2026-03-20 12:57:16 PDT (19:57:16 UTC) +**Duration:** 15m02s + +## Parameters +- OpenSearch VUs: 200 (ramping 0→50→100→200, hold 3m, ramp down) +- Prometheus VUs: 100 (ramping 0→25→50→100, hold 3m, ramp down) +- Total peak VUs: 300 +- Background: OTel Demo load generator active + +## Summary + +| Metric | Value | +|--------|-------| +| Total iterations | 80,879 | +| Total HTTP requests | 215,779 | +| Requests/sec | 239 | +| http_req_duration p(50) | 5.93ms | +| http_req_duration p(90) | 12.34ms | +| http_req_duration p(95) | 16.15ms | +| http_req_duration max | 543.18ms | +| http_req_failed | 0.00% | +| Data received | 11 GB (12 MB/s) | + +## Per-Check Breakdown + +| Check | Pass | Fail | Rate | +|-------|------|------|------| +| PPL 2xx | 54,148 | 0 | 100% ✅ | +| Search 2xx | 54,148 | 0 | 100% ✅ | +| ServiceMap 2xx | 54,148 | 0 | 100% ✅ | +| PromQL instant 2xx | 26,667 | 0 | 100% ✅ | +| PromQL range 2xx | 26,668 | 0 | 100% ✅ | + +## Analysis + +**The stack did not break at 300 concurrent API VUs (200 OpenSearch + 100 Prometheus).** + +- All thresholds passed: p95 latency 16.15ms (well under 5s threshold), 0% error rate +- OpenSearch single node handled 200 concurrent PPL + _search + service map queries without degradation +- Prometheus single pod handled 100 concurrent instant + range PromQL queries at p95 of ~16ms +- Max latency spike was 543ms — a single outlier, not sustained degradation +- Throughput was steady at ~239 req/s throughout the hold period + +**Conclusion:** 300 VUs is not the breaking point. Need to ramp significantly higher (500–1000+ VUs) to find where things start to degrade. + +## Next Steps +- Run Test 003 with TARGET_VUS=500 (750 total VUs) to push harder +- If that holds, jump to TARGET_VUS=1000 diff --git a/load-testing/results/003-api-queries-1500vu.md b/load-testing/results/003-api-queries-1500vu.md new file mode 100644 index 00000000..d1b9595b --- /dev/null +++ b/load-testing/results/003-api-queries-1500vu.md @@ -0,0 +1,53 @@ +# Test 003: API Query Load — 1500 VUs (Finding the Ceiling) + +**Status:** ⚠️ Passed but showing significant latency degradation +**Script:** `k6/scenarios/api-queries.js` +**Start:** 2026-03-20 12:57:50 PDT (19:57:50 UTC) +**End:** 2026-03-20 13:12:53 PDT (20:12:53 UTC) +**Duration:** 15m03s + +## Parameters +- OpenSearch VUs: 1000 (ramping) +- Prometheus VUs: 500 (ramping) +- Total peak VUs: 1500 +- Background: OTel Demo load generator active + +## Summary + +| Metric | Test 002 (300 VUs) | Test 003 (1500 VUs) | Delta | +|--------|-------------------|---------------------|-------| +| Total requests | 215,779 | 771,403 | 3.6x | +| Requests/sec | 239 | 855 | 3.6x | +| http_req_duration p(50) | 5.93ms | 13.67ms | 2.3x ⚠️ | +| http_req_duration p(90) | 12.34ms | 1.04s | 84x 🔴 | +| http_req_duration p(95) | 16.15ms | 2.28s | 141x 🔴 | +| http_req_duration max | 543ms | 5.49s | 10x 🔴 | +| http_req_failed | 0.00% | 0.00% | — | +| Data received | 11 GB | 34 GB | 3x | + +## Analysis + +**No errors, but the system is clearly saturated at 1500 VUs.** + +- p50 only doubled (5.9ms → 13.7ms) — the median request is still fast +- p90 exploded from 12ms to 1.04s — the tail is getting crushed +- p95 hit 2.28s — approaching the 5s threshold +- Max latency hit 5.49s — individual requests are timing out +- Iteration duration p95 hit 5.16s (vs 2.91s at 300 VUs) + +**The breaking point is between 300 and 1500 VUs.** The system handles the load without errors but latency degrades severely. At 1500 VUs, 10% of requests take over 1 second and 5% take over 2.3 seconds. + +**For a dashboard user experience**, p95 > 2s means the UI feels sluggish. A reasonable "usable" threshold would be p95 < 500ms, which was comfortably met at 300 VUs but blown past at 1500. + +## Estimated Capacity + +| User Experience | Estimated Max VUs | p95 Latency | +|----------------|-------------------|-------------| +| Excellent (< 100ms p95) | ~300 | 16ms | +| Good (< 500ms p95) | ~500-700 (estimated) | ~500ms | +| Degraded (< 2s p95) | ~1200-1500 | ~2s | +| Broken (errors appear) | > 1500 (not yet found) | > 5s | + +## Next Steps +- Run Test 004 at TARGET_VUS=500 to find the "good experience" ceiling +- Or jump to TARGET_VUS=2000 to find where errors actually start From 97c67d7830c395ac895b73a898120e5276f47c92 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 20 Mar 2026 13:28:28 -0700 Subject: [PATCH 44/64] feat: add self-monitoring saved queries + capacity planning dashboard panels - New saved-queries-self-monitoring.yaml: thread pool rejections, search latency, Prometheus query latency P99, Data Prepper buffer capacity, OTel dropped spans - OpenSearch Health dashboard: added thread pool rejections, active searches, search latency, fetch rate panels - Pipeline Health dashboard: added Prometheus range query latency P99 panel - init-dashboards-configmap: include new saved queries file --- .../files/dashboard-opensearch-health.yaml | 22 +++++ .../files/dashboard-pipeline-health.yaml | 30 ++++++ .../files/saved-queries-self-monitoring.yaml | 91 +++++++++++++++++++ .../templates/init-dashboards-configmap.yaml | 2 + 4 files changed, 145 insertions(+) create mode 100644 charts/observability-stack/files/saved-queries-self-monitoring.yaml diff --git a/charts/observability-stack/files/dashboard-opensearch-health.yaml b/charts/observability-stack/files/dashboard-opensearch-health.yaml index cc507285..42fd6695 100644 --- a/charts/observability-stack/files/dashboard-opensearch-health.yaml +++ b/charts/observability-stack/files/dashboard-opensearch-health.yaml @@ -60,3 +60,25 @@ panels: title: "OpenSearch CPU %" query: "elasticsearch_os_cpu_percent" chartType: line + + # --- Row 6: Query Pressure (Load Testing) --- + - id: os-thread-pool-rejections + title: "Thread Pool Rejections/sec" + query: "rate(elasticsearch_thread_pool_rejected_count[5m])" + chartType: line + + - id: os-active-searches + title: "Active Searches" + query: "elasticsearch_indices_search_query_current" + chartType: line + + # --- Row 7: Search Latency & Fetch --- + - id: os-search-latency + title: "Search Latency (sec/query)" + query: "rate(elasticsearch_indices_search_query_time_seconds_total[5m]) / rate(elasticsearch_indices_search_query_total[5m])" + chartType: line + + - id: os-search-fetch-rate + title: "Search Fetch Rate (fetches/sec)" + query: "rate(elasticsearch_indices_search_fetch_total[5m])" + chartType: line diff --git a/charts/observability-stack/files/dashboard-pipeline-health.yaml b/charts/observability-stack/files/dashboard-pipeline-health.yaml index 1fdd2ea3..1932fda9 100644 --- a/charts/observability-stack/files/dashboard-pipeline-health.yaml +++ b/charts/observability-stack/files/dashboard-pipeline-health.yaml @@ -1,112 +1,142 @@ # Observability Pipeline Health Dashboard — OTel Collector and Prometheus self-monitoring + dashboard: id: observability-pipeline-health-dashboard title: Observability Pipeline Health description: OTel Collector throughput, Prometheus ingestion, and pipeline health + panels: # --- Row 1: OTel Collector Throughput --- - id: pipeline-otel-spans-received title: "OTel Spans Received/sec" query: "rate(otelcol_receiver_accepted_spans_total[5m])" chartType: line + - id: pipeline-otel-spans-exported title: "OTel Spans Exported/sec" query: "rate(otelcol_exporter_sent_spans_total[5m])" chartType: line + # --- Row 2: OTel Metrics & Failures --- - id: pipeline-otel-metrics-received title: "OTel Metrics Received/sec" query: "rate(otelcol_receiver_accepted_metric_points_total[5m])" chartType: line + - id: pipeline-otel-spans-dropped title: "OTel Spans Dropped/sec" query: "rate(otelcol_exporter_send_failed_spans_total[5m])" chartType: line + # --- Row 3: OTel Collector Resources --- - id: pipeline-otel-queue-size title: "OTel Exporter Queue Size" query: "otelcol_exporter_queue_size" chartType: line + - id: pipeline-otel-collector-memory title: "OTel Collector Memory (bytes)" query: "otelcol_process_memory_rss_bytes" chartType: line + # --- Row 4: OTel Collector CPU & Uptime --- - id: pipeline-otel-collector-cpu title: "OTel Collector CPU Usage" query: "rate(otelcol_process_cpu_seconds_total[5m])" chartType: line + - id: pipeline-otel-batch-cardinality title: "OTel Batch Metadata Cardinality" query: "otelcol_processor_batch_metadata_cardinality" chartType: line + # --- Row 5: Prometheus Health --- - id: pipeline-prometheus-ingestion title: "Prometheus Ingestion Rate (chunks/sec)" query: "rate(prometheus_tsdb_head_chunks_created_total[5m])" chartType: line + - id: pipeline-prometheus-active-series title: "Prometheus Active Time Series" query: "prometheus_tsdb_head_series" chartType: line + # --- Row 6: Prometheus Storage --- - id: pipeline-prometheus-wal-size title: "Prometheus WAL Size (bytes)" query: "prometheus_tsdb_wal_storage_size_bytes" chartType: line + - id: pipeline-prometheus-head-chunks title: "Prometheus Head Chunks Size (bytes)" query: "prometheus_tsdb_head_chunks_storage_size_bytes" chartType: line + - id: pipeline-prometheus-query-latency title: "Prometheus Query Latency P99 (sec)" query: "histogram_quantile(0.99, rate(prometheus_http_request_duration_seconds_bucket{handler=\"/api/v1/query\"}[5m]))" chartType: line + + - id: pipeline-prometheus-range-query-latency + title: "Prometheus Range Query Latency P99 (sec)" + query: "histogram_quantile(0.99, rate(prometheus_http_request_duration_seconds_bucket{handler=\"/api/v1/query_range\"}[5m]))" + chartType: line + # --- Row 7: Data Prepper — Logs Pipeline --- - id: pipeline-dp-logs-processed title: "DP Logs Processed/sec" query: "rate(otel_logs_pipeline_recordsProcessed_total[5m])" chartType: line + - id: pipeline-dp-logs-latency title: "DP Logs Pipeline Latency (avg sec)" query: "rate(otel_logs_pipeline_opensearch_PipelineLatency_seconds_sum[5m]) / rate(otel_logs_pipeline_opensearch_PipelineLatency_seconds_count[5m])" chartType: line + # --- Row 8: Data Prepper — Traces Pipeline --- - id: pipeline-dp-traces-processed title: "DP Traces Processed/sec" query: "rate(otel_traces_pipeline_recordsProcessed_total[5m])" chartType: line + - id: pipeline-dp-traces-latency title: "DP Traces Pipeline Latency (avg sec)" query: "rate(traces_raw_pipeline_opensearch_PipelineLatency_seconds_sum[5m]) / rate(traces_raw_pipeline_opensearch_PipelineLatency_seconds_count[5m])" chartType: line + # --- Row 9: Data Prepper — Metrics Pipeline --- - id: pipeline-dp-metrics-received title: "DP Metrics Received/sec" query: "rate(otlp_metrics_requestsReceived_total[5m])" chartType: line + - id: pipeline-dp-otlp-requests title: "DP OTLP Requests Received/sec (all)" query: "rate(otlp_traces_requestsReceived_total[5m]) + rate(otlp_logs_requestsReceived_total[5m]) + rate(otlp_metrics_requestsReceived_total[5m])" chartType: line + # --- Row 10: Data Prepper — Writes & Errors --- - id: pipeline-dp-logs-docs-written title: "DP Logs Docs Written/sec" query: "rate(otel_logs_pipeline_opensearch_documentsSuccess_total[5m])" chartType: line + - id: pipeline-dp-traces-docs-written title: "DP Traces Docs Written/sec" query: "rate(traces_raw_pipeline_opensearch_documentsSuccess_total[5m])" chartType: line + # --- Row 11: Data Prepper — Errors & Buffer --- - id: pipeline-dp-bulk-errors title: "DP Bulk Request Errors" query: "sum(rate(otel_logs_pipeline_opensearch_bulkRequestErrors_total[5m])) + sum(rate(traces_raw_pipeline_opensearch_bulkRequestErrors_total[5m]))" chartType: line + - id: pipeline-dp-buffer-usage title: "DP Buffer Writes/sec" query: "rate(otel_logs_pipeline_BlockingBuffer_recordsWritten_total[5m]) + rate(otel_traces_pipeline_BlockingBuffer_recordsWritten_total[5m])" chartType: line + - id: pipeline-dp-buffer-capacity title: "DP Buffer Capacity Used %" query: "otlp_pipeline_BlockingBuffer_capacityUsed + otel_logs_pipeline_BlockingBuffer_capacityUsed + otel_traces_pipeline_BlockingBuffer_capacityUsed" diff --git a/charts/observability-stack/files/saved-queries-self-monitoring.yaml b/charts/observability-stack/files/saved-queries-self-monitoring.yaml new file mode 100644 index 00000000..f21c8290 --- /dev/null +++ b/charts/observability-stack/files/saved-queries-self-monitoring.yaml @@ -0,0 +1,91 @@ +# Saved queries for OpenSearch Dashboards - Self-Monitoring & Capacity Planning +# PromQL queries for monitoring the observability stack itself under load + +queries: + # --- OpenSearch Health --- + - id: os_jvm_heap_used_pct + title: "OpenSearch JVM Heap Used %" + description: JVM heap utilization — sustained >85% indicates need to scale + language: PROMQL + query: | + 100 * elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"} + + - id: os_thread_pool_rejections + title: "OpenSearch Thread Pool Rejections" + description: Rejected requests per second — non-zero means OpenSearch is overloaded + language: PROMQL + query: | + rate(elasticsearch_thread_pool_rejected_count[5m]) + + - id: os_active_searches + title: "OpenSearch Active Searches" + description: Currently executing search queries + language: PROMQL + query: | + elasticsearch_indices_search_query_current + + - id: os_search_latency + title: "OpenSearch Search Latency (sec/query)" + description: Average time per search query + language: PROMQL + query: | + rate(elasticsearch_indices_search_query_time_seconds_total[5m]) / rate(elasticsearch_indices_search_query_total[5m]) + + # --- OTel Collector --- + - id: otel_spans_dropped + title: "OTel Collector Spans Dropped/sec" + description: Failed span exports — non-zero indicates backpressure or downstream failure + language: PROMQL + query: | + rate(otelcol_exporter_send_failed_spans_total[5m]) + + - id: otel_batch_timeout_sends + title: "OTel Batch Timeout Sends/sec" + description: Batches sent due to timeout rather than reaching batch size — high rate means low throughput + language: PROMQL + query: | + rate(otelcol_processor_batch_timeout_trigger_send_total[5m]) + + # --- Prometheus --- + - id: prom_query_latency_p99 + title: "Prometheus Query Latency P99" + description: 99th percentile query duration — degrades under concurrent PromQL load + language: PROMQL + query: | + histogram_quantile(0.99, rate(prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query"}[5m])) + + - id: prom_range_query_latency_p99 + title: "Prometheus Range Query Latency P99" + description: 99th percentile range query duration — expensive for long time ranges + language: PROMQL + query: | + histogram_quantile(0.99, rate(prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_range"}[5m])) + + - id: prom_active_series + title: "Prometheus Active Time Series" + description: Total active time series — drives memory usage + language: PROMQL + query: | + prometheus_tsdb_head_series + + - id: prom_ingestion_rate + title: "Prometheus Sample Ingestion Rate" + description: Samples appended per second + language: PROMQL + query: | + rate(prometheus_tsdb_head_samples_appended_total[5m]) + + # --- Data Prepper --- + - id: dp_buffer_capacity_used + title: "Data Prepper Buffer Capacity Used" + description: Combined buffer usage across all pipelines — approaching max means backpressure + language: PROMQL + query: | + otlp_pipeline_BlockingBuffer_capacityUsed + otel_logs_pipeline_BlockingBuffer_capacityUsed + otel_traces_pipeline_BlockingBuffer_capacityUsed + + - id: dp_bulk_write_errors + title: "Data Prepper Bulk Write Errors/sec" + description: OpenSearch bulk request errors from Data Prepper + language: PROMQL + query: | + sum(rate(otel_logs_pipeline_opensearch_bulkRequestErrors_total[5m])) + sum(rate(traces_raw_pipeline_opensearch_bulkRequestErrors_total[5m])) diff --git a/charts/observability-stack/templates/init-dashboards-configmap.yaml b/charts/observability-stack/templates/init-dashboards-configmap.yaml index 2496dff0..63e179ef 100644 --- a/charts/observability-stack/templates/init-dashboards-configmap.yaml +++ b/charts/observability-stack/templates/init-dashboards-configmap.yaml @@ -16,6 +16,8 @@ data: {{ .Files.Get "files/saved-queries-traces.yaml" | indent 4 }} saved-queries-metrics.yaml: | {{ .Files.Get "files/saved-queries-metrics.yaml" | indent 4 }} + saved-queries-self-monitoring.yaml: | +{{ .Files.Get "files/saved-queries-self-monitoring.yaml" | indent 4 }} dashboard-k8s-cluster-health.yaml: | {{ .Files.Get "files/dashboard-k8s-cluster-health.yaml" | indent 4 }} dashboard-pipeline-health.yaml: | From a51b97a5f847c2feca3f6b5ee6cd6935df7514ca Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 20 Mar 2026 13:33:43 -0700 Subject: [PATCH 45/64] fix: wrap high-cardinality PromQL in sum() to fix OSD Prometheus deserialization error --- .../files/dashboard-opensearch-health.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/observability-stack/files/dashboard-opensearch-health.yaml b/charts/observability-stack/files/dashboard-opensearch-health.yaml index 42fd6695..7a806917 100644 --- a/charts/observability-stack/files/dashboard-opensearch-health.yaml +++ b/charts/observability-stack/files/dashboard-opensearch-health.yaml @@ -64,21 +64,21 @@ panels: # --- Row 6: Query Pressure (Load Testing) --- - id: os-thread-pool-rejections title: "Thread Pool Rejections/sec" - query: "rate(elasticsearch_thread_pool_rejected_count[5m])" + query: "sum by (type) (rate(elasticsearch_thread_pool_rejected_count[5m]))" chartType: line - id: os-active-searches title: "Active Searches" - query: "elasticsearch_indices_search_query_current" + query: "sum(elasticsearch_indices_search_query_current)" chartType: line # --- Row 7: Search Latency & Fetch --- - id: os-search-latency title: "Search Latency (sec/query)" - query: "rate(elasticsearch_indices_search_query_time_seconds_total[5m]) / rate(elasticsearch_indices_search_query_total[5m])" + query: "sum(rate(elasticsearch_indices_search_query_time_seconds_total[5m])) / sum(rate(elasticsearch_indices_search_query_total[5m]))" chartType: line - id: os-search-fetch-rate title: "Search Fetch Rate (fetches/sec)" - query: "rate(elasticsearch_indices_search_fetch_total[5m])" + query: "sum(rate(elasticsearch_indices_search_fetch_total[5m]))" chartType: line From 2c70dd70087c0039247877ebfc4181004c4bcc17 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 20 Mar 2026 13:44:46 -0700 Subject: [PATCH 46/64] fix: use delete-then-recreate for PromQL dashboards (sync with docker-compose) --- .../observability-stack/files/init-opensearch-dashboards.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/observability-stack/files/init-opensearch-dashboards.py b/charts/observability-stack/files/init-opensearch-dashboards.py index 6f32b98d..2e05d7ca 100644 --- a/charts/observability-stack/files/init-opensearch-dashboards.py +++ b/charts/observability-stack/files/init-opensearch-dashboards.py @@ -1049,10 +1049,10 @@ def create_promql_dashboard_from_yaml(workspace_id, config_path, prometheus_data else: url = f"{BASE_URL}/api/saved_objects/dashboard/{dashboard_id}" try: + # Always delete and recreate the dashboard so panel order matches YAML + requests.delete(url, auth=(USERNAME, PASSWORD), headers={"osd-xsrf": "true"}, verify=False, timeout=10) response = requests.post(url, auth=(USERNAME, PASSWORD), headers={"Content-Type": "application/json", "osd-xsrf": "true"}, json=dashboard_payload, verify=False, timeout=10) - if response.status_code in (200, 409): - if response.status_code == 409: - requests.put(url, auth=(USERNAME, PASSWORD), headers={"Content-Type": "application/json", "osd-xsrf": "true"}, json={"attributes": dashboard_payload["attributes"], "references": references}, verify=False, timeout=10) + if response.status_code == 200: print(f"✅ Created {dashboard_config['title']} dashboard ({len(created_ids)} panels)") return dashboard_id else: From bcf3b37f63b35300c4749efd303097c11a244cc5 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 20 Mar 2026 13:56:30 -0700 Subject: [PATCH 47/64] fix: remove strict-mapping-rejected fields from explore objects (OSD 3.6 compat) --- charts/observability-stack/files/init-opensearch-dashboards.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/charts/observability-stack/files/init-opensearch-dashboards.py b/charts/observability-stack/files/init-opensearch-dashboards.py index 2e05d7ca..ea21040c 100644 --- a/charts/observability-stack/files/init-opensearch-dashboards.py +++ b/charts/observability-stack/files/init-opensearch-dashboards.py @@ -996,9 +996,6 @@ def create_promql_dashboard_from_yaml(workspace_id, config_path, prometheus_data payload = { "attributes": { "title": panel_def["title"], "description": "", "hits": 0, - "columns": ["_source"], "sort": [], "version": 1, "type": "metrics", - "visualization": viz_template, - "uiState": json.dumps({"activeTab": "explore_visualization_tab"}), "kibanaSavedObjectMeta": {"searchSourceJSON": search_source} }, "references": [{"name": "kibanaSavedObjectMeta.searchSourceJSON.index", "type": "index-pattern", "id": prometheus_datasource_title}] From 84d2afeaaa6e28a37f4b86161e3d39652cf7283b Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 20 Mar 2026 14:10:09 -0700 Subject: [PATCH 48/64] fix: restore explore object fields + use sum() without by for thread pool rejections --- .../observability-stack/files/dashboard-opensearch-health.yaml | 2 +- charts/observability-stack/files/init-opensearch-dashboards.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/charts/observability-stack/files/dashboard-opensearch-health.yaml b/charts/observability-stack/files/dashboard-opensearch-health.yaml index 7a806917..3e933cba 100644 --- a/charts/observability-stack/files/dashboard-opensearch-health.yaml +++ b/charts/observability-stack/files/dashboard-opensearch-health.yaml @@ -64,7 +64,7 @@ panels: # --- Row 6: Query Pressure (Load Testing) --- - id: os-thread-pool-rejections title: "Thread Pool Rejections/sec" - query: "sum by (type) (rate(elasticsearch_thread_pool_rejected_count[5m]))" + query: "sum(rate(elasticsearch_thread_pool_rejected_count[5m]))" chartType: line - id: os-active-searches diff --git a/charts/observability-stack/files/init-opensearch-dashboards.py b/charts/observability-stack/files/init-opensearch-dashboards.py index ea21040c..2e05d7ca 100644 --- a/charts/observability-stack/files/init-opensearch-dashboards.py +++ b/charts/observability-stack/files/init-opensearch-dashboards.py @@ -996,6 +996,9 @@ def create_promql_dashboard_from_yaml(workspace_id, config_path, prometheus_data payload = { "attributes": { "title": panel_def["title"], "description": "", "hits": 0, + "columns": ["_source"], "sort": [], "version": 1, "type": "metrics", + "visualization": viz_template, + "uiState": json.dumps({"activeTab": "explore_visualization_tab"}), "kibanaSavedObjectMeta": {"searchSourceJSON": search_source} }, "references": [{"name": "kibanaSavedObjectMeta.searchSourceJSON.index", "type": "index-pattern", "id": prometheus_datasource_title}] From a8ca9bb3d012a729e2d7fc61631648b66716536c Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 20 Mar 2026 14:18:16 -0700 Subject: [PATCH 49/64] fix: use correct metric names for active searches and search latency panels --- .../files/dashboard-opensearch-health.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/observability-stack/files/dashboard-opensearch-health.yaml b/charts/observability-stack/files/dashboard-opensearch-health.yaml index 3e933cba..99204091 100644 --- a/charts/observability-stack/files/dashboard-opensearch-health.yaml +++ b/charts/observability-stack/files/dashboard-opensearch-health.yaml @@ -69,13 +69,13 @@ panels: - id: os-active-searches title: "Active Searches" - query: "sum(elasticsearch_indices_search_query_current)" + query: "sum(elasticsearch_search_active_queries)" chartType: line # --- Row 7: Search Latency & Fetch --- - id: os-search-latency title: "Search Latency (sec/query)" - query: "sum(rate(elasticsearch_indices_search_query_time_seconds_total[5m])) / sum(rate(elasticsearch_indices_search_query_total[5m]))" + query: "rate(elasticsearch_indices_search_query_time_seconds[5m])" chartType: line - id: os-search-fetch-rate From 60c6ff56dadc9daf35e0b3ce8f7f01c0712d7859 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 20 Mar 2026 14:28:03 -0700 Subject: [PATCH 50/64] feat: add query cache size panel to OpenSearch health dashboard --- .../files/dashboard-opensearch-health.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/charts/observability-stack/files/dashboard-opensearch-health.yaml b/charts/observability-stack/files/dashboard-opensearch-health.yaml index 99204091..05a76e0e 100644 --- a/charts/observability-stack/files/dashboard-opensearch-health.yaml +++ b/charts/observability-stack/files/dashboard-opensearch-health.yaml @@ -68,8 +68,8 @@ panels: chartType: line - id: os-active-searches - title: "Active Searches" - query: "sum(elasticsearch_search_active_queries)" + title: "Query Cache Hit Ratio %" + query: "sum(rate(elasticsearch_indices_query_cache_total{result=\"hit\"}[5m])) / sum(rate(elasticsearch_indices_query_cache_total[5m])) * 100" chartType: line # --- Row 7: Search Latency & Fetch --- @@ -82,3 +82,9 @@ panels: title: "Search Fetch Rate (fetches/sec)" query: "sum(rate(elasticsearch_indices_search_fetch_total[5m]))" chartType: line + + # --- Row 8: Cache Performance --- + - id: os-query-cache-hit-ratio + title: "Query Cache Size (entries)" + query: "sum(elasticsearch_index_stats_query_cache_size)" + chartType: line From 531ea36d6e8a690955a281abf3a79ca2dc2ebe5f Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 20 Mar 2026 14:39:25 -0700 Subject: [PATCH 51/64] feat: add EC2 load generator terraform + ALB-routed k6 scripts - Terraform module spins up m5.xlarge in same VPC as EKS cluster - k6 scripts hit OSD through ALB (real user path: TLS + WAF + ingress) - api-queries-alb.js: PPL, search, PromQL, dashboard loads, service map - run-remote.sh: upload scripts + run tests on EC2 - Previous tests via port-forward were bottlenecked by kubectl tunnel --- load-testing/k6/scenarios/api-queries-alb.js | 126 +++++++++++++ load-testing/run-remote.sh | 44 +++++ load-testing/terraform/.gitignore | 4 + load-testing/terraform/main.tf | 176 +++++++++++++++++++ load-testing/terraform/terraform.tfvars | 4 + 5 files changed, 354 insertions(+) create mode 100644 load-testing/k6/scenarios/api-queries-alb.js create mode 100755 load-testing/run-remote.sh create mode 100644 load-testing/terraform/.gitignore create mode 100644 load-testing/terraform/main.tf create mode 100644 load-testing/terraform/terraform.tfvars diff --git a/load-testing/k6/scenarios/api-queries-alb.js b/load-testing/k6/scenarios/api-queries-alb.js new file mode 100644 index 00000000..44061e5e --- /dev/null +++ b/load-testing/k6/scenarios/api-queries-alb.js @@ -0,0 +1,126 @@ +// k6 API-level load test — hits OpenSearch Dashboards through ALB. +// Simulates the actual HTTP requests that a dashboard user generates. +// +// Usage (from EC2 load generator): +// k6 run --env TARGET_VUS=500 scenarios/api-queries-alb.js +// k6 run --env TARGET_VUS=1000 --env DASHBOARDS_URL=https://your-alb-dns scenarios/api-queries-alb.js + +import http from 'k6/http'; +import { check, sleep } from 'k6'; +import encoding from 'k6/encoding'; + +const TARGET_VUS = parseInt(__ENV.TARGET_VUS || '200'); +const OSD = __ENV.DASHBOARDS_URL || 'https://localhost:5601'; +const USER = __ENV.OSD_USER || 'admin'; +const PASS = __ENV.OSD_PASSWORD || 'My_password_123!@#'; +const AUTH_HEADER = `Basic ${encoding.b64encode(`${USER}:${PASS}`)}`; + +export const options = { + insecureSkipTLSVerify: true, + scenarios: { + osd_queries: { + executor: 'ramping-vus', + startVUs: 0, + stages: [ + { duration: '2m', target: Math.round(TARGET_VUS * 0.25) }, + { duration: '3m', target: Math.round(TARGET_VUS * 0.5) }, + { duration: '5m', target: TARGET_VUS }, + { duration: '3m', target: TARGET_VUS }, + { duration: '2m', target: 0 }, + ], + exec: 'osdLoad', + }, + }, + thresholds: { + http_req_duration: ['p(95)<5000'], + http_req_failed: ['rate<0.05'], + }, +}; + +const headers = { + 'Content-Type': 'application/json', + 'Authorization': AUTH_HEADER, + 'osd-xsrf': 'true', +}; + +// --- PPL queries via OSD's query endpoint --- +const pplQueries = [ + 'source=otel-v1-apm-span-000001 | head 50', + 'source=otel-v1-apm-span-000001 | stats count() by serviceName', + 'source=otel-v1-apm-span-000001 | stats count() by serviceName, kind', + 'source=otel-v1-apm-span-000001 | where durationInNanos > 1000000000 | stats count() by serviceName', + 'source=logs-otel-v1-000001 | head 50', + 'source=logs-otel-v1-000001 | stats count() by severityText', +]; + +// --- PromQL queries via OSD's datasource proxy --- +const promQueries = [ + 'up', + 'rate(otelcol_exporter_sent_spans_total[5m])', + 'sum(rate(otelcol_exporter_sent_spans_total[5m]))', + 'histogram_quantile(0.99, rate(prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query"}[5m]))', + 'prometheus_tsdb_head_series', + 'sum(rate(elasticsearch_indices_search_query_time_seconds[5m]))', + 'sum(elasticsearch_index_stats_query_cache_size)', +]; + +export function osdLoad() { + const action = Math.random(); + + if (action < 0.3) { + // PPL query through OSD + const q = pplQueries[Math.floor(Math.random() * pplQueries.length)]; + const res = http.post(`${OSD}/api/ppl/search`, JSON.stringify({ + query: q, + format: 'jdbc', + }), { headers }); + check(res, { 'PPL 2xx': (r) => r.status >= 200 && r.status < 300 }); + + } else if (action < 0.5) { + // Discover-style search through OSD + const res = http.post(`${OSD}/api/console/proxy`, JSON.stringify({ + size: 50, + query: { match_all: {} }, + sort: [{ startTime: 'desc' }], + }), { + headers: { + ...headers, + 'opensearch-endpoint': 'otel-v1-apm-span-*/_search', + }, + }); + check(res, { 'Search 2xx': (r) => r.status >= 200 && r.status < 400 }); + + } else if (action < 0.7) { + // PromQL through OSD datasource proxy + const q = promQueries[Math.floor(Math.random() * promQueries.length)]; + const now = Math.floor(Date.now() / 1000); + const res = http.get( + `${OSD}/api/datasources/proxy/ObservabilityStack_Prometheus/api/v1/query_range?query=${encodeURIComponent(q)}&start=${now - 3600}&end=${now}&step=60`, + { headers }, + ); + check(res, { 'PromQL 2xx': (r) => r.status >= 200 && r.status < 300 }); + + } else if (action < 0.85) { + // Load saved objects (simulates opening a dashboard) + const res = http.get( + `${OSD}/api/saved_objects/_find?type=dashboard&per_page=10`, + { headers }, + ); + check(res, { 'Dashboards list 2xx': (r) => r.status >= 200 && r.status < 300 }); + + } else { + // Service map query + const res = http.post(`${OSD}/api/console/proxy`, JSON.stringify({ + size: 200, + query: { match_all: {} }, + }), { + headers: { + ...headers, + 'opensearch-endpoint': 'otel-v2-apm-service-map-*/_search', + }, + }); + check(res, { 'ServiceMap 2xx': (r) => r.status >= 200 && r.status < 400 }); + } + + sleep(Math.random() * 2 + 0.5); // 0.5-2.5s think time +} diff --git a/load-testing/run-remote.sh b/load-testing/run-remote.sh new file mode 100755 index 00000000..55f2e7df --- /dev/null +++ b/load-testing/run-remote.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Upload k6 scripts to the load generator EC2 and optionally run a test. +# +# Usage: +# ./run-remote.sh # upload scripts only +# ./run-remote.sh 500 # upload + run with 500 VUs +# ./run-remote.sh 1000 api-queries-alb.js # upload + run specific script +# +# Prerequisites: +# - terraform apply in load-testing/terraform/ +# - EC2 key pair configured (or use SSM) + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +# Get EC2 IP from terraform +cd "$SCRIPT_DIR/terraform" +IP=$(terraform output -raw public_ip 2>/dev/null) +KEY_NAME=$(terraform output -raw ssh_command 2>/dev/null | grep -oP '(?<=-i )\S+' || echo "") + +if [[ -z "$IP" ]]; then + echo "❌ No load generator running. Run: cd terraform && terraform apply" + exit 1 +fi + +echo "📤 Uploading k6 scripts to $IP..." +SSH_OPTS="-o StrictHostKeyChecking=no" +[[ -n "$KEY_NAME" ]] && SSH_OPTS="$SSH_OPTS -i $KEY_NAME" + +scp $SSH_OPTS -r "$SCRIPT_DIR/k6/" "ec2-user@${IP}:/home/ec2-user/k6/" +echo "✅ Scripts uploaded" + +TARGET_VUS="${1:-}" +SCENARIO="${2:-api-queries-alb.js}" + +if [[ -n "$TARGET_VUS" ]]; then + TARGET_URL=$(terraform output -raw run_test 2>/dev/null | grep -oP '(?<=DASHBOARDS_URL=)\S+' || echo "") + echo "" + echo "🚀 Running k6 with $TARGET_VUS VUs ($SCENARIO)..." + echo " Target: $TARGET_URL" + echo "" + ssh $SSH_OPTS "ec2-user@${IP}" \ + "cd /home/ec2-user/k6 && source .env 2>/dev/null; k6 run --env TARGET_VUS=$TARGET_VUS scenarios/$SCENARIO" +fi diff --git a/load-testing/terraform/.gitignore b/load-testing/terraform/.gitignore new file mode 100644 index 00000000..41b04302 --- /dev/null +++ b/load-testing/terraform/.gitignore @@ -0,0 +1,4 @@ +.terraform/ +*.tfstate +*.tfstate.backup +.terraform.lock.hcl diff --git a/load-testing/terraform/main.tf b/load-testing/terraform/main.tf new file mode 100644 index 00000000..460bc038 --- /dev/null +++ b/load-testing/terraform/main.tf @@ -0,0 +1,176 @@ +# Load Generator EC2 instance in the same VPC as the EKS cluster. +# Runs k6 against the ALB endpoint — real end-to-end path including TLS + WAF. +# +# Usage: +# cd load-testing/terraform +# terraform init +# terraform apply -var="vpc_id=vpc-xxx" -var="subnet_id=subnet-xxx" -var="target_url=https://obs-playground-dev-..." +# +# # SSH in and run tests: +# ssh -i ~/.ssh/load-test-key.pem ec2-user@ +# cd /home/ec2-user/k6 && k6 run scenarios/api-queries.js +# +# # Destroy when done: +# terraform destroy + +terraform { + required_providers { + aws = { source = "hashicorp/aws", version = "~> 5.0" } + } +} + +provider "aws" { + region = var.region +} + +variable "region" { + default = "us-west-2" +} + +variable "vpc_id" { + description = "VPC ID where EKS cluster runs (from main terraform output)" + type = string +} + +variable "subnet_id" { + description = "Public subnet ID in the same VPC" + type = string +} + +variable "target_url" { + description = "ALB URL for OpenSearch Dashboards (e.g. https://obs-playground-dev-....people.aws.dev)" + type = string +} + +variable "opensearch_user" { + default = "admin" +} + +variable "opensearch_password" { + default = "My_password_123!@#" +} + +variable "instance_type" { + description = "EC2 instance type — m5.xlarge recommended for 1000+ VUs" + default = "m5.xlarge" +} + +variable "key_name" { + description = "EC2 key pair name for SSH access. Leave empty to skip SSH." + default = "" +} + +# --- Security Group --- +resource "aws_security_group" "load_generator" { + name_prefix = "load-test-" + vpc_id = var.vpc_id + + # SSH (optional) + dynamic "ingress" { + for_each = var.key_name != "" ? [1] : [] + content { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + } + + # All outbound (to reach ALB) + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { Name = "load-test-generator" } +} + +# --- Latest Amazon Linux 2023 AMI --- +data "aws_ami" "al2023" { + most_recent = true + owners = ["amazon"] + filter { + name = "name" + values = ["al2023-ami-*-x86_64"] + } + filter { + name = "state" + values = ["available"] + } +} + +# --- User Data: install k6 + copy scripts --- +locals { + user_data = <<-EOF + #!/bin/bash + set -euo pipefail + + # Install k6 + dnf install -y https://dl.k6.io/rpm/repo.rpm || true + dnf install -y k6 || { + # Fallback: install from binary + curl -sL https://github.com/grafana/k6/releases/latest/download/k6-linux-amd64.tar.gz | tar xz + mv k6-*/k6 /usr/local/bin/ + } + + # Create k6 scripts directory + mkdir -p /home/ec2-user/k6/scenarios + + # Write environment config + cat > /home/ec2-user/k6/.env <<'ENVEOF' + export DASHBOARDS_URL="${var.target_url}" + export OSD_USER="${var.opensearch_user}" + export OSD_PASSWORD="${var.opensearch_password}" + export OPENSEARCH_URL="${var.target_url}/api/console/proxy?path=/" + export PROMETHEUS_URL="${var.target_url}/api/console/proxy?path=/" + ENVEOF + + chown -R ec2-user:ec2-user /home/ec2-user/k6 + + echo "✅ Load generator ready. Upload k6 scripts to /home/ec2-user/k6/" + EOF +} + +# --- EC2 Instance --- +resource "aws_instance" "load_generator" { + ami = data.aws_ami.al2023.id + instance_type = var.instance_type + subnet_id = var.subnet_id + vpc_security_group_ids = [aws_security_group.load_generator.id] + associate_public_ip_address = true + key_name = var.key_name != "" ? var.key_name : null + user_data = local.user_data + + root_block_device { + volume_size = 20 + } + + tags = { + Name = "load-test-generator" + Project = "observability-stack" + ManagedBy = "terraform" + } +} + +# --- Outputs --- +output "instance_id" { + value = aws_instance.load_generator.id +} + +output "public_ip" { + value = aws_instance.load_generator.public_ip +} + +output "ssh_command" { + value = var.key_name != "" ? "ssh -i ~/.ssh/${var.key_name}.pem ec2-user@${aws_instance.load_generator.public_ip}" : "No SSH key configured — use SSM: aws ssm start-session --target ${aws_instance.load_generator.id}" +} + +output "upload_scripts" { + value = var.key_name != "" ? "scp -i ~/.ssh/${var.key_name}.pem -r ../k6/ ec2-user@${aws_instance.load_generator.public_ip}:/home/ec2-user/k6/" : "Use SSM or attach scripts via user_data" +} + +output "run_test" { + value = "k6 run --env TARGET_VUS=1000 --env DASHBOARDS_URL=${var.target_url} scenarios/api-queries.js" +} diff --git a/load-testing/terraform/terraform.tfvars b/load-testing/terraform/terraform.tfvars new file mode 100644 index 00000000..3df83af7 --- /dev/null +++ b/load-testing/terraform/terraform.tfvars @@ -0,0 +1,4 @@ +vpc_id = "vpc-03aefcf5aa0581d7a" +subnet_id = "subnet-0103d5dff6443d113" +target_url = "https://obs-playground-dev-027423573553.kylhouns.people.aws.dev" +# key_name = "your-key-pair" # Uncomment if you have an EC2 key pair for SSH From 1d33373c3785ad8e8258cae6df299d9e51b700de Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 20 Mar 2026 15:00:33 -0700 Subject: [PATCH 52/64] =?UTF-8?q?results:=20Test=20004=20=E2=80=94=20OSD?= =?UTF-8?q?=20is=20the=20bottleneck=20at=20100m=20CPU,=203s+=20latency=20u?= =?UTF-8?q?nder=201000=20VUs=20via=20ALB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- load-testing/RESULTS.md | 1 + .../results/004-alb-1000vu-osd-bottleneck.md | 70 ++++++++++ load-testing/terraform/main.tf | 125 +++++++----------- 3 files changed, 120 insertions(+), 76 deletions(-) create mode 100644 load-testing/results/004-alb-1000vu-osd-bottleneck.md diff --git a/load-testing/RESULTS.md b/load-testing/RESULTS.md index 8780327c..b5ace5fd 100644 --- a/load-testing/RESULTS.md +++ b/load-testing/RESULTS.md @@ -34,3 +34,4 @@ | 1 | API Query Load (200 OS VUs + 100 Prom VUs) | 2026-03-20 12:04–12:19 | ⚠️ Auth bug | [001-api-queries-auth-bug.md](results/001-api-queries-auth-bug.md) | | 2 | API Query Load (300 VUs, auth fixed) | 2026-03-20 12:42–12:57 | ✅ 0% errors, p95=16ms | [002-api-queries.md](results/002-api-queries.md) | | 3 | API Query Load (1500 VUs) | 2026-03-20 12:57–13:12 | ⚠️ p95=2.28s, 0% errors | [003-api-queries-1500vu.md](results/003-api-queries-1500vu.md) | +| 4 | ALB E2E (1000 VUs from EC2) | 2026-03-20 14:55–14:59 | 🔴 OSD saturated at 100m CPU, 3s+ latency | [004-alb-1000vu-osd-bottleneck.md](results/004-alb-1000vu-osd-bottleneck.md) | diff --git a/load-testing/results/004-alb-1000vu-osd-bottleneck.md b/load-testing/results/004-alb-1000vu-osd-bottleneck.md new file mode 100644 index 00000000..baf8a365 --- /dev/null +++ b/load-testing/results/004-alb-1000vu-osd-bottleneck.md @@ -0,0 +1,70 @@ +# Test 004: ALB End-to-End Load Test — 1000 VUs from EC2 + +**Status:** 🔴 Broke immediately — massive latency, multiple error types +**Script:** `k6/scenarios/api-queries-alb.js` +**Start:** 2026-03-20 14:55:00 PDT (21:55:00 UTC) +**End:** 2026-03-20 ~14:59:00 PDT (interrupted — already broken) +**Duration:** ~4 min before manual stop +**Source:** EC2 m5.xlarge in same VPC → ALB → OSD → OpenSearch/Prometheus + +## Parameters +- Target VUs: 1000 (ramping) +- Source: EC2 `i-08f9652631fe73302` in same VPC +- Path: EC2 → ALB (TLS) → OSD → OpenSearch/Prometheus +- Background: OTel Demo load generator active + +## What Broke + +### 1. OpenSearch Dashboards is the bottleneck (100m CPU / 512M memory) + +Every request through OSD took **1.3–3.6 seconds** even for simple operations: +- PPL search: 3,199ms – 3,590ms response times +- Saved objects list: 3,306ms – 3,503ms +- Console proxy: 2,298ms – 2,502ms + +OSD is a Node.js single-threaded server with **100m CPU limit** (0.1 cores). Under 1000 concurrent connections, it's completely CPU-starved. The event loop is blocked processing requests sequentially. + +### 2. Console proxy returns 400 for search/service-map queries + +`POST /api/console/proxy` with `opensearch-endpoint` header returns 400. The console proxy API requires different parameters than what the script sends. These need to be fixed in the script, but the latency issue is the real finding. + +### 3. Prometheus datasource proxy returns 404 + +`GET /api/datasources/proxy/ObservabilityStack_Prometheus/...` returns 404. OSD doesn't expose a datasource proxy at that path — PromQL queries go through a different internal API. Script needs fixing, but again the OSD bottleneck is the headline. + +### 4. OpenSearch itself is fine + +- JVM Heap: 55% (285MB / 512MB) — not stressed +- OS CPU: 8% — barely working +- Thread pool rejections: 0 +- Cluster status: yellow (normal for single-node) +- OS Memory: 99% — high but stable (JVM + OS caches) + +### 5. No pod crashes or OOMs + +All pods stayed running. No restarts. The system degraded gracefully — it didn't crash, it just became unusably slow. + +## Root Cause Analysis + +**The bottleneck is OpenSearch Dashboards, not OpenSearch.** + +| Component | CPU Limit | Memory Limit | Status Under Load | +|-----------|-----------|--------------|-------------------| +| **OSD** | **100m (0.1 cores)** | **512M** | **🔴 Saturated — 3s+ response times** | +| OpenSearch | 500m (no limit) | 2Gi | ✅ Fine — 55% heap, 8% CPU | +| Prometheus | none | none | ✅ Fine (not reached — OSD blocked) | +| Data Prepper | none | none | ✅ Fine | + +OSD is the gateway for all user traffic. With 100m CPU, it can handle roughly **5-10 concurrent requests** before the Node.js event loop saturates. At 1000 VUs, requests queue up and each takes 3+ seconds just waiting for OSD to process them. + +## Recommendations + +1. **Increase OSD CPU limit**: 100m → 1000m (1 core) minimum, 2000m for production +2. **Increase OSD memory**: 512M → 1Gi minimum +3. **Scale OSD replicas**: Add 2-3 replicas behind the ALB for horizontal scaling +4. **Fix k6 script**: Console proxy and Prometheus proxy endpoints need correct API paths +5. **Re-run test** after OSD scaling to find the next bottleneck (likely OpenSearch) + +## Key Insight + +Previous tests via port-forward (Tests 001-003) were testing OpenSearch directly, bypassing OSD entirely. The real user path goes through OSD, which is severely under-resourced at 100m CPU. This is the first thing to fix before any other capacity tuning matters. diff --git a/load-testing/terraform/main.tf b/load-testing/terraform/main.tf index 460bc038..e5839143 100644 --- a/load-testing/terraform/main.tf +++ b/load-testing/terraform/main.tf @@ -1,17 +1,9 @@ -# Load Generator EC2 instance in the same VPC as the EKS cluster. -# Runs k6 against the ALB endpoint — real end-to-end path including TLS + WAF. +# Load Generator EC2 — same VPC as EKS, hits ALB end-to-end. # # Usage: -# cd load-testing/terraform -# terraform init -# terraform apply -var="vpc_id=vpc-xxx" -var="subnet_id=subnet-xxx" -var="target_url=https://obs-playground-dev-..." -# -# # SSH in and run tests: -# ssh -i ~/.ssh/load-test-key.pem ec2-user@ -# cd /home/ec2-user/k6 && k6 run scenarios/api-queries.js -# -# # Destroy when done: -# terraform destroy +# terraform init && terraform apply +# # Then from your laptop: +# ../run-remote.sh 1000 terraform { required_providers { @@ -28,17 +20,16 @@ variable "region" { } variable "vpc_id" { - description = "VPC ID where EKS cluster runs (from main terraform output)" - type = string + type = string } variable "subnet_id" { - description = "Public subnet ID in the same VPC" + description = "Public subnet in the same VPC" type = string } variable "target_url" { - description = "ALB URL for OpenSearch Dashboards (e.g. https://obs-playground-dev-....people.aws.dev)" + description = "ALB URL for OpenSearch Dashboards" type = string } @@ -51,13 +42,30 @@ variable "opensearch_password" { } variable "instance_type" { - description = "EC2 instance type — m5.xlarge recommended for 1000+ VUs" - default = "m5.xlarge" + default = "m5.xlarge" } -variable "key_name" { - description = "EC2 key pair name for SSH access. Leave empty to skip SSH." - default = "" +# --- IAM role for SSM access (no SSH key needed) --- +resource "aws_iam_role" "load_generator" { + name_prefix = "load-test-" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { Service = "ec2.amazonaws.com" } + }] + }) +} + +resource "aws_iam_role_policy_attachment" "ssm" { + role = aws_iam_role.load_generator.name + policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" +} + +resource "aws_iam_instance_profile" "load_generator" { + name_prefix = "load-test-" + role = aws_iam_role.load_generator.name } # --- Security Group --- @@ -65,18 +73,6 @@ resource "aws_security_group" "load_generator" { name_prefix = "load-test-" vpc_id = var.vpc_id - # SSH (optional) - dynamic "ingress" { - for_each = var.key_name != "" ? [1] : [] - content { - from_port = 22 - to_port = 22 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } - } - - # All outbound (to reach ALB) egress { from_port = 0 to_port = 0 @@ -87,7 +83,7 @@ resource "aws_security_group" "load_generator" { tags = { Name = "load-test-generator" } } -# --- Latest Amazon Linux 2023 AMI --- +# --- Latest AL2023 AMI --- data "aws_ami" "al2023" { most_recent = true owners = ["amazon"] @@ -101,50 +97,36 @@ data "aws_ami" "al2023" { } } -# --- User Data: install k6 + copy scripts --- -locals { +# --- EC2 Instance --- +resource "aws_instance" "load_generator" { + ami = data.aws_ami.al2023.id + instance_type = var.instance_type + subnet_id = var.subnet_id + vpc_security_group_ids = [aws_security_group.load_generator.id] + iam_instance_profile = aws_iam_instance_profile.load_generator.name + associate_public_ip_address = true + user_data = <<-EOF #!/bin/bash set -euo pipefail # Install k6 - dnf install -y https://dl.k6.io/rpm/repo.rpm || true - dnf install -y k6 || { - # Fallback: install from binary - curl -sL https://github.com/grafana/k6/releases/latest/download/k6-linux-amd64.tar.gz | tar xz - mv k6-*/k6 /usr/local/bin/ - } - - # Create k6 scripts directory - mkdir -p /home/ec2-user/k6/scenarios + curl -sL https://github.com/grafana/k6/releases/download/v1.0.0/k6-v1.0.0-linux-amd64.tar.gz | tar xz -C /usr/local/bin --strip-components=1 k6-v1.0.0-linux-amd64/k6 - # Write environment config + # Write env config + mkdir -p /home/ec2-user/k6/scenarios /home/ec2-user/k6/results cat > /home/ec2-user/k6/.env <<'ENVEOF' export DASHBOARDS_URL="${var.target_url}" export OSD_USER="${var.opensearch_user}" - export OSD_PASSWORD="${var.opensearch_password}" - export OPENSEARCH_URL="${var.target_url}/api/console/proxy?path=/" - export PROMETHEUS_URL="${var.target_url}/api/console/proxy?path=/" + export OSD_PASSWORD='${var.opensearch_password}' ENVEOF chown -R ec2-user:ec2-user /home/ec2-user/k6 - - echo "✅ Load generator ready. Upload k6 scripts to /home/ec2-user/k6/" + echo "✅ Load generator ready" > /home/ec2-user/k6/STATUS EOF -} - -# --- EC2 Instance --- -resource "aws_instance" "load_generator" { - ami = data.aws_ami.al2023.id - instance_type = var.instance_type - subnet_id = var.subnet_id - vpc_security_group_ids = [aws_security_group.load_generator.id] - associate_public_ip_address = true - key_name = var.key_name != "" ? var.key_name : null - user_data = local.user_data root_block_device { - volume_size = 20 + volume_size = 30 } tags = { @@ -154,23 +136,14 @@ resource "aws_instance" "load_generator" { } } -# --- Outputs --- output "instance_id" { value = aws_instance.load_generator.id } -output "public_ip" { - value = aws_instance.load_generator.public_ip -} - -output "ssh_command" { - value = var.key_name != "" ? "ssh -i ~/.ssh/${var.key_name}.pem ec2-user@${aws_instance.load_generator.public_ip}" : "No SSH key configured — use SSM: aws ssm start-session --target ${aws_instance.load_generator.id}" -} - -output "upload_scripts" { - value = var.key_name != "" ? "scp -i ~/.ssh/${var.key_name}.pem -r ../k6/ ec2-user@${aws_instance.load_generator.public_ip}:/home/ec2-user/k6/" : "Use SSM or attach scripts via user_data" +output "ssm_command" { + value = "aws ssm start-session --target ${aws_instance.load_generator.id} --region ${var.region}" } -output "run_test" { - value = "k6 run --env TARGET_VUS=1000 --env DASHBOARDS_URL=${var.target_url} scenarios/api-queries.js" +output "upload_command" { + value = "aws ssm start-session --target ${aws_instance.load_generator.id} --region ${var.region} --document-name AWS-StartInteractiveCommand --parameters command='cat > /home/ec2-user/k6/scenarios/api-queries-alb.js'" } From b17c00b42f6fc0ada8818abd74419947ba9226a5 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 20 Mar 2026 15:02:13 -0700 Subject: [PATCH 53/64] perf: scale OSD to 3 replicas, 2 CPU / 2Gi memory (was 100m/512M) --- charts/observability-stack/values.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/charts/observability-stack/values.yaml b/charts/observability-stack/values.yaml index 090c58f8..f8936398 100644 --- a/charts/observability-stack/values.yaml +++ b/charts/observability-stack/values.yaml @@ -36,10 +36,17 @@ opensearch: # -- OpenSearch Dashboards opensearch-dashboards: enabled: true - replicas: 1 + replicas: 3 image: repository: "opensearchstaging/opensearch-dashboards" tag: "3.6.0" + resources: + requests: + cpu: "500m" + memory: "1Gi" + limits: + cpu: "2000m" + memory: "2Gi" opensearchHosts: "https://opensearch-cluster-master:9200" config: opensearch_dashboards.yml: | From 7013ea3bd49a318243d9389698628b6ce3b1b793 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 20 Mar 2026 15:28:36 -0700 Subject: [PATCH 54/64] =?UTF-8?q?results:=20Test=20005=20=E2=80=94=20OSD?= =?UTF-8?q?=20bottleneck=20resolved,=20OpenSearch=20single=20node=20at=209?= =?UTF-8?q?9%=20CPU=20is=20next?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - OSD scaled 1×100m → 3×2CPU: median latency 3s → 824ms, 0% errors - OpenSearch now the bottleneck: 99-100% CPU, search queue peak 34 - Hot threads: write/refresh contention from OTel Demo indexing - p95=14.57s at 1000 VUs — need to scale OpenSearch next --- load-testing/RESULTS.md | 8 ++ load-testing/k6/scenarios/api-queries-alb.js | 39 ++++------ .../005-alb-1000vu-opensearch-bottleneck.md | 73 +++++++++++++++++++ 3 files changed, 95 insertions(+), 25 deletions(-) create mode 100644 load-testing/results/005-alb-1000vu-opensearch-bottleneck.md diff --git a/load-testing/RESULTS.md b/load-testing/RESULTS.md index b5ace5fd..e25bb5ee 100644 --- a/load-testing/RESULTS.md +++ b/load-testing/RESULTS.md @@ -35,3 +35,11 @@ | 2 | API Query Load (300 VUs, auth fixed) | 2026-03-20 12:42–12:57 | ✅ 0% errors, p95=16ms | [002-api-queries.md](results/002-api-queries.md) | | 3 | API Query Load (1500 VUs) | 2026-03-20 12:57–13:12 | ⚠️ p95=2.28s, 0% errors | [003-api-queries-1500vu.md](results/003-api-queries-1500vu.md) | | 4 | ALB E2E (1000 VUs from EC2) | 2026-03-20 14:55–14:59 | 🔴 OSD saturated at 100m CPU, 3s+ latency | [004-alb-1000vu-osd-bottleneck.md](results/004-alb-1000vu-osd-bottleneck.md) | +| 5 | ALB E2E (1000 VUs, 3x OSD 2CPU) | 2026-03-20 15:08–15:24 | ⚠️ OSD fixed, OpenSearch at 99% CPU, p95=14.57s | [005-alb-1000vu-opensearch-bottleneck.md](results/005-alb-1000vu-opensearch-bottleneck.md) | + +## Bottleneck Progression + +| Test | Bottleneck | Fix Applied | Result | +|------|-----------|-------------|--------| +| 004 | OSD (100m CPU, 1 replica) | Scaled to 3 replicas, 2 CPU each | ✅ Resolved | +| 005 | OpenSearch (single node, 4 vCPU, 99% CPU) | **Next: scale OpenSearch with search/data node separation** | Pending | diff --git a/load-testing/k6/scenarios/api-queries-alb.js b/load-testing/k6/scenarios/api-queries-alb.js index 44061e5e..a23abc66 100644 --- a/load-testing/k6/scenarios/api-queries-alb.js +++ b/load-testing/k6/scenarios/api-queries-alb.js @@ -77,29 +77,23 @@ export function osdLoad() { check(res, { 'PPL 2xx': (r) => r.status >= 200 && r.status < 300 }); } else if (action < 0.5) { - // Discover-style search through OSD - const res = http.post(`${OSD}/api/console/proxy`, JSON.stringify({ + // PPL query on logs + const q = pplQueries[Math.floor(Math.random() * pplQueries.length)]; + const res = http.post(`${OSD}/api/ppl/search`, JSON.stringify({ + query: q, + format: 'jdbc', + }), { headers }); + check(res, { 'PPL logs 2xx': (r) => r.status >= 200 && r.status < 300 }); + + } else if (action < 0.7) { + // Direct OpenSearch query through OSD (DSL search) + const res = http.post(`${OSD}/api/console/proxy?path=${encodeURIComponent('otel-v1-apm-span-*/_search')}&method=POST`, JSON.stringify({ size: 50, query: { match_all: {} }, sort: [{ startTime: 'desc' }], - }), { - headers: { - ...headers, - 'opensearch-endpoint': 'otel-v1-apm-span-*/_search', - }, - }); + }), { headers }); check(res, { 'Search 2xx': (r) => r.status >= 200 && r.status < 400 }); - } else if (action < 0.7) { - // PromQL through OSD datasource proxy - const q = promQueries[Math.floor(Math.random() * promQueries.length)]; - const now = Math.floor(Date.now() / 1000); - const res = http.get( - `${OSD}/api/datasources/proxy/ObservabilityStack_Prometheus/api/v1/query_range?query=${encodeURIComponent(q)}&start=${now - 3600}&end=${now}&step=60`, - { headers }, - ); - check(res, { 'PromQL 2xx': (r) => r.status >= 200 && r.status < 300 }); - } else if (action < 0.85) { // Load saved objects (simulates opening a dashboard) const res = http.get( @@ -110,15 +104,10 @@ export function osdLoad() { } else { // Service map query - const res = http.post(`${OSD}/api/console/proxy`, JSON.stringify({ + const res = http.post(`${OSD}/api/console/proxy?path=${encodeURIComponent('otel-v2-apm-service-map-*/_search')}&method=POST`, JSON.stringify({ size: 200, query: { match_all: {} }, - }), { - headers: { - ...headers, - 'opensearch-endpoint': 'otel-v2-apm-service-map-*/_search', - }, - }); + }), { headers }); check(res, { 'ServiceMap 2xx': (r) => r.status >= 200 && r.status < 400 }); } diff --git a/load-testing/results/005-alb-1000vu-opensearch-bottleneck.md b/load-testing/results/005-alb-1000vu-opensearch-bottleneck.md new file mode 100644 index 00000000..f0539bab --- /dev/null +++ b/load-testing/results/005-alb-1000vu-opensearch-bottleneck.md @@ -0,0 +1,73 @@ +# Test 005: ALB E2E — 1000 VUs, OSD Scaled to 3 Replicas + +**Status:** ⚠️ 0% errors but p95=14.57s — OpenSearch is now the bottleneck +**Script:** `k6/scenarios/api-queries-alb.js` +**Start:** 2026-03-20 15:08:58 PDT (22:08:58 UTC) +**End:** 2026-03-20 15:24:01 PDT (22:24:01 UTC) +**Duration:** 15m01s +**Source:** EC2 m5.xlarge in same VPC → ALB → OSD (3 replicas) → OpenSearch + +## Configuration Changes (from Test 004) +- OSD: 1 replica (100m CPU, 512M) → **3 replicas (2 CPU, 2Gi each)** + +## k6 Summary + +| Metric | Test 004 (1x OSD 100m) | Test 005 (3x OSD 2CPU) | +|--------|------------------------|------------------------| +| Total requests | ~few hundred (broke immediately) | 93,912 | +| Requests/sec | N/A | 104 | +| http_req_duration p(50) | 3+ seconds | 824ms | +| http_req_duration p(90) | N/A | 13.89s | +| http_req_duration p(95) | N/A | **14.57s** 🔴 | +| http_req_duration max | N/A | 16.95s | +| http_req_failed | immediate failures | **0.00%** ✅ | +| Data received | N/A | 5.4 GB | + +## Per-Check Breakdown + +| Check | Rate | Notes | +|-------|------|-------| +| PPL 2xx | 100% ✅ | Working through OSD | +| PPL logs 2xx | 100% ✅ | Working | +| Search 2xx | 100% ✅ | Console proxy fixed | +| ServiceMap 2xx | 100% ✅ | Console proxy fixed | +| Dashboards list 2xx | 100% ✅ | Saved objects API | + +## OpenSearch Cluster During Test + +| Metric | Value | Notes | +|--------|-------|-------| +| CPU | **99-100%** | Pegged for entire test | +| JVM Heap | 34-78% (oscillating) | GC keeping up but under pressure | +| Search threads | 7 (pool size) | Only 7 concurrent searches possible | +| Search queue peak | **34** | Queries waiting in line | +| Search rejections | 0 | Queue didn't overflow (size=1000) | +| Write threads | 4 active | OTel Demo continuously indexing | +| Write queue | 6 | Steady write backlog | + +## Hot Threads Analysis + +Top CPU consumer: **write thread doing Lucene segment refresh** (`ReferenceManager.maybeRefreshBlocking`). The OTel Demo's continuous indexing forces segment refreshes that contend with search threads for CPU and lock access. + +## Root Cause + +**OpenSearch single node (4 vCPU) is CPU-saturated.** + +- Search thread pool: 7 threads on 4 vCPU — can't parallelize enough +- Write/refresh contention: indexing from OTel Demo competes with search for CPU +- JVM heap at 512MB (50% of 1Gi request) — adequate but tight +- Single node = all shards, all searches, all writes on one box + +## OSD Bottleneck: RESOLVED ✅ + +Scaling OSD from 1×100m to 3×2CPU completely eliminated the OSD bottleneck: +- Median response dropped from 3+ seconds to 824ms +- 0% error rate (was failing immediately before) +- OSD is no longer the constraint + +## Recommendations for Next Test + +1. **Scale OpenSearch horizontally**: Add dedicated search nodes (separate from data/ingest) +2. **Increase OpenSearch JVM heap**: 512MB → 2GB+ for query caching +3. **Consider node roles**: Dedicated cluster-manager, data, and search nodes +4. Follow OpenSearch official scaling guide for search-heavy workloads From 307e3eb40bf1b0e96c07672ad1e6801b22dd0674 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 20 Mar 2026 15:30:08 -0700 Subject: [PATCH 55/64] docs: add OpenSearch scaling strategy from official docs + OSD scaling applied - OSD scaled to 3 replicas, 2 CPU / 2Gi (resolved OSD bottleneck) - Documented 3 OpenSearch scaling options: horizontal, dedicated search nodes, vertical - Official approach: separate index/search with remote store + search replicas - Recommended: start with 3 data nodes (Option A), simplest path --- .../005-alb-1000vu-opensearch-bottleneck.md | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/load-testing/results/005-alb-1000vu-opensearch-bottleneck.md b/load-testing/results/005-alb-1000vu-opensearch-bottleneck.md index f0539bab..e526c7ea 100644 --- a/load-testing/results/005-alb-1000vu-opensearch-bottleneck.md +++ b/load-testing/results/005-alb-1000vu-opensearch-bottleneck.md @@ -71,3 +71,28 @@ Scaling OSD from 1×100m to 3×2CPU completely eliminated the OSD bottleneck: 2. **Increase OpenSearch JVM heap**: 512MB → 2GB+ for query caching 3. **Consider node roles**: Dedicated cluster-manager, data, and search nodes 4. Follow OpenSearch official scaling guide for search-heavy workloads + +## OpenSearch Scaling Strategy (from official docs) + +Reference: [Separate index and search workloads](https://docs.opensearch.org/3.4/tuning-your-cluster/separate-index-and-search-workloads/) + +### Option A: Simple horizontal scaling (recommended first step) +Add more data nodes to spread shards and search threads across nodes. No architecture change needed. +- Current: 1 node (4 vCPU, 7 search threads, 99% CPU) +- Target: 3 data nodes → 21 search threads, ~33% CPU each + +### Option B: Dedicated search nodes (official OpenSearch approach) +Requires remote store (S3) + segment replication. Provides full isolation between indexing and search. +- Configure nodes with `node.roles: [search]` +- Add `number_of_search_replicas` to indices +- Search replicas are allocated only to search nodes +- Enables independent scaling of search vs ingest capacity +- Can use `_scale` API to turn off write workloads for read-heavy indices + +### Option C: Bigger instance type (vertical scaling) +Switch from m5.xlarge (4 vCPU) to m5.2xlarge (8 vCPU) or r5.2xlarge (8 vCPU, 64GB RAM). +- Doubles search thread pool immediately +- More JVM heap for query caching +- Simplest change but has a ceiling + +### Recommended approach: Start with Option A (3 data nodes), then Option C if needed, then Option B for production. From a144c9118800766e8c6659807918b5101ca1a60e Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 20 Mar 2026 15:44:25 -0700 Subject: [PATCH 56/64] perf: scale OpenSearch to 3 nodes (4Gi RAM, 2Gi JVM heap, 2 CPU each) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - singleNode: false, replicas: 3 - JVM heap: 1g → 2g (50% of 4Gi RAM) - CPU: 500m req / 2000m limit - EKS scaled to 4 nodes to fit the cluster --- charts/observability-stack/values.yaml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/charts/observability-stack/values.yaml b/charts/observability-stack/values.yaml index f8936398..f650c4f1 100644 --- a/charts/observability-stack/values.yaml +++ b/charts/observability-stack/values.yaml @@ -10,15 +10,18 @@ # Heap-to-shard ratio: max 25 shards per GB of JVM heap opensearch: enabled: true - singleNode: true - replicas: 1 + singleNode: false + replicas: 3 image: repository: "opensearchstaging/opensearch" tag: "3.6.0" resources: requests: - memory: "2Gi" - cpu: "500m" + memory: "4Gi" + cpu: "1000m" + limits: + memory: "4Gi" + cpu: "2000m" persistence: enabled: true size: 8Gi # Increase for production (e.g. 100Gi, 500Gi) @@ -28,7 +31,7 @@ opensearch: value: "My_password_123!@#" # JVM heap — set to 50% of resources.requests.memory, max 31g - name: OPENSEARCH_JAVA_OPTS - value: "-Xms1g -Xmx1g" + value: "-Xms2g -Xmx2g" config: opensearch.yml: | plugins.query.datasources.encryption.masterkey: "BTqK4Ytdz67La1kShIKV3Pu9" From 5c51b46dfefc48eeb6dbe29b0fd71b265a802733 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 20 Mar 2026 16:04:20 -0700 Subject: [PATCH 57/64] =?UTF-8?q?results:=20Test=20006=20=E2=80=94=203=20O?= =?UTF-8?q?S=20nodes:=2037%=20more=20throughput,=20p95=2010.5s=20(was=2014?= =?UTF-8?q?.5s),=20uneven=20shard=20distribution?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- load-testing/RESULTS.md | 4 +- .../006-alb-1000vu-3node-opensearch.md | 53 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 load-testing/results/006-alb-1000vu-3node-opensearch.md diff --git a/load-testing/RESULTS.md b/load-testing/RESULTS.md index e25bb5ee..cd6f8547 100644 --- a/load-testing/RESULTS.md +++ b/load-testing/RESULTS.md @@ -36,10 +36,12 @@ | 3 | API Query Load (1500 VUs) | 2026-03-20 12:57–13:12 | ⚠️ p95=2.28s, 0% errors | [003-api-queries-1500vu.md](results/003-api-queries-1500vu.md) | | 4 | ALB E2E (1000 VUs from EC2) | 2026-03-20 14:55–14:59 | 🔴 OSD saturated at 100m CPU, 3s+ latency | [004-alb-1000vu-osd-bottleneck.md](results/004-alb-1000vu-osd-bottleneck.md) | | 5 | ALB E2E (1000 VUs, 3x OSD 2CPU) | 2026-03-20 15:08–15:24 | ⚠️ OSD fixed, OpenSearch at 99% CPU, p95=14.57s | [005-alb-1000vu-opensearch-bottleneck.md](results/005-alb-1000vu-opensearch-bottleneck.md) | +| 6 | ALB E2E (1000 VUs, 3x OS nodes) | 2026-03-20 15:47–16:02 | ⚠️ 37% better throughput, p95=10.57s, uneven shards | [006-alb-1000vu-3node-opensearch.md](results/006-alb-1000vu-3node-opensearch.md) | ## Bottleneck Progression | Test | Bottleneck | Fix Applied | Result | |------|-----------|-------------|--------| | 004 | OSD (100m CPU, 1 replica) | Scaled to 3 replicas, 2 CPU each | ✅ Resolved | -| 005 | OpenSearch (single node, 4 vCPU, 99% CPU) | **Next: scale OpenSearch with search/data node separation** | Pending | +| 005 | OpenSearch (single node, 4 vCPU, 99% CPU) | Scaled to 3 nodes, 2 CPU / 4Gi each | ✅ Improved | +| 006 | Uneven shard distribution across 3 nodes | **Next: rebalance shards, increase replica count** | Pending | diff --git a/load-testing/results/006-alb-1000vu-3node-opensearch.md b/load-testing/results/006-alb-1000vu-3node-opensearch.md new file mode 100644 index 00000000..5aa01bbf --- /dev/null +++ b/load-testing/results/006-alb-1000vu-3node-opensearch.md @@ -0,0 +1,53 @@ +# Test 006: ALB E2E — 1000 VUs, 3-Node OpenSearch Cluster + +**Status:** ⚠️ Improved but p95 still high — uneven shard distribution +**Script:** `k6/scenarios/api-queries-alb.js` +**Start:** 2026-03-20 15:47:44 PDT (22:47:44 UTC) +**End:** 2026-03-20 16:02:47 PDT (23:02:47 UTC) +**Duration:** 15m01s +**Source:** EC2 m5.xlarge → ALB → 3x OSD → 3x OpenSearch + +## Configuration Changes (from Test 005) +- OpenSearch: 1 node (500m CPU, 2Gi, 1Gi JVM) → **3 nodes (2 CPU, 4Gi, 2Gi JVM each)** +- EKS: 2 nodes → **4 nodes** (m5.xlarge) + +## Comparison + +| Metric | Test 005 (1 OS node) | Test 006 (3 OS nodes) | Delta | +|--------|---------------------|----------------------|-------| +| Total requests | 93,912 | **129,083** | +37% ✅ | +| Requests/sec | 104 | **143** | +37% ✅ | +| p(50) | 824ms | **1.1s** | worse ⚠️ | +| p(90) | 13.89s | **7.09s** | 49% better ✅ | +| p(95) | 14.57s | **10.57s** | 27% better ✅ | +| max | 16.95s | 18.12s | similar | +| Error rate | 0% | **0%** | ✅ | +| Data received | 5.4 GB | **7.3 GB** | +35% | + +## Cluster Observations During Test + +| Node | Heap % | Load 1m | Search Queries | Search Queue Peak | +|------|--------|---------|----------------|-------------------| +| master-0 | 36-67% | **6-8** | 14,422 | 26 | +| master-1 | 47-78% | **1.9-2.8** | 18,886 | 23 | +| master-2 | 38-76% | **1.1-1.9** | 4,188 | 1 | + +**Key finding: Uneven shard distribution.** Node-2 handled only 4,188 searches vs 18,886 on Node-1. The shards from the original single-node cluster are concentrated on Node-0 and Node-1. Node-2 has few shards and is underutilized. + +## Analysis + +- **37% more throughput** (143 vs 104 req/s) — the extra nodes help +- **p90 halved** (7s vs 14s) — significant improvement in tail latency +- **Still too slow** — p95 of 10.57s means 5% of requests take >10 seconds +- **Uneven load** — Node-2 is barely working while Node-0 is overloaded (load_1m=8) +- **Zero rejections** — search thread pools not overflowing + +## Root Cause + +Shard allocation is unbalanced. The original indices were created with 1 primary shard on the single node. With 3 nodes, the primary stays on one node and replicas go to others, but the search routing still favors the primary. + +## Next Steps + +1. **Rebalance shards** — force shard reallocation or increase replica count +2. **Increase shard count** — more shards = better distribution across nodes +3. **Consider dedicated search nodes** — for full isolation (requires remote store) From 16abdcaed6290c452af5a21c8ae98a91d636f958 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 20 Mar 2026 16:07:47 -0700 Subject: [PATCH 58/64] docs: add capacity sizing chart with VU estimates, data projections, and scaling recommendations - Estimated concurrent user capacity by experience tier - 7-day and 30-day data volume projections - Scaling recommendations by user count with cost estimates - Load test history summary with key findings - Tracks what hasn't been tested yet --- load-testing/SIZING.md | 102 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 load-testing/SIZING.md diff --git a/load-testing/SIZING.md b/load-testing/SIZING.md new file mode 100644 index 00000000..d23b9158 --- /dev/null +++ b/load-testing/SIZING.md @@ -0,0 +1,102 @@ +# Capacity Sizing Chart + +## Current Deployment (2026-03-20) + +### Infrastructure +| Component | Replicas | CPU (req/limit) | Memory (req/limit) | Nodes | +|-----------|----------|-----------------|-------------------|-------| +| OpenSearch | 3 | 1000m / 2000m | 4Gi / 4Gi | 3x m5.xlarge | +| OpenSearch Dashboards | 3 | 500m / 2000m | 1Gi / 2Gi | spread across nodes | +| OTel Collector | 1 | none | none | 1 node | +| Data Prepper | 2 | none | none | 2 nodes | +| Prometheus | 1 | none | none | 1 node | +| EKS Nodes | 4x m5.xlarge | 4 vCPU each | 16 GB each | 16 vCPU / 64 GB total | + +### Data Volume (OTel Demo + example agents) +| Metric | Current (1.7 days) | 7-Day Projection | 30-Day Projection | +|--------|-------------------|-------------------|-------------------| +| Spans | 379,025 | ~1.5M | ~6.6M | +| Logs | 139,909 | ~568K | ~2.4M | +| Service map entries | 128,604 | ~522K | ~2.2M | +| Primary store size | 316 MB | ~1.3 GB | ~5.6 GB | +| Total store (w/ replicas) | 632 MB | ~2.5 GB | ~11 GB | +| Ingestion rate (spans) | 9,163/hr | — | — | +| Ingestion rate (logs) | 3,382/hr | — | — | + +--- + +## Concurrent User Capacity (Estimated) + +Based on load tests 002–006, hitting OpenSearch Dashboards through ALB with PPL queries, _search, saved object loads, and service map queries. + +### Current Config: 3 OS Nodes + 3 OSD Replicas, ~1.7 days of data (~316 MB primary) + +| User Experience | Est. Concurrent Users (VUs) | p95 Latency | Throughput | +|----------------|---------------------------|-------------|------------| +| Excellent (< 200ms p95) | ~50 | < 200ms | ~50 req/s | +| Good (< 1s p95) | ~150–200 | < 1s | ~80 req/s | +| Acceptable (< 2s p95) | ~250–350 | < 2s | ~100 req/s | +| Degraded (< 5s p95) | ~500–700 | < 5s | ~120 req/s | +| Saturated | 1000 | **10.57s** | 143 req/s | +| Breaking (errors appear) | > 1000 (not yet found) | > 15s | — | + +### Projected: 7 Days of Data (~1.3 GB primary) + +With 4x more data, search queries scan more segments and use more heap. Expected impact: + +| User Experience | Est. Concurrent Users | Notes | +|----------------|----------------------|-------| +| Excellent (< 200ms p95) | ~30–40 | Larger indices = slower scans | +| Good (< 1s p95) | ~100–150 | Query cache helps for repeated queries | +| Acceptable (< 2s p95) | ~150–250 | JVM heap pressure increases | +| Saturated | ~500–700 | Heap at 80%+, GC pauses start | + +### Projected: 30 Days of Data (~5.6 GB primary) + +| User Experience | Est. Concurrent Users | Notes | +|----------------|----------------------|-------| +| Excellent (< 200ms p95) | ~15–25 | Need shard optimization | +| Good (< 1s p95) | ~50–100 | Need more JVM heap or nodes | +| Acceptable (< 2s p95) | ~100–150 | ISM rollover policies critical | +| Saturated | ~300–500 | Need dedicated search nodes | + +⚠️ **These are estimates** based on extrapolation from 1000 VU tests. Actual numbers depend on query complexity, time range selected, and index management policies. The 7-day and 30-day projections assume linear degradation which is optimistic — real degradation is often worse due to GC pressure and segment merge overhead. + +--- + +## Scaling Recommendations by User Count + +| Target Users | OpenSearch | OSD | EKS Nodes | Est. Monthly Cost | +|-------------|-----------|-----|-----------|-------------------| +| 10–50 | 1 node (4Gi, 2 CPU) | 1 replica | 2x m5.xlarge | ~$350 | +| 50–200 | 3 nodes (4Gi, 2 CPU) | 2 replicas | 3x m5.xlarge | ~$530 | +| 200–500 | 3 nodes (8Gi, 4 CPU) | 3 replicas | 4x m5.2xlarge | ~$1,100 | +| 500–1000 | 3 data + 2 search nodes | 3 replicas | 5x m5.2xlarge | ~$1,400 | +| 1000+ | 3 data + 3 search + 3 CM | 3+ replicas | 8x m5.2xlarge | ~$2,200 | + +--- + +## Load Test History + +| # | Date | Config | VUs | p95 | req/s | Bottleneck | +|---|------|--------|-----|-----|-------|-----------| +| 002 | 03-20 | 1 OS (direct, no OSD) | 300 | 16ms | 239 | None | +| 003 | 03-20 | 1 OS (direct, no OSD) | 1500 | 2.28s | 855 | OS CPU 99% | +| 004 | 03-20 | 1 OS + 1 OSD (ALB) | 1000 | 3s+ (broke) | ~0 | OSD 100m CPU | +| 005 | 03-20 | 1 OS + 3 OSD (ALB) | 1000 | 14.57s | 104 | OS CPU 99% | +| 006 | 03-20 | 3 OS + 3 OSD (ALB) | 1000 | 10.57s | 143 | Uneven shards | + +### Key Findings +1. **OSD is the first bottleneck** — default 100m CPU is unusable under load. Minimum 500m request, 2000m limit. +2. **OpenSearch single node saturates at ~100 concurrent dashboard users** through OSD. +3. **3 OS nodes improve throughput 37%** but shard distribution must be balanced. +4. **Data volume directly impacts capacity** — more data = slower queries = fewer concurrent users. +5. **Write/search contention** — continuous indexing from OTel Demo competes with search for CPU (Lucene segment refresh). + +### What We Haven't Tested Yet +- [ ] Balanced shard distribution (increase replica count) +- [ ] 7-day data volume impact +- [ ] Dedicated search nodes (requires remote store) +- [ ] Prometheus under concurrent PromQL load through OSD +- [ ] Browser-based load (real Chromium sessions) +- [ ] WAF impact on throughput From 6c3c38f2e7134483c0a1940d03cc97c8b4f0c40c Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 20 Mar 2026 16:11:16 -0700 Subject: [PATCH 59/64] docs: add AGENTS.md with full load testing procedures for reproducibility - Exact commands for uploading scripts, running tests, monitoring, retrieving results - Current deployment state and access points - k6 script details and known issues - Key learnings and gotchas discovered during testing - File structure and next steps --- load-testing/AGENTS.md | 255 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 255 insertions(+) create mode 100644 load-testing/AGENTS.md diff --git a/load-testing/AGENTS.md b/load-testing/AGENTS.md new file mode 100644 index 00000000..50c2eac8 --- /dev/null +++ b/load-testing/AGENTS.md @@ -0,0 +1,255 @@ +# AGENTS.md — Load Testing Procedures + +This document captures the exact procedures for reproducing load tests against the Observability Stack Helm deployment. Designed for AI coding assistants to execute without prior context. + +## Repository Context + +- Load testing files live in `load-testing/` within the `feat/helm-charts` worktree at `.worktrees/feat-helm-charts/` +- Helm chart is at `.worktrees/feat-helm-charts/charts/observability-stack/` +- Terraform for EKS cluster is at `.worktrees/feat-helm-charts/terraform/aws/` +- Terraform for EC2 load generator is at `load-testing/terraform/` +- Results are tracked in `load-testing/RESULTS.md`, sizing in `load-testing/SIZING.md` + +## Current Deployment State (as of 2026-03-20) + +### EKS Cluster +- Name: `observability-stack`, region: `us-west-2` +- 4x m5.xlarge nodes (4 vCPU, 16 GB each) +- Helm release: `obs-stack` in namespace `observability-stack` + +### Stack Configuration +| Component | Replicas | CPU (req/limit) | Memory (req/limit) | +|-----------|----------|-----------------|-------------------| +| OpenSearch | 3 (StatefulSet) | 1000m/2000m | 4Gi/4Gi (JVM: 2Gi) | +| OpenSearch Dashboards | 3 | 500m/2000m | 1Gi/2Gi | +| OTel Collector | 1 | none | none | +| Data Prepper | 2 | none | none | +| Prometheus | 1 | none | none | +| OTel Demo | enabled | ~20 microservices | built-in load generator | + +### Access Points +- Dashboards ALB: `https://obs-playground-dev-027423573553.kylhouns.people.aws.dev` +- Credentials: `admin` / `My_password_123!@#` +- DNS configured via terraform at `.worktrees/feat-helm-charts/terraform/aws/terraform.tfvars` + +### EC2 Load Generator +- Instance: `i-08f9652631fe73302` (m5.xlarge, same VPC) +- Access: SSM only (no SSH key) +- k6 installed at `/usr/local/bin/k6` (v1.0.0) +- Scripts at `/home/ec2-user/k6/scenarios/` +- Results at `/home/ec2-user/k6/results/` +- Terraform state at `load-testing/terraform/` + +## Procedures + +### 1. Upload k6 Scripts to EC2 + +```bash +INSTANCE_ID="i-08f9652631fe73302" +SCRIPT=$(cat load-testing/k6/scenarios/api-queries-alb.js | base64) +aws ssm send-command \ + --instance-ids "$INSTANCE_ID" \ + --document-name "AWS-RunShellScript" \ + --parameters "commands=[\"echo '$SCRIPT' | base64 -d > /home/ec2-user/k6/scenarios/api-queries-alb.js\"]" \ + --region us-west-2 --output text --query 'Command.CommandId' +``` + +### 2. Run a Load Test + +```bash +INSTANCE_ID="i-08f9652631fe73302" +TARGET_URL="https://obs-playground-dev-027423573553.kylhouns.people.aws.dev" +VUS=1000 +TEST_NUM=007 + +CMD_ID=$(aws ssm send-command \ + --instance-ids "$INSTANCE_ID" \ + --document-name "AWS-RunShellScript" \ + --parameters "commands=[\"cd /home/ec2-user/k6 && k6 run --env TARGET_VUS=$VUS --env DASHBOARDS_URL=$TARGET_URL --env OSD_USER=admin --env OSD_PASSWORD='My_password_123!@#' scenarios/api-queries-alb.js 2>&1 | tee results/test-${TEST_NUM}.log\"]" \ + --timeout-seconds 1200 \ + --region us-west-2 \ + --output text --query 'Command.CommandId') +echo "Command: $CMD_ID" +``` + +### 3. Monitor During Test + +```bash +# Check test status +aws ssm get-command-invocation --command-id "$CMD_ID" --instance-id "$INSTANCE_ID" --region us-west-2 --query 'Status' --output text + +# Monitor OpenSearch nodes +kubectl exec -n observability-stack opensearch-cluster-master-0 -- curl -sk -u admin:'My_password_123!@#' \ + 'https://localhost:9200/_cat/nodes?h=name,heap.percent,cpu,load_1m,search.query_total,search.query_current' + +# Thread pool pressure +kubectl exec -n observability-stack opensearch-cluster-master-0 -- curl -sk -u admin:'My_password_123!@#' \ + 'https://localhost:9200/_cat/thread_pool/search?v&h=name,node_name,active,queue,rejected' + +# Hot threads (what's consuming CPU) +kubectl exec -n observability-stack opensearch-cluster-master-0 -- curl -sk -u admin:'My_password_123!@#' \ + 'https://localhost:9200/_nodes/hot_threads?threads=3' + +# JVM and OS stats +kubectl exec -n observability-stack opensearch-cluster-master-0 -- curl -sk -u admin:'My_password_123!@#' \ + 'https://localhost:9200/_nodes/stats/jvm,os?pretty' +``` + +### 4. Retrieve Results + +SSM truncates long output. Always read from the log file: + +```bash +RESULT_CMD=$(aws ssm send-command \ + --instance-ids "$INSTANCE_ID" \ + --document-name "AWS-RunShellScript" \ + --parameters "commands=[\"tail -35 /home/ec2-user/k6/results/test-${TEST_NUM}.log\"]" \ + --region us-west-2 --output text --query 'Command.CommandId') +sleep 5 +aws ssm get-command-invocation --command-id "$RESULT_CMD" --instance-id "$INSTANCE_ID" --region us-west-2 --query 'StandardOutputContent' --output text +``` + +### 5. Record Results + +Create `load-testing/results/NNN-description.md` with: +- Start/end timestamps (UTC and PDT) +- Configuration at time of test +- k6 summary (p50, p90, p95, max, error rate, req/s) +- Cluster observations during test (CPU, heap, thread pool, queue depths) +- Root cause analysis +- Next steps + +Update `load-testing/RESULTS.md` index table and bottleneck progression. +Update `load-testing/SIZING.md` if capacity estimates change. + +### 6. Apply Configuration Changes + +```bash +# Edit values.yaml +vim .worktrees/feat-helm-charts/charts/observability-stack/values.yaml + +# Deploy +helm upgrade obs-stack .worktrees/feat-helm-charts/charts/observability-stack \ + -n observability-stack --reuse-values + +# Or override specific values +helm upgrade obs-stack .worktrees/feat-helm-charts/charts/observability-stack \ + -n observability-stack --reuse-values \ + --set opensearch.replicas=3 +``` + +### 7. Scale EKS Nodes + +```bash +NODEGROUP=$(aws eks list-nodegroups --cluster-name observability-stack --region us-west-2 --query 'nodegroups[0]' --output text) +aws eks update-nodegroup-config \ + --cluster-name observability-stack \ + --nodegroup-name "$NODEGROUP" \ + --scaling-config minSize=2,maxSize=5,desiredSize=4 \ + --region us-west-2 +``` + +### 8. Manage EC2 Load Generator + +```bash +# Create (from load-testing/terraform/) +cd load-testing/terraform && terraform init && terraform apply + +# Destroy when done +terraform destroy + +# SSM session +aws ssm start-session --target i-08f9652631fe73302 --region us-west-2 +``` + +### 9. Redeploy Dashboards (after changing saved queries/dashboard YAMLs) + +```bash +helm upgrade obs-stack .worktrees/feat-helm-charts/charts/observability-stack \ + -n observability-stack --reuse-values +# Init job runs automatically as post-install/post-upgrade hook +``` + +## k6 Script Details + +### api-queries-alb.js +The primary load test script. Hits OSD through ALB with a mix of: +- **30% PPL queries** on span index (`/api/ppl/search`) +- **20% PPL queries** on log index +- **20% OpenSearch DSL search** via console proxy (`/api/console/proxy?path=...&method=POST`) +- **15% Saved objects list** (`/api/saved_objects/_find?type=dashboard`) +- **15% Service map query** via console proxy + +Key env vars: +- `TARGET_VUS` — peak virtual users (default 200) +- `DASHBOARDS_URL` — ALB endpoint +- `OSD_USER` / `OSD_PASSWORD` — credentials + +Ramp stages: 0→25%→50%→100% (hold 3min) →0 over 15 minutes. + +### Known Script Issues +- Console proxy path (`/api/console/proxy?path=...&method=POST`) returns 400 for some queries — needs investigation +- Prometheus queries not yet routed through OSD (datasource proxy path TBD) +- `insecureSkipTLSVerify: true` required in options block (not per-request) +- Auth uses manual `Authorization: Basic ` header via `k6/encoding` module + +## Key Learnings + +### Bottleneck Discovery Order +1. **OSD (100m CPU)** — Node.js single-threaded, saturates immediately. Fix: 3 replicas, 2 CPU each. +2. **OpenSearch (single node, 4 vCPU)** — 99% CPU, search queue depth 34. Fix: 3 data nodes. +3. **Uneven shard distribution** — original indices have 1 primary shard, load concentrates on 2 of 3 nodes. Fix: increase replica count or reindex with more shards. +4. **Data volume** (not yet tested) — 7-day data projected to reduce capacity ~40%. + +### Important Gotchas +- `kubectl port-forward` is NOT a valid load test path — it bottlenecks at the tunnel, not the cluster. Always use EC2 in the same VPC hitting the ALB. +- OSD workspace IDs differ between internal cluster access and external port-forward. The init script uses the internal workspace ID. +- The opensearch-dashboards Helm subchart uses `replicaCount` not `replicas` for scaling. +- OpenSearch `singleNode: true` must be set to `false` when scaling to multiple nodes. +- SSM command output is truncated for long-running tests. Always `tee` to a log file and read from there. + +## File Structure + +``` +load-testing/ +├── AGENTS.md # This file — procedures for AI assistants +├── README.md # Load testing plan and approach +├── RESULTS.md # Test result index with bottleneck progression +├── SIZING.md # Capacity sizing chart and projections +├── results/ +│ ├── 001-api-queries-auth-bug.md +│ ├── 002-api-queries.md +│ ├── 003-api-queries-1500vu.md +│ ├── 004-alb-1000vu-osd-bottleneck.md +│ ├── 005-alb-1000vu-opensearch-bottleneck.md +│ └── 006-alb-1000vu-3node-opensearch.md +├── k6/ +│ ├── full-test.js # Combined API + browser test +│ └── scenarios/ +│ ├── api-queries.js # Direct OpenSearch/Prometheus (port-forward) +│ ├── api-queries-alb.js # Through ALB/OSD (EC2 → ALB) +│ ├── browser-traces.js # Chromium: trace analytics flow +│ ├── browser-discover.js # Chromium: discover + PPL +│ └── browser-metrics.js # Chromium: metric dashboards +├── osb/ +│ ├── run-osb.sh # OpenSearch Benchmark runner +│ ├── workload.json # Custom trace/log workload +│ └── index-settings.json # Index mappings +├── pipeline/ +│ └── run-telemetrygen.sh # OTLP pipeline throughput test +├── terraform/ +│ ├── main.tf # EC2 load generator +│ ├── terraform.tfvars # VPC/subnet/target config +│ └── .gitignore +└── run-remote.sh # Upload + run helper +``` + +## Next Steps (Pending) + +1. **Fix shard distribution** — increase replica count on span/log indices so all 3 nodes serve searches equally +2. **Run 300 VU test** — validate the "good experience" threshold estimate +3. **7-day data test** — let OTel Demo run for a week, then re-run 1000 VU test +4. **Dedicated search nodes** — set up remote store (S3) + search node role for production config +5. **Prometheus load** — route PromQL through OSD to test single-pod Prometheus under concurrent dashboard users +6. **WAF testing** — enable WAF on ALB, measure throughput impact +7. **Browser tests** — run k6 browser module for real Chromium sessions From d33c8639ce52ce2d18b3e75b4cb2cb810d0d5590 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 20 Mar 2026 16:35:14 -0700 Subject: [PATCH 60/64] =?UTF-8?q?results:=20Test=20007=20=E2=80=94=20balan?= =?UTF-8?q?ced=20shards:=20p95=206.32s=20(was=2010.57s),=20168=20req/s=20(?= =?UTF-8?q?was=20143)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - number_of_replicas=2 gives every node a copy of every shard - Node-2 went from 4k to 53k queries (12.6x improvement) - 62% throughput improvement over single-node baseline - Remaining bottleneck: primary shard routing preference on Node-0 --- load-testing/RESULTS.md | 4 +- load-testing/SIZING.md | 1 + .../results/007-alb-1000vu-balanced-shards.md | 65 +++++++++++++++++++ 3 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 load-testing/results/007-alb-1000vu-balanced-shards.md diff --git a/load-testing/RESULTS.md b/load-testing/RESULTS.md index cd6f8547..f66d8cd4 100644 --- a/load-testing/RESULTS.md +++ b/load-testing/RESULTS.md @@ -37,6 +37,7 @@ | 4 | ALB E2E (1000 VUs from EC2) | 2026-03-20 14:55–14:59 | 🔴 OSD saturated at 100m CPU, 3s+ latency | [004-alb-1000vu-osd-bottleneck.md](results/004-alb-1000vu-osd-bottleneck.md) | | 5 | ALB E2E (1000 VUs, 3x OSD 2CPU) | 2026-03-20 15:08–15:24 | ⚠️ OSD fixed, OpenSearch at 99% CPU, p95=14.57s | [005-alb-1000vu-opensearch-bottleneck.md](results/005-alb-1000vu-opensearch-bottleneck.md) | | 6 | ALB E2E (1000 VUs, 3x OS nodes) | 2026-03-20 15:47–16:02 | ⚠️ 37% better throughput, p95=10.57s, uneven shards | [006-alb-1000vu-3node-opensearch.md](results/006-alb-1000vu-3node-opensearch.md) | +| 7 | ALB E2E (1000 VUs, balanced shards) | 2026-03-20 16:18–16:33 | ⚠️ p95=6.32s (+40%), 168 req/s (+62% from baseline) | [007-alb-1000vu-balanced-shards.md](results/007-alb-1000vu-balanced-shards.md) | ## Bottleneck Progression @@ -44,4 +45,5 @@ |------|-----------|-------------|--------| | 004 | OSD (100m CPU, 1 replica) | Scaled to 3 replicas, 2 CPU each | ✅ Resolved | | 005 | OpenSearch (single node, 4 vCPU, 99% CPU) | Scaled to 3 nodes, 2 CPU / 4Gi each | ✅ Improved | -| 006 | Uneven shard distribution across 3 nodes | **Next: rebalance shards, increase replica count** | Pending | +| 006 | Uneven shard distribution across 3 nodes | Set number_of_replicas=2 | ✅ Improved | +| 007 | Primary shard routing preference (Node-0 overloaded) | **Next: search routing or dedicated search nodes** | Pending | diff --git a/load-testing/SIZING.md b/load-testing/SIZING.md index d23b9158..c9db48d7 100644 --- a/load-testing/SIZING.md +++ b/load-testing/SIZING.md @@ -85,6 +85,7 @@ With 4x more data, search queries scan more segments and use more heap. Expected | 004 | 03-20 | 1 OS + 1 OSD (ALB) | 1000 | 3s+ (broke) | ~0 | OSD 100m CPU | | 005 | 03-20 | 1 OS + 3 OSD (ALB) | 1000 | 14.57s | 104 | OS CPU 99% | | 006 | 03-20 | 3 OS + 3 OSD (ALB) | 1000 | 10.57s | 143 | Uneven shards | +| 007 | 03-20 | 3 OS + 3 OSD, 2 replicas | 1000 | **6.32s** | **168** | Primary routing | ### Key Findings 1. **OSD is the first bottleneck** — default 100m CPU is unusable under load. Minimum 500m request, 2000m limit. diff --git a/load-testing/results/007-alb-1000vu-balanced-shards.md b/load-testing/results/007-alb-1000vu-balanced-shards.md new file mode 100644 index 00000000..e603b6cf --- /dev/null +++ b/load-testing/results/007-alb-1000vu-balanced-shards.md @@ -0,0 +1,65 @@ +# Test 007: ALB E2E — 1000 VUs, 3-Node OpenSearch, Balanced Shards + +**Status:** ✅ Significant improvement — p95 down 40% from Test 006 +**Script:** `k6/scenarios/api-queries-alb.js` +**Start:** 2026-03-20 16:18:26 PDT (23:18:26 UTC) +**End:** 2026-03-20 16:33:30 PDT (23:33:30 UTC) +**Duration:** 15m02s +**Source:** EC2 m5.xlarge → ALB → 3x OSD → 3x OpenSearch + +## Configuration Change (from Test 006) +- Set `number_of_replicas: 2` on all data indices (was 1) +- Every node now has a copy of every shard (3 copies total) +- No other changes + +## Comparison + +| Metric | Test 005 (1 node) | Test 006 (3 nodes, 1 replica) | Test 007 (3 nodes, 2 replicas) | Delta 006→007 | +|--------|-------------------|-------------------------------|-------------------------------|---------------| +| Total requests | 93,912 | 129,083 | **152,035** | +18% ✅ | +| Requests/sec | 104 | 143 | **168** | +18% ✅ | +| p(50) | 824ms | 1.1s | **1.43s** | worse ⚠️ | +| p(90) | 13.89s | 7.09s | **4.92s** | 31% better ✅ | +| p(95) | 14.57s | 10.57s | **6.32s** | 40% better ✅ | +| max | 16.95s | 18.12s | 60s (1 timeout) | ⚠️ | +| Error rate | 0% | 0% | **0.00%** (1 of 152k) | ✅ | + +## Cumulative Improvement (Test 005 → 007) + +| Metric | Test 005 | Test 007 | Improvement | +|--------|----------|----------|-------------| +| Throughput | 104 req/s | **168 req/s** | **+62%** | +| p(95) | 14.57s | **6.32s** | **57% faster** | +| p(90) | 13.89s | **4.92s** | **65% faster** | + +## Node Distribution During Test + +| Node | Search Queries | Load 1m (peak) | Heap % Range | +|------|---------------|----------------|-------------| +| master-0 (primaries) | 98,771 | **8.0** | 36-78% | +| master-1 | 129,613 | **2.1** | 40-76% | +| master-2 | 52,898 | **5.8** | 35-77% | + +**Improvement over Test 006:** Node-2 went from 4,188 → 52,898 queries (12.6x more). Load is much better distributed but still not perfectly even — Node-0 still gets primary preference. + +## Analysis + +- **40% p95 improvement** from just adding replicas — confirms shard imbalance was a major factor +- **168 req/s** throughput — 62% better than single-node baseline +- **Node-0 still overloaded** (load_1m=8) — it holds all primary shards and gets routing preference +- **1 timeout** (60s max) — likely a single request that hit during a GC pause or segment merge +- **p50 got slightly worse** (1.43s vs 1.1s) — more replication overhead, but tail latency improved significantly + +## Remaining Bottleneck + +Node-0 is still doing disproportionate work because it holds all primary shards. OpenSearch's adaptive replica selection prefers primaries. Options: +1. Force reindex to spread primaries across nodes +2. Set `preference=_replica` on search requests to avoid primaries +3. Move to dedicated search nodes (production path) + +## Next Steps + +1. Try `preference=_replica` search routing to offload Node-0 +2. Consider increasing primary shard count for new indices (ISM template) +3. Run 300 VU test to find the "good experience" threshold +4. Let data accumulate to 7 days and re-test From 8d006374c040f28fe59be35db67bbc784df4ada9 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 20 Mar 2026 22:15:44 -0700 Subject: [PATCH 61/64] fix: bake all runtime config into values.yaml (never use --set for persistent config) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Ingress: HTTPS/443 with ACM cert, TLS 1.3, external-dns hostname - Health check: /app/login (unauthenticated, returns 200) - OSD: replicaCount (not replicas) — correct subchart key - OpenSearch: 4 CPU limit, 2 CPU request, 4Gi RAM, 2Gi JVM - Prometheus: 50Gi PV, 2Gi/4Gi memory, 500m/1000m CPU - OTel Demo: enabled in values.yaml - preference=_replica in k6 search queries Lesson: never use helm --reset-values or --set for config that should persist --- charts/observability-stack/values.yaml | 37 ++++++++++++++++---- load-testing/k6/scenarios/api-queries-alb.js | 4 +-- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/charts/observability-stack/values.yaml b/charts/observability-stack/values.yaml index f29ac2ab..f37c7af7 100644 --- a/charts/observability-stack/values.yaml +++ b/charts/observability-stack/values.yaml @@ -25,10 +25,10 @@ opensearch: resources: requests: memory: "4Gi" - cpu: "1000m" + cpu: "2000m" limits: memory: "4Gi" - cpu: "2000m" + cpu: "4000m" persistence: enabled: true size: 8Gi # Increase for production (e.g. 100Gi, 500Gi) @@ -49,7 +49,7 @@ opensearch: # -- OpenSearch Dashboards opensearch-dashboards: enabled: true - replicas: 3 + replicaCount: 3 image: repository: "opensearchstaging/opensearch-dashboards" tag: "3.6.0" @@ -60,6 +60,24 @@ opensearch-dashboards: limits: cpu: "2000m" memory: "2Gi" + ingress: + enabled: true + ingressClassName: alb + annotations: + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/target-type: ip + alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}]' + alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:027423573553:certificate/489eff8e-d8b5-4ec3-a924-9768f518a49d + alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS13-1-2-2021-06 + alb.ingress.kubernetes.io/healthcheck-path: /app/login + alb.ingress.kubernetes.io/success-codes: "200" + external-dns.alpha.kubernetes.io/hostname: obs-playground-dev-027423573553.kylhouns.people.aws.dev + hosts: + - host: "" + paths: + - path: / + backend: + servicePort: 5601 opensearchHosts: "https://opensearch-cluster-master:9200" config: opensearch_dashboards.yml: | @@ -325,8 +343,15 @@ prometheus: # Retention — how long Prometheus keeps metrics. Increase for longer history. retention: "15d" persistentVolume: - enabled: false # Enable for production (survives pod restarts) - # size: 50Gi + enabled: true + size: 50Gi + resources: + requests: + memory: "2Gi" + cpu: "500m" + limits: + memory: "4Gi" + cpu: "1000m" extraFlags: - "web.enable-remote-write-receiver" - "web.enable-otlp-receiver" @@ -450,7 +475,7 @@ examples: # All bundled backends (jaeger, grafana, prometheus, opensearch, collector) are disabled — # demo services send telemetry to our OTel Collector instead. opentelemetry-demo: - enabled: false + enabled: true default: envOverrides: - name: OTEL_COLLECTOR_NAME diff --git a/load-testing/k6/scenarios/api-queries-alb.js b/load-testing/k6/scenarios/api-queries-alb.js index a23abc66..4338386c 100644 --- a/load-testing/k6/scenarios/api-queries-alb.js +++ b/load-testing/k6/scenarios/api-queries-alb.js @@ -87,7 +87,7 @@ export function osdLoad() { } else if (action < 0.7) { // Direct OpenSearch query through OSD (DSL search) - const res = http.post(`${OSD}/api/console/proxy?path=${encodeURIComponent('otel-v1-apm-span-*/_search')}&method=POST`, JSON.stringify({ + const res = http.post(`${OSD}/api/console/proxy?path=${encodeURIComponent('otel-v1-apm-span-*/_search?preference=_replica')}&method=POST`, JSON.stringify({ size: 50, query: { match_all: {} }, sort: [{ startTime: 'desc' }], @@ -104,7 +104,7 @@ export function osdLoad() { } else { // Service map query - const res = http.post(`${OSD}/api/console/proxy?path=${encodeURIComponent('otel-v2-apm-service-map-*/_search')}&method=POST`, JSON.stringify({ + const res = http.post(`${OSD}/api/console/proxy?path=${encodeURIComponent('otel-v2-apm-service-map-*/_search?preference=_replica')}&method=POST`, JSON.stringify({ size: 200, query: { match_all: {} }, }), { headers }); From 7fb5e73d96657b2db089abb3e2a4aae0b12ce3ce Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 20 Mar 2026 22:34:03 -0700 Subject: [PATCH 62/64] feat: production EKS sizing (3-node OS, persistent Prometheus, otel-demo) --- terraform/aws/values-eks.yaml | 38 +++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/terraform/aws/values-eks.yaml b/terraform/aws/values-eks.yaml index 4febb834..32aa76e1 100644 --- a/terraform/aws/values-eks.yaml +++ b/terraform/aws/values-eks.yaml @@ -8,10 +8,30 @@ # gp2 is the default EBS StorageClass on EKS. Without this, OpenSearch PVCs # stay Pending because the chart default ("") doesn't match any provisioner. opensearch: + singleNode: false + replicas: 3 persistence: storageClass: gp2 + resources: + requests: + memory: "4Gi" + cpu: "2000m" + limits: + memory: "4Gi" + cpu: "4000m" + extraEnvs: + - name: OPENSEARCH_JAVA_OPTS + value: "-Xms2g -Xmx2g" opensearch-dashboards: + replicaCount: 3 + resources: + requests: + cpu: "500m" + memory: "1Gi" + limits: + cpu: "2000m" + memory: "2Gi" ingress: enabled: true ingressClassName: alb @@ -30,6 +50,24 @@ opensearch-dashboards: examples: enabled: true +# Prometheus — persistent storage + resource limits +prometheus: + server: + persistentVolume: + enabled: true + size: 50Gi + resources: + requests: + memory: "2Gi" + cpu: "500m" + limits: + memory: "4Gi" + cpu: "1000m" + +# OTel Demo — realistic telemetry from ~20 microservices +opentelemetry-demo: + enabled: true + # Gateway not needed — ALB handles ingress directly. gateway: enabled: false From 86baf96cdf0882d883a12300b0f82f45f841b02e Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 20 Mar 2026 22:59:37 -0700 Subject: [PATCH 63/64] Revert "Merge pull request #8 from kylehounslow/feat/helm-anon-auth" This reverts commit 00169ec7a4a7ac609c17c1acc1a304e4cb3f1020, reversing changes made to d33c8639ce52ce2d18b3e75b4cb2cb810d0d5590. --- charts/observability-stack/README.md | 28 ----- .../files/init-opensearch-dashboards.py | 3 +- .../templates/init-dashboards-job.yaml | 2 - .../templates/opensearch-security-config.yaml | 103 ------------------ .../tests/anonymous_auth_test.yaml | 58 ---------- charts/observability-stack/values.yaml | 23 ---- terraform/aws/observability-stack.tf | 16 --- 7 files changed, 1 insertion(+), 232 deletions(-) delete mode 100644 charts/observability-stack/templates/opensearch-security-config.yaml delete mode 100644 charts/observability-stack/tests/anonymous_auth_test.yaml diff --git a/charts/observability-stack/README.md b/charts/observability-stack/README.md index c99fd644..d46c830c 100644 --- a/charts/observability-stack/README.md +++ b/charts/observability-stack/README.md @@ -162,10 +162,6 @@ Sources: [OpenSearch shard sizing](https://opensearch.org/blog/optimize-opensear See `values.yaml` for all options. Notable settings: ```yaml -# Anonymous auth — skip login page for demos/workshops -anonymousAuth: - enabled: false # Set true to allow access without credentials - # Credentials (update opensearchPassword before any real deployment) opensearchUsername: "admin" opensearchPassword: "My_password_123!@#" @@ -183,30 +179,6 @@ prometheus: # ... etc ``` -## Anonymous Authentication - -By default, OpenSearch Dashboards requires login. Enable anonymous auth to skip the login page — useful for demos, workshops, or shared dev environments. - -```bash -helm install obs-stack charts/observability-stack \ - --set anonymousAuth.enabled=true \ - --set global.anonymousAuth.enabled=true -``` - -> **Note:** Both `anonymousAuth.enabled` and `global.anonymousAuth.enabled` must be set. The `global` value is needed because the OpenSearch Dashboards subchart config uses Go templating and can only access global values. - -**What anonymous users can do:** -- Browse all data (logs, traces, metrics) -- View, create, and modify saved objects (visualizations, dashboards, saved queries) -- Explore traces and service maps -- Run queries and access the OpenSearch REST API - -**What anonymous users cannot do:** -- Delete existing saved objects -- Perform admin operations (user management, security config) - -**To disable:** Remove the `--set` flags (or set both to `false`) and redeploy. - ## OpenTelemetry Demo (Optional) The [OpenTelemetry Demo](https://opentelemetry.io/docs/demo/) is available as an optional subchart. It deploys a full microservices e-commerce app (20+ services) that generates realistic telemetry — useful for load testing and showcasing the stack. diff --git a/charts/observability-stack/files/init-opensearch-dashboards.py b/charts/observability-stack/files/init-opensearch-dashboards.py index 76b05103..2e05d7ca 100644 --- a/charts/observability-stack/files/init-opensearch-dashboards.py +++ b/charts/observability-stack/files/init-opensearch-dashboards.py @@ -15,7 +15,6 @@ PROMETHEUS_PORT = os.getenv("PROMETHEUS_PORT", "9090") _opensearch_protocol = os.getenv("OPENSEARCH_PROTOCOL", "https") OPENSEARCH_ENDPOINT = f"{_opensearch_protocol}://{os.getenv('OPENSEARCH_HOST', 'opensearch')}:{os.getenv('OPENSEARCH_PORT', '9200')}" -ANONYMOUS_AUTH_ENABLED = os.getenv("OPENSEARCH_ANONYMOUS_AUTH_ENABLED", "false").lower() == "true" def wait_for_dashboards(): """Wait for OpenSearch Dashboards to be ready""" @@ -233,7 +232,7 @@ def create_prometheus_datasource(workspace_id): payload = { "name": datasource_name, - "allowedRoles": ["all_access", "opendistro_security_anonymous_role"] if ANONYMOUS_AUTH_ENABLED else ["all_access"], + "allowedRoles": [], "connector": "prometheus", "properties": { "prometheus.uri": prometheus_endpoint, diff --git a/charts/observability-stack/templates/init-dashboards-job.yaml b/charts/observability-stack/templates/init-dashboards-job.yaml index 6b646b3b..3c957106 100644 --- a/charts/observability-stack/templates/init-dashboards-job.yaml +++ b/charts/observability-stack/templates/init-dashboards-job.yaml @@ -44,8 +44,6 @@ spec: value: "80" - name: OPENSEARCH_ENDPOINT value: "https://opensearch-cluster-master:9200" - - name: OPENSEARCH_ANONYMOUS_AUTH_ENABLED - value: {{ .Values.anonymousAuth.enabled | quote }} volumeMounts: - name: init-script mountPath: /scripts diff --git a/charts/observability-stack/templates/opensearch-security-config.yaml b/charts/observability-stack/templates/opensearch-security-config.yaml deleted file mode 100644 index 6c42faff..00000000 --- a/charts/observability-stack/templates/opensearch-security-config.yaml +++ /dev/null @@ -1,103 +0,0 @@ -{{- if index .Values "opensearch" "enabled" }} -apiVersion: v1 -kind: Secret -metadata: - name: opensearch-security-config - labels: - {{- include "observability-stack.labels" . | nindent 4 }} -type: Opaque -stringData: - config.yml: | - _meta: - type: "config" - config_version: 2 - config: - dynamic: - http: - anonymous_auth_enabled: {{ .Values.anonymousAuth.enabled }} - xff: - enabled: false - internalProxies: "192\\.168\\.0\\.10|192\\.168\\.0\\.11" - authc: - basic_internal_auth_domain: - description: "Authenticate via HTTP Basic against internal users database" - http_enabled: true - transport_enabled: true - order: 4 - http_authenticator: - type: "basic" - challenge: true - authentication_backend: - type: "intern" - roles.yml: | - _meta: - type: "roles" - config_version: 2 - opendistro_security_anonymous_role: - reserved: true - cluster_permissions: - - "read" - - "cluster_monitor" - - "cluster_composite_ops" - - "indices:data/read/scroll*" - - "cluster:admin/opensearch/ppl" - - "cluster:admin/opensearch/sql" - - "cluster:admin/opensearch/ql/datasources/read" - - "cluster:admin/opensearch/ql/async_query/read" - - "cluster:admin/opensearch/direct_query/read/query" - index_permissions: - - index_patterns: - - ".kibana" - - ".kibana-6" - - ".kibana_*" - - ".opensearch_dashboards" - - ".opensearch_dashboards-6" - - ".opensearch_dashboards_*" - allowed_actions: - - "read" - - "indices:data/write/index*" - - "indices:data/write/update*" - - "indices:data/write/bulk*" - - index_patterns: - - ".tasks" - - ".management-beats" - - "*:.tasks" - - "*:.management-beats" - allowed_actions: - - "read" - - index_patterns: - - '*' - allowed_actions: - - "read" - - "indices:data/read/*" - - "indices:admin/get" - - "indices:admin/exists" - - "indices:admin/aliases/exists*" - - "indices:admin/aliases/get*" - - "indices:admin/mappings/get" - - "indices:admin/resolve/index" - - "indices:monitor/settings/get" - - "indices:monitor/stats" - tenant_permissions: - - tenant_patterns: - - '*' - allowed_actions: - - "kibana_all_write" - roles_mapping.yml: | - _meta: - type: "rolesmapping" - config_version: 2 - opendistro_security_anonymous_role: - backend_roles: - - "opendistro_security_anonymous_backendrole" - all_access: - reserved: true - backend_roles: - - "admin" - description: "Maps admin to all_access" - kibana_server: - reserved: true - users: - - "kibanaserver" - description: "Maps kibana_server role to kibanaserver user" -{{- end }} diff --git a/charts/observability-stack/tests/anonymous_auth_test.yaml b/charts/observability-stack/tests/anonymous_auth_test.yaml deleted file mode 100644 index 2c1e7f94..00000000 --- a/charts/observability-stack/tests/anonymous_auth_test.yaml +++ /dev/null @@ -1,58 +0,0 @@ -suite: anonymous authentication -templates: - - templates/opensearch-security-config.yaml - - templates/init-dashboards-job.yaml -tests: - # --- Security config Secret --- - - it: should set anonymous_auth_enabled to false by default - template: templates/opensearch-security-config.yaml - asserts: - - isKind: - of: Secret - - matchRegex: - path: stringData["config.yml"] - pattern: "anonymous_auth_enabled: false" - - - it: should set anonymous_auth_enabled to true when enabled - template: templates/opensearch-security-config.yaml - set: - anonymousAuth.enabled: true - asserts: - - matchRegex: - path: stringData["config.yml"] - pattern: "anonymous_auth_enabled: true" - - - it: should always include anonymous role definition - template: templates/opensearch-security-config.yaml - asserts: - - matchRegex: - path: stringData["roles.yml"] - pattern: "opendistro_security_anonymous_role" - - - it: should always include anonymous role mapping - template: templates/opensearch-security-config.yaml - asserts: - - matchRegex: - path: stringData["roles_mapping.yml"] - pattern: "opendistro_security_anonymous_backendrole" - - # --- Init job env var --- - - it: should pass OPENSEARCH_ANONYMOUS_AUTH_ENABLED=false by default - template: templates/init-dashboards-job.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: OPENSEARCH_ANONYMOUS_AUTH_ENABLED - value: "false" - - - it: should pass OPENSEARCH_ANONYMOUS_AUTH_ENABLED=true when enabled - template: templates/init-dashboards-job.yaml - set: - anonymousAuth.enabled: true - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: OPENSEARCH_ANONYMOUS_AUTH_ENABLED - value: "true" diff --git a/charts/observability-stack/values.yaml b/charts/observability-stack/values.yaml index f37c7af7..a0f1e76a 100644 --- a/charts/observability-stack/values.yaml +++ b/charts/observability-stack/values.yaml @@ -1,13 +1,6 @@ # Default values for observability-stack umbrella chart # Mirrors the docker-compose setup for Kubernetes deployment -# -- Anonymous authentication (skip login page for demos/workshops) -# When enabled, users can access OpenSearch Dashboards without logging in. -# Anonymous users can browse data, create/modify saved objects, but cannot -# delete existing saved objects or perform admin operations. -anonymousAuth: - enabled: false - # -- OpenSearch # Sizing guide: # Storage: daily_ingest_GB × 1.45 × (replicas + 1) × retention_days @@ -42,9 +35,6 @@ opensearch: config: opensearch.yml: | plugins.query.datasources.encryption.masterkey: "BTqK4Ytdz67La1kShIKV3Pu9" - securityConfig: - config: - securityConfigSecret: "opensearch-security-config" # -- OpenSearch Dashboards opensearch-dashboards: @@ -91,7 +81,6 @@ opensearch-dashboards: opensearch.requestHeadersAllowlist: ["authorization", "securitytenant"] opensearch_security.multitenancy.enabled: false opensearch_security.readonly_mode.roles: ["kibana_read_only"] - opensearch_security.auth.anonymous_auth_enabled: {{ .Values.global.anonymousAuth.enabled }} console.enabled: true server.maxPayloadBytes: 1048576 savedObjects.maxImportPayloadBytes: 26214400 @@ -101,11 +90,6 @@ opensearch-dashboards: explore.discoverMetrics.enabled: true explore.agentTraces.enabled: true workspace.enabled: true - {{- if .Values.global.anonymousAuth.enabled }} - savedObjects.permission.enabled: false - {{- else }} - savedObjects.permission.enabled: true - {{- end }} data_source.enabled: true data_source.ssl.verificationMode: none datasetManagement.enabled: true @@ -522,10 +506,3 @@ gateway: # host: dashboards.example.com # annotations: # application-networking.k8s.aws/certificate-arn: arn:aws:acm:REGION:ACCOUNT:certificate/ID - -# -- Global values (accessible to all subcharts via .Values.global) -# Used to pass anonymousAuth.enabled to opensearch-dashboards subchart config -# which uses tpl() for Go template rendering. -global: - anonymousAuth: - enabled: false diff --git a/terraform/aws/observability-stack.tf b/terraform/aws/observability-stack.tf index 236c15d9..cc4e20fc 100644 --- a/terraform/aws/observability-stack.tf +++ b/terraform/aws/observability-stack.tf @@ -163,22 +163,6 @@ resource "helm_release" "observability_stack" { } } - # --- Anonymous auth (conditional) --- - dynamic "set" { - for_each = var.anonymous_auth ? [1] : [] - content { - name = "anonymousAuth.enabled" - value = "true" - } - } - dynamic "set" { - for_each = var.anonymous_auth ? [1] : [] - content { - name = "global.anonymousAuth.enabled" - value = "true" - } - } - depends_on = [ helm_release.aws_lb_controller, ] From 6be852abc4c483d8d265e9f004fb7bf4c3e8d175 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 20 Mar 2026 23:00:36 -0700 Subject: [PATCH 64/64] fix: revert anon auth PR, add missing admin password, disable examples/demo for clean deploy --- .../aws/terraform.tfstate.1774071781.backup | 8632 +++++++++++++++++ terraform/aws/values-eks.yaml | 6 +- 2 files changed, 8636 insertions(+), 2 deletions(-) create mode 100644 terraform/aws/terraform.tfstate.1774071781.backup diff --git a/terraform/aws/terraform.tfstate.1774071781.backup b/terraform/aws/terraform.tfstate.1774071781.backup new file mode 100644 index 00000000..230125db --- /dev/null +++ b/terraform/aws/terraform.tfstate.1774071781.backup @@ -0,0 +1,8632 @@ +{ + "version": 4, + "terraform_version": "1.5.7", + "serial": 120, + "lineage": "704c9890-077b-244e-fa87-318acab97ff8", + "outputs": { + "cluster_name": { + "value": "observability-stack", + "type": "string" + }, + "credentials": { + "value": "Username: admin | Password: My_password_123!@#", + "type": "string" + }, + "dashboards_url": { + "value": "https://obs-playground-dev-027423573553.kylhouns.people.aws.dev", + "type": "string" + }, + "kubeconfig_command": { + "value": "aws eks update-kubeconfig --name observability-stack --region us-west-2", + "type": "string" + }, + "next_steps": { + "value": "✅ TLS enabled at https://obs-playground-dev-027423573553.kylhouns.people.aws.dev", + "type": "string" + }, + "otlp_endpoint": { + "value": "kubectl port-forward -n observability-stack svc/obs-stack-opentelemetry-collector 4317:4317 4318:4318", + "type": "string" + } + }, + "resources": [ + { + "mode": "data", + "type": "aws_availability_zones", + "name": "available", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "all_availability_zones": null, + "exclude_names": null, + "exclude_zone_ids": null, + "filter": null, + "group_names": [ + "us-west-2-zg-1" + ], + "id": "us-west-2", + "names": [ + "us-west-2a", + "us-west-2b", + "us-west-2c", + "us-west-2d" + ], + "state": "available", + "timeouts": null, + "zone_ids": [ + "usw2-az2", + "usw2-az1", + "usw2-az3", + "usw2-az4" + ] + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "managed", + "type": "aws_acm_certificate", + "name": "dashboards", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:acm:us-west-2:027423573553:certificate/f792348a-14b1-474b-9e46-98e1a1091485", + "certificate_authority_arn": "", + "certificate_body": null, + "certificate_chain": null, + "domain_name": "obs-playground-dev-027423573553.kylhouns.people.aws.dev", + "domain_validation_options": [ + { + "domain_name": "obs-playground-dev-027423573553.kylhouns.people.aws.dev", + "resource_record_name": "_f30b5b31581053b9f97ac519286c87b2.obs-playground-dev-027423573553.kylhouns.people.aws.dev.", + "resource_record_type": "CNAME", + "resource_record_value": "_4ae4baccc596f519b264e75bb6f73d23.jkddzztszm.acm-validations.aws." + } + ], + "early_renewal_duration": "", + "id": "arn:aws:acm:us-west-2:027423573553:certificate/f792348a-14b1-474b-9e46-98e1a1091485", + "key_algorithm": "RSA_2048", + "not_after": "2026-10-04T23:59:59Z", + "not_before": "2026-03-21T00:00:00Z", + "options": [ + { + "certificate_transparency_logging_preference": "ENABLED" + } + ], + "pending_renewal": false, + "private_key": null, + "renewal_eligibility": "INELIGIBLE", + "renewal_summary": [], + "status": "ISSUED", + "subject_alternative_names": [ + "obs-playground-dev-027423573553.kylhouns.people.aws.dev" + ], + "tags": {}, + "tags_all": { + "ManagedBy": "terraform", + "Project": "observability-stack" + }, + "type": "AMAZON_ISSUED", + "validation_emails": [], + "validation_method": "DNS", + "validation_option": [] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "create_before_destroy": true + } + ] + }, + { + "mode": "managed", + "type": "aws_acm_certificate_validation", + "name": "dashboards", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "certificate_arn": "arn:aws:acm:us-west-2:027423573553:certificate/f792348a-14b1-474b-9e46-98e1a1091485", + "id": "2026-03-21 05:26:13.57 +0000 UTC", + "timeouts": null, + "validation_record_fqdns": [ + "_f30b5b31581053b9f97ac519286c87b2.obs-playground-dev-027423573553.kylhouns.people.aws.dev" + ] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo0NTAwMDAwMDAwMDAwfX0=", + "dependencies": [ + "aws_acm_certificate.dashboards", + "aws_route53_record.cert_validation" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_route53_record", + "name": "cert_validation", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": "obs-playground-dev-027423573553.kylhouns.people.aws.dev", + "schema_version": 2, + "attributes": { + "alias": [], + "allow_overwrite": true, + "cidr_routing_policy": [], + "failover_routing_policy": [], + "fqdn": "_f30b5b31581053b9f97ac519286c87b2.obs-playground-dev-027423573553.kylhouns.people.aws.dev", + "geolocation_routing_policy": [], + "geoproximity_routing_policy": [], + "health_check_id": "", + "id": "Z07457962NTDHS0A6G9LH__f30b5b31581053b9f97ac519286c87b2.obs-playground-dev-027423573553.kylhouns.people.aws.dev._CNAME", + "latency_routing_policy": [], + "multivalue_answer_routing_policy": false, + "name": "_f30b5b31581053b9f97ac519286c87b2.obs-playground-dev-027423573553.kylhouns.people.aws.dev", + "records": [ + "_4ae4baccc596f519b264e75bb6f73d23.jkddzztszm.acm-validations.aws." + ], + "set_identifier": "", + "timeouts": null, + "ttl": 60, + "type": "CNAME", + "weighted_routing_policy": [], + "zone_id": "Z07457962NTDHS0A6G9LH" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjE4MDAwMDAwMDAwMDAsInVwZGF0ZSI6MTgwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMiJ9", + "dependencies": [ + "aws_acm_certificate.dashboards" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_wafv2_web_acl", + "name": "rate_limit", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "mode": "managed", + "type": "helm_release", + "name": "aws_lb_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/helm\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "atomic": false, + "chart": "aws-load-balancer-controller", + "cleanup_on_fail": false, + "create_namespace": false, + "dependency_update": false, + "description": null, + "devel": null, + "disable_crd_hooks": false, + "disable_openapi_validation": false, + "disable_webhooks": false, + "force_update": false, + "id": "aws-load-balancer-controller", + "keyring": null, + "lint": false, + "manifest": null, + "max_history": 0, + "metadata": [ + { + "app_version": "v2.12.0", + "chart": "aws-load-balancer-controller", + "first_deployed": 1773866318, + "last_deployed": 1773866318, + "name": "aws-load-balancer-controller", + "namespace": "kube-system", + "notes": "AWS Load Balancer controller installed!\n", + "revision": 1, + "values": "{\"clusterName\":\"observability-stack\",\"region\":\"us-west-2\",\"serviceAccount\":{\"annotations\":{\"eks.amazonaws.com/role-arn\":\"arn:aws:iam::027423573553:role/observability-stack-lb-controller\"}},\"vpcId\":\"vpc-03aefcf5aa0581d7a\"}", + "version": "1.12.0" + } + ], + "name": "aws-load-balancer-controller", + "namespace": "kube-system", + "pass_credentials": false, + "postrender": [], + "recreate_pods": false, + "render_subchart_notes": true, + "replace": false, + "repository": "https://aws.github.io/eks-charts", + "repository_ca_file": null, + "repository_cert_file": null, + "repository_key_file": null, + "repository_password": null, + "repository_username": null, + "reset_values": false, + "reuse_values": false, + "set": [ + { + "name": "clusterName", + "type": "", + "value": "observability-stack" + }, + { + "name": "region", + "type": "", + "value": "us-west-2" + }, + { + "name": "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn", + "type": "", + "value": "arn:aws:iam::027423573553:role/observability-stack-lb-controller" + }, + { + "name": "vpcId", + "type": "", + "value": "vpc-03aefcf5aa0581d7a" + } + ], + "set_list": [], + "set_sensitive": [], + "skip_crds": false, + "status": "deployed", + "timeout": 300, + "upgrade_install": null, + "values": null, + "verify": false, + "version": "1.12.0", + "wait": true, + "wait_for_jobs": false + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "data.aws_availability_zones.available", + "module.ebs_csi_irsa.aws_iam_role.this", + "module.ebs_csi_irsa.data.aws_caller_identity.current", + "module.ebs_csi_irsa.data.aws_iam_policy_document.this", + "module.ebs_csi_irsa.data.aws_partition.current", + "module.eks.aws_cloudwatch_log_group.this", + "module.eks.aws_ec2_tag.cluster_primary_security_group", + "module.eks.aws_eks_access_entry.this", + "module.eks.aws_eks_access_policy_association.this", + "module.eks.aws_eks_addon.before_compute", + "module.eks.aws_eks_addon.this", + "module.eks.aws_eks_cluster.this", + "module.eks.aws_eks_identity_provider_config.this", + "module.eks.aws_iam_openid_connect_provider.oidc_provider", + "module.eks.aws_iam_policy.cluster_encryption", + "module.eks.aws_iam_policy.cni_ipv6_policy", + "module.eks.aws_iam_policy.custom", + "module.eks.aws_iam_role.eks_auto", + "module.eks.aws_iam_role.this", + "module.eks.aws_iam_role_policy_attachment.additional", + "module.eks.aws_iam_role_policy_attachment.cluster_encryption", + "module.eks.aws_iam_role_policy_attachment.custom", + "module.eks.aws_iam_role_policy_attachment.eks_auto", + "module.eks.aws_iam_role_policy_attachment.eks_auto_additional", + "module.eks.aws_iam_role_policy_attachment.this", + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.eks.aws_security_group_rule.cluster", + "module.eks.aws_security_group_rule.node", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_eks_addon_version.this", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_policy_document.cni_ipv6_policy", + "module.eks.data.aws_iam_policy_document.custom", + "module.eks.data.aws_iam_policy_document.node_assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.data.aws_partition.current", + "module.eks.data.tls_certificate.this", + "module.eks.module.eks_managed_node_group.aws_autoscaling_schedule.this", + "module.eks.module.eks_managed_node_group.aws_eks_node_group.this", + "module.eks.module.eks_managed_node_group.aws_iam_role.this", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy.this", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy_attachment.additional", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy_attachment.this", + "module.eks.module.eks_managed_node_group.aws_launch_template.this", + "module.eks.module.eks_managed_node_group.aws_placement_group.this", + "module.eks.module.eks_managed_node_group.data.aws_caller_identity.current", + "module.eks.module.eks_managed_node_group.data.aws_ec2_instance_type.this", + "module.eks.module.eks_managed_node_group.data.aws_ec2_instance_type_offerings.this", + "module.eks.module.eks_managed_node_group.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.eks_managed_node_group.data.aws_iam_policy_document.role", + "module.eks.module.eks_managed_node_group.data.aws_partition.current", + "module.eks.module.eks_managed_node_group.data.aws_ssm_parameter.ami", + "module.eks.module.eks_managed_node_group.data.aws_subnets.placement_group", + "module.eks.module.eks_managed_node_group.module.user_data.data.cloudinit_config.al2023_eks_managed_node_group", + "module.eks.module.eks_managed_node_group.module.user_data.data.cloudinit_config.linux_eks_managed_node_group", + "module.eks.module.eks_managed_node_group.module.user_data.null_resource.validate_cluster_service_cidr", + "module.eks.module.fargate_profile.aws_eks_fargate_profile.this", + "module.eks.module.fargate_profile.aws_iam_role.this", + "module.eks.module.fargate_profile.aws_iam_role_policy.this", + "module.eks.module.fargate_profile.aws_iam_role_policy_attachment.additional", + "module.eks.module.fargate_profile.aws_iam_role_policy_attachment.this", + "module.eks.module.fargate_profile.data.aws_caller_identity.current", + "module.eks.module.fargate_profile.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.fargate_profile.data.aws_iam_policy_document.role", + "module.eks.module.fargate_profile.data.aws_partition.current", + "module.eks.module.fargate_profile.data.aws_region.current", + "module.eks.module.kms.aws_kms_alias.this", + "module.eks.module.kms.aws_kms_external_key.this", + "module.eks.module.kms.aws_kms_grant.this", + "module.eks.module.kms.aws_kms_key.this", + "module.eks.module.kms.aws_kms_replica_external_key.this", + "module.eks.module.kms.aws_kms_replica_key.this", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current", + "module.eks.module.self_managed_node_group.aws_autoscaling_group.this", + "module.eks.module.self_managed_node_group.aws_autoscaling_schedule.this", + "module.eks.module.self_managed_node_group.aws_eks_access_entry.this", + "module.eks.module.self_managed_node_group.aws_iam_instance_profile.this", + "module.eks.module.self_managed_node_group.aws_iam_role.this", + "module.eks.module.self_managed_node_group.aws_iam_role_policy.this", + "module.eks.module.self_managed_node_group.aws_iam_role_policy_attachment.additional", + "module.eks.module.self_managed_node_group.aws_iam_role_policy_attachment.this", + "module.eks.module.self_managed_node_group.aws_launch_template.this", + "module.eks.module.self_managed_node_group.aws_placement_group.this", + "module.eks.module.self_managed_node_group.data.aws_caller_identity.current", + "module.eks.module.self_managed_node_group.data.aws_ec2_instance_type.this", + "module.eks.module.self_managed_node_group.data.aws_ec2_instance_type_offerings.this", + "module.eks.module.self_managed_node_group.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.self_managed_node_group.data.aws_iam_policy_document.role", + "module.eks.module.self_managed_node_group.data.aws_partition.current", + "module.eks.module.self_managed_node_group.data.aws_ssm_parameter.ami", + "module.eks.module.self_managed_node_group.data.aws_subnets.placement_group", + "module.eks.module.self_managed_node_group.module.user_data.data.cloudinit_config.al2023_eks_managed_node_group", + "module.eks.module.self_managed_node_group.module.user_data.data.cloudinit_config.linux_eks_managed_node_group", + "module.eks.module.self_managed_node_group.module.user_data.null_resource.validate_cluster_service_cidr", + "module.eks.time_sleep.this", + "module.lb_controller_irsa.aws_iam_role.this", + "module.lb_controller_irsa.data.aws_caller_identity.current", + "module.lb_controller_irsa.data.aws_iam_policy_document.this", + "module.lb_controller_irsa.data.aws_partition.current", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "mode": "managed", + "type": "helm_release", + "name": "external_dns", + "provider": "provider[\"registry.terraform.io/hashicorp/helm\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 1, + "attributes": { + "atomic": false, + "chart": "external-dns", + "cleanup_on_fail": false, + "create_namespace": false, + "dependency_update": false, + "description": "Install complete", + "devel": null, + "disable_crd_hooks": false, + "disable_openapi_validation": false, + "disable_webhooks": false, + "force_update": false, + "id": "external-dns", + "keyring": null, + "lint": false, + "manifest": null, + "max_history": 0, + "metadata": [ + { + "app_version": "0.16.1", + "chart": "external-dns", + "first_deployed": 1773895745, + "last_deployed": 1774070874, + "name": "external-dns", + "namespace": "kube-system", + "notes": "***********************************************************************\n* External DNS *\n***********************************************************************\n Chart version: 1.16.1\n App version: 0.16.1\n Image tag: registry.k8s.io/external-dns/external-dns:v0.16.1\n***********************************************************************\n", + "revision": 2, + "values": "{\"domainFilters\":[\"obs-playground-dev-027423573553.kylhouns.people.aws.dev\"],\"policy\":\"sync\",\"provider\":{\"name\":\"aws\"},\"serviceAccount\":{\"annotations\":{\"eks.amazonaws.com/role-arn\":\"arn:aws:iam::027423573553:role/observability-stack-external-dns\"}},\"txtOwnerId\":\"observability-stack\"}", + "version": "1.16.1" + } + ], + "name": "external-dns", + "namespace": "kube-system", + "pass_credentials": false, + "postrender": [], + "recreate_pods": false, + "render_subchart_notes": true, + "replace": false, + "repository": "https://kubernetes-sigs.github.io/external-dns", + "repository_ca_file": null, + "repository_cert_file": null, + "repository_key_file": null, + "repository_password": null, + "repository_username": null, + "reset_values": false, + "reuse_values": false, + "set": [ + { + "name": "domainFilters[0]", + "type": "", + "value": "obs-playground-dev-027423573553.kylhouns.people.aws.dev" + }, + { + "name": "policy", + "type": "", + "value": "sync" + }, + { + "name": "provider.name", + "type": "", + "value": "aws" + }, + { + "name": "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn", + "type": "", + "value": "arn:aws:iam::027423573553:role/observability-stack-external-dns" + }, + { + "name": "txtOwnerId", + "type": "", + "value": "observability-stack" + } + ], + "set_list": [], + "set_sensitive": [], + "skip_crds": false, + "status": "deployed", + "timeout": 300, + "upgrade_install": null, + "values": null, + "verify": false, + "version": "1.16.1", + "wait": true, + "wait_for_jobs": false + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "module.ebs_csi_irsa.aws_iam_role.this", + "module.eks.aws_cloudwatch_log_group.this", + "module.eks.aws_ec2_tag.cluster_primary_security_group", + "module.eks.aws_eks_access_entry.this", + "module.eks.aws_eks_access_policy_association.this", + "module.eks.aws_eks_addon.before_compute", + "module.eks.aws_eks_addon.this", + "module.eks.aws_eks_cluster.this", + "module.eks.aws_eks_identity_provider_config.this", + "module.eks.aws_iam_openid_connect_provider.oidc_provider", + "module.eks.aws_iam_policy.cluster_encryption", + "module.eks.aws_iam_policy.cni_ipv6_policy", + "module.eks.aws_iam_policy.custom", + "module.eks.aws_iam_role.eks_auto", + "module.eks.aws_iam_role.this", + "module.eks.aws_iam_role_policy_attachment.additional", + "module.eks.aws_iam_role_policy_attachment.cluster_encryption", + "module.eks.aws_iam_role_policy_attachment.custom", + "module.eks.aws_iam_role_policy_attachment.eks_auto", + "module.eks.aws_iam_role_policy_attachment.eks_auto_additional", + "module.eks.aws_iam_role_policy_attachment.this", + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.eks.aws_security_group_rule.cluster", + "module.eks.aws_security_group_rule.node", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_eks_addon_version.this", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_policy_document.cni_ipv6_policy", + "module.eks.data.aws_iam_policy_document.custom", + "module.eks.data.aws_iam_policy_document.node_assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.data.aws_partition.current", + "module.eks.data.tls_certificate.this", + "module.eks.module.eks_managed_node_group.aws_autoscaling_schedule.this", + "module.eks.module.eks_managed_node_group.aws_eks_node_group.this", + "module.eks.module.eks_managed_node_group.aws_iam_role.this", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy.this", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy_attachment.additional", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy_attachment.this", + "module.eks.module.eks_managed_node_group.aws_launch_template.this", + "module.eks.module.eks_managed_node_group.aws_placement_group.this", + "module.eks.module.eks_managed_node_group.data.aws_caller_identity.current", + "module.eks.module.eks_managed_node_group.data.aws_ec2_instance_type.this", + "module.eks.module.eks_managed_node_group.data.aws_ec2_instance_type_offerings.this", + "module.eks.module.eks_managed_node_group.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.eks_managed_node_group.data.aws_iam_policy_document.role", + "module.eks.module.eks_managed_node_group.data.aws_partition.current", + "module.eks.module.eks_managed_node_group.data.aws_ssm_parameter.ami", + "module.eks.module.eks_managed_node_group.data.aws_subnets.placement_group", + "module.eks.module.eks_managed_node_group.module.user_data.data.cloudinit_config.al2023_eks_managed_node_group", + "module.eks.module.eks_managed_node_group.module.user_data.data.cloudinit_config.linux_eks_managed_node_group", + "module.eks.module.eks_managed_node_group.module.user_data.null_resource.validate_cluster_service_cidr", + "module.eks.module.fargate_profile.aws_eks_fargate_profile.this", + "module.eks.module.fargate_profile.aws_iam_role.this", + "module.eks.module.fargate_profile.aws_iam_role_policy.this", + "module.eks.module.fargate_profile.aws_iam_role_policy_attachment.additional", + "module.eks.module.fargate_profile.aws_iam_role_policy_attachment.this", + "module.eks.module.fargate_profile.data.aws_caller_identity.current", + "module.eks.module.fargate_profile.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.fargate_profile.data.aws_iam_policy_document.role", + "module.eks.module.fargate_profile.data.aws_partition.current", + "module.eks.module.fargate_profile.data.aws_region.current", + "module.eks.module.kms.aws_kms_alias.this", + "module.eks.module.kms.aws_kms_external_key.this", + "module.eks.module.kms.aws_kms_grant.this", + "module.eks.module.kms.aws_kms_key.this", + "module.eks.module.kms.aws_kms_replica_external_key.this", + "module.eks.module.kms.aws_kms_replica_key.this", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current", + "module.eks.module.self_managed_node_group.aws_autoscaling_group.this", + "module.eks.module.self_managed_node_group.aws_autoscaling_schedule.this", + "module.eks.module.self_managed_node_group.aws_eks_access_entry.this", + "module.eks.module.self_managed_node_group.aws_iam_instance_profile.this", + "module.eks.module.self_managed_node_group.aws_iam_role.this", + "module.eks.module.self_managed_node_group.aws_iam_role_policy.this", + "module.eks.module.self_managed_node_group.aws_iam_role_policy_attachment.additional", + "module.eks.module.self_managed_node_group.aws_iam_role_policy_attachment.this", + "module.eks.module.self_managed_node_group.aws_launch_template.this", + "module.eks.module.self_managed_node_group.aws_placement_group.this", + "module.eks.module.self_managed_node_group.data.aws_caller_identity.current", + "module.eks.module.self_managed_node_group.data.aws_ec2_instance_type.this", + "module.eks.module.self_managed_node_group.data.aws_ec2_instance_type_offerings.this", + "module.eks.module.self_managed_node_group.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.self_managed_node_group.data.aws_iam_policy_document.role", + "module.eks.module.self_managed_node_group.data.aws_partition.current", + "module.eks.module.self_managed_node_group.data.aws_ssm_parameter.ami", + "module.eks.module.self_managed_node_group.data.aws_subnets.placement_group", + "module.eks.module.self_managed_node_group.module.user_data.data.cloudinit_config.al2023_eks_managed_node_group", + "module.eks.module.self_managed_node_group.module.user_data.data.cloudinit_config.linux_eks_managed_node_group", + "module.eks.module.self_managed_node_group.module.user_data.null_resource.validate_cluster_service_cidr", + "module.eks.time_sleep.this", + "module.external_dns_irsa.aws_iam_role.this", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this" + ] + } + ] + }, + { + "mode": "managed", + "type": "helm_release", + "name": "observability_stack", + "provider": "provider[\"registry.terraform.io/hashicorp/helm\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "atomic": false, + "chart": "./../../charts/observability-stack", + "cleanup_on_fail": true, + "create_namespace": false, + "dependency_update": false, + "description": null, + "devel": null, + "disable_crd_hooks": false, + "disable_openapi_validation": false, + "disable_webhooks": false, + "force_update": false, + "id": "obs-stack", + "keyring": null, + "lint": false, + "manifest": null, + "max_history": 0, + "metadata": [ + { + "app_version": "3.6.0", + "chart": "observability-stack", + "first_deployed": 1773869439, + "last_deployed": 1774071552, + "name": "obs-stack", + "namespace": "observability-stack", + "notes": "🚀 Observability Stack deployed!\n\nAccess OpenSearch Dashboards:\n\n kubectl port-forward -n observability-stack svc/obs-stack-opensearch-dashboards 5601:5601\n\n Then open: http://localhost:5601\n Username: admin\n Password: My_password_123!@#\n\nSend telemetry:\n\n kubectl port-forward -n observability-stack svc/obs-stack-opentelemetry-collector 4317:4317 4318:4318\n\n OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317\n", + "revision": 59, + "values": "{}", + "version": "0.1.0" + } + ], + "name": "obs-stack", + "namespace": "observability-stack", + "pass_credentials": false, + "postrender": [], + "recreate_pods": false, + "render_subchart_notes": true, + "replace": false, + "repository": null, + "repository_ca_file": null, + "repository_cert_file": null, + "repository_key_file": null, + "repository_password": null, + "repository_username": null, + "reset_values": false, + "reuse_values": false, + "set": [ + { + "name": "opensearch-dashboards.ingress.annotations.alb\\.ingress\\.kubernetes\\.io/certificate-arn", + "type": "", + "value": "arn:aws:acm:us-west-2:027423573553:certificate/f792348a-14b1-474b-9e46-98e1a1091485" + }, + { + "name": "opensearch-dashboards.ingress.annotations.alb\\.ingress\\.kubernetes\\.io/listen-ports", + "type": "", + "value": "[{\"HTTPS\":443}]" + }, + { + "name": "opensearch-dashboards.ingress.annotations.alb\\.ingress\\.kubernetes\\.io/ssl-redirect", + "type": "string", + "value": "443" + }, + { + "name": "opensearch-dashboards.ingress.annotations.external-dns\\.alpha\\.kubernetes\\.io/hostname", + "type": "", + "value": "obs-playground-dev-027423573553.kylhouns.people.aws.dev" + }, + { + "name": "opensearch-dashboards.ingress.hosts[0].host", + "type": "", + "value": "obs-playground-dev-027423573553.kylhouns.people.aws.dev" + } + ], + "set_list": [], + "set_sensitive": [], + "skip_crds": false, + "status": "deployed", + "timeout": 900, + "upgrade_install": null, + "values": [ + "opensearch:\n persistence:\n storageClass: gp2\n\nopensearch-dashboards:\n ingress:\n enabled: true\n ingressClassName: alb\n annotations:\n alb.ingress.kubernetes.io/scheme: internet-facing\n alb.ingress.kubernetes.io/target-type: ip\n alb.ingress.kubernetes.io/listen-ports: '[{\"HTTP\":80}]'\n hosts:\n - paths:\n - path: /\n backend:\n servicePort: 5601\n\ngateway:\n enabled: false\n\nexamples:\n enabled: false\n" + ], + "verify": false, + "version": "0.1.0", + "wait": true, + "wait_for_jobs": true + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "aws_acm_certificate.dashboards", + "aws_wafv2_web_acl.rate_limit", + "helm_release.aws_lb_controller", + "kubernetes_namespace.observability", + "module.eks.aws_eks_access_entry.this", + "module.eks.aws_eks_access_policy_association.this", + "module.eks.aws_eks_cluster.this", + "module.eks.data.aws_iam_session_context.current", + "module.eks.data.aws_partition.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "kubernetes_namespace", + "name": "observability", + "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "observability-stack", + "metadata": [ + { + "annotations": {}, + "generate_name": "", + "generation": 0, + "labels": {}, + "name": "observability-stack", + "resource_version": "3170", + "uid": "56cc0765-b461-4537-9cfc-921846cfdc24" + } + ], + "timeouts": null, + "wait_for_default_service_account": false + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjozMDAwMDAwMDAwMDB9fQ==", + "dependencies": [ + "data.aws_availability_zones.available", + "module.ebs_csi_irsa.aws_iam_role.this", + "module.ebs_csi_irsa.data.aws_caller_identity.current", + "module.ebs_csi_irsa.data.aws_iam_policy_document.this", + "module.ebs_csi_irsa.data.aws_partition.current", + "module.eks.aws_cloudwatch_log_group.this", + "module.eks.aws_ec2_tag.cluster_primary_security_group", + "module.eks.aws_eks_access_entry.this", + "module.eks.aws_eks_access_policy_association.this", + "module.eks.aws_eks_addon.before_compute", + "module.eks.aws_eks_addon.this", + "module.eks.aws_eks_cluster.this", + "module.eks.aws_eks_identity_provider_config.this", + "module.eks.aws_iam_openid_connect_provider.oidc_provider", + "module.eks.aws_iam_policy.cluster_encryption", + "module.eks.aws_iam_policy.cni_ipv6_policy", + "module.eks.aws_iam_policy.custom", + "module.eks.aws_iam_role.eks_auto", + "module.eks.aws_iam_role.this", + "module.eks.aws_iam_role_policy_attachment.additional", + "module.eks.aws_iam_role_policy_attachment.cluster_encryption", + "module.eks.aws_iam_role_policy_attachment.custom", + "module.eks.aws_iam_role_policy_attachment.eks_auto", + "module.eks.aws_iam_role_policy_attachment.eks_auto_additional", + "module.eks.aws_iam_role_policy_attachment.this", + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.eks.aws_security_group_rule.cluster", + "module.eks.aws_security_group_rule.node", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_eks_addon_version.this", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_policy_document.cni_ipv6_policy", + "module.eks.data.aws_iam_policy_document.custom", + "module.eks.data.aws_iam_policy_document.node_assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.data.aws_partition.current", + "module.eks.data.tls_certificate.this", + "module.eks.module.eks_managed_node_group.aws_autoscaling_schedule.this", + "module.eks.module.eks_managed_node_group.aws_eks_node_group.this", + "module.eks.module.eks_managed_node_group.aws_iam_role.this", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy.this", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy_attachment.additional", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy_attachment.this", + "module.eks.module.eks_managed_node_group.aws_launch_template.this", + "module.eks.module.eks_managed_node_group.aws_placement_group.this", + "module.eks.module.eks_managed_node_group.data.aws_caller_identity.current", + "module.eks.module.eks_managed_node_group.data.aws_ec2_instance_type.this", + "module.eks.module.eks_managed_node_group.data.aws_ec2_instance_type_offerings.this", + "module.eks.module.eks_managed_node_group.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.eks_managed_node_group.data.aws_iam_policy_document.role", + "module.eks.module.eks_managed_node_group.data.aws_partition.current", + "module.eks.module.eks_managed_node_group.data.aws_ssm_parameter.ami", + "module.eks.module.eks_managed_node_group.data.aws_subnets.placement_group", + "module.eks.module.eks_managed_node_group.module.user_data.data.cloudinit_config.al2023_eks_managed_node_group", + "module.eks.module.eks_managed_node_group.module.user_data.data.cloudinit_config.linux_eks_managed_node_group", + "module.eks.module.eks_managed_node_group.module.user_data.null_resource.validate_cluster_service_cidr", + "module.eks.module.fargate_profile.aws_eks_fargate_profile.this", + "module.eks.module.fargate_profile.aws_iam_role.this", + "module.eks.module.fargate_profile.aws_iam_role_policy.this", + "module.eks.module.fargate_profile.aws_iam_role_policy_attachment.additional", + "module.eks.module.fargate_profile.aws_iam_role_policy_attachment.this", + "module.eks.module.fargate_profile.data.aws_caller_identity.current", + "module.eks.module.fargate_profile.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.fargate_profile.data.aws_iam_policy_document.role", + "module.eks.module.fargate_profile.data.aws_partition.current", + "module.eks.module.fargate_profile.data.aws_region.current", + "module.eks.module.kms.aws_kms_alias.this", + "module.eks.module.kms.aws_kms_external_key.this", + "module.eks.module.kms.aws_kms_grant.this", + "module.eks.module.kms.aws_kms_key.this", + "module.eks.module.kms.aws_kms_replica_external_key.this", + "module.eks.module.kms.aws_kms_replica_key.this", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current", + "module.eks.module.self_managed_node_group.aws_autoscaling_group.this", + "module.eks.module.self_managed_node_group.aws_autoscaling_schedule.this", + "module.eks.module.self_managed_node_group.aws_eks_access_entry.this", + "module.eks.module.self_managed_node_group.aws_iam_instance_profile.this", + "module.eks.module.self_managed_node_group.aws_iam_role.this", + "module.eks.module.self_managed_node_group.aws_iam_role_policy.this", + "module.eks.module.self_managed_node_group.aws_iam_role_policy_attachment.additional", + "module.eks.module.self_managed_node_group.aws_iam_role_policy_attachment.this", + "module.eks.module.self_managed_node_group.aws_launch_template.this", + "module.eks.module.self_managed_node_group.aws_placement_group.this", + "module.eks.module.self_managed_node_group.data.aws_caller_identity.current", + "module.eks.module.self_managed_node_group.data.aws_ec2_instance_type.this", + "module.eks.module.self_managed_node_group.data.aws_ec2_instance_type_offerings.this", + "module.eks.module.self_managed_node_group.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.self_managed_node_group.data.aws_iam_policy_document.role", + "module.eks.module.self_managed_node_group.data.aws_partition.current", + "module.eks.module.self_managed_node_group.data.aws_ssm_parameter.ami", + "module.eks.module.self_managed_node_group.data.aws_subnets.placement_group", + "module.eks.module.self_managed_node_group.module.user_data.data.cloudinit_config.al2023_eks_managed_node_group", + "module.eks.module.self_managed_node_group.module.user_data.data.cloudinit_config.linux_eks_managed_node_group", + "module.eks.module.self_managed_node_group.module.user_data.null_resource.validate_cluster_service_cidr", + "module.eks.time_sleep.this", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "data", + "type": "aws_caller_identity", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "account_id": "027423573553", + "arn": "arn:aws:sts::027423573553:assumed-role/Admin/kylhouns-Isengard", + "id": "027423573553", + "user_id": "AROAQMYUSQYYXMMZMGMPD:kylhouns-Isengard" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "amazon_managed_service_prometheus", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "appmesh_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "appmesh_envoy_proxy", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "aws_gateway_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cert_manager", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cluster_autoscaler", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "ebs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "id": "4189668531", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVolumesModifications\",\n \"ec2:DescribeVolumes\",\n \"ec2:DescribeTags\",\n \"ec2:DescribeSnapshots\",\n \"ec2:DescribeInstances\",\n \"ec2:DescribeAvailabilityZones\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:ModifyVolume\",\n \"ec2:CreateSnapshot\"\n ],\n \"Resource\": \"arn:aws:ec2:*:*:volume/*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DetachVolume\",\n \"ec2:AttachVolume\"\n ],\n \"Resource\": [\n \"arn:aws:ec2:*:*:volume/*\",\n \"arn:aws:ec2:*:*:instance/*\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:EnableFastSnapshotRestores\",\n \"ec2:CreateVolume\"\n ],\n \"Resource\": \"arn:aws:ec2:*:*:snapshot/*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateTags\",\n \"Resource\": [\n \"arn:aws:ec2:*:*:volume/*\",\n \"arn:aws:ec2:*:*:snapshot/*\"\n ],\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:CreateAction\": [\n \"CreateVolume\",\n \"CreateSnapshot\"\n ]\n }\n }\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:DeleteTags\",\n \"Resource\": [\n \"arn:aws:ec2:*:*:volume/*\",\n \"arn:aws:ec2:*:*:snapshot/*\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateVolume\",\n \"Resource\": \"arn:aws:ec2:*:*:volume/*\",\n \"Condition\": {\n \"StringLike\": {\n \"aws:RequestTag/ebs.csi.aws.com/cluster\": \"true\"\n }\n }\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateVolume\",\n \"Resource\": \"arn:aws:ec2:*:*:volume/*\",\n \"Condition\": {\n \"StringLike\": {\n \"aws:RequestTag/CSIVolumeName\": \"*\"\n }\n }\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:DeleteVolume\",\n \"Resource\": \"arn:aws:ec2:*:*:volume/*\",\n \"Condition\": {\n \"StringLike\": {\n \"aws:ResourceTag/ebs.csi.aws.com/cluster\": \"true\"\n }\n }\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:DeleteVolume\",\n \"Resource\": \"arn:aws:ec2:*:*:volume/*\",\n \"Condition\": {\n \"StringLike\": {\n \"aws:ResourceTag/CSIVolumeName\": \"*\"\n }\n }\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:DeleteVolume\",\n \"Resource\": \"arn:aws:ec2:*:*:volume/*\",\n \"Condition\": {\n \"StringLike\": {\n \"ec2:ResourceTag/kubernetes.io/created-for/pvc/name\": \"*\"\n }\n }\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateSnapshot\",\n \"Resource\": \"arn:aws:ec2:*:*:snapshot/*\",\n \"Condition\": {\n \"StringLike\": {\n \"aws:RequestTag/CSIVolumeSnapshotName\": \"*\"\n }\n }\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateSnapshot\",\n \"Resource\": \"arn:aws:ec2:*:*:snapshot/*\",\n \"Condition\": {\n \"StringLike\": {\n \"aws:RequestTag/ebs.csi.aws.com/cluster\": \"true\"\n }\n }\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:DeleteSnapshot\",\n \"Resource\": \"arn:aws:ec2:*:*:snapshot/*\",\n \"Condition\": {\n \"StringLike\": {\n \"aws:ResourceTag/CSIVolumeSnapshotName\": \"*\"\n }\n }\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:DeleteSnapshot\",\n \"Resource\": \"arn:aws:ec2:*:*:snapshot/*\",\n \"Condition\": {\n \"StringLike\": {\n \"aws:ResourceTag/ebs.csi.aws.com/cluster\": \"true\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVolumesModifications\",\"ec2:DescribeVolumes\",\"ec2:DescribeTags\",\"ec2:DescribeSnapshots\",\"ec2:DescribeInstances\",\"ec2:DescribeAvailabilityZones\"],\"Resource\":\"*\"},{\"Effect\":\"Allow\",\"Action\":[\"ec2:ModifyVolume\",\"ec2:CreateSnapshot\"],\"Resource\":\"arn:aws:ec2:*:*:volume/*\"},{\"Effect\":\"Allow\",\"Action\":[\"ec2:DetachVolume\",\"ec2:AttachVolume\"],\"Resource\":[\"arn:aws:ec2:*:*:volume/*\",\"arn:aws:ec2:*:*:instance/*\"]},{\"Effect\":\"Allow\",\"Action\":[\"ec2:EnableFastSnapshotRestores\",\"ec2:CreateVolume\"],\"Resource\":\"arn:aws:ec2:*:*:snapshot/*\"},{\"Effect\":\"Allow\",\"Action\":\"ec2:CreateTags\",\"Resource\":[\"arn:aws:ec2:*:*:volume/*\",\"arn:aws:ec2:*:*:snapshot/*\"],\"Condition\":{\"StringEquals\":{\"ec2:CreateAction\":[\"CreateVolume\",\"CreateSnapshot\"]}}},{\"Effect\":\"Allow\",\"Action\":\"ec2:DeleteTags\",\"Resource\":[\"arn:aws:ec2:*:*:volume/*\",\"arn:aws:ec2:*:*:snapshot/*\"]},{\"Effect\":\"Allow\",\"Action\":\"ec2:CreateVolume\",\"Resource\":\"arn:aws:ec2:*:*:volume/*\",\"Condition\":{\"StringLike\":{\"aws:RequestTag/ebs.csi.aws.com/cluster\":\"true\"}}},{\"Effect\":\"Allow\",\"Action\":\"ec2:CreateVolume\",\"Resource\":\"arn:aws:ec2:*:*:volume/*\",\"Condition\":{\"StringLike\":{\"aws:RequestTag/CSIVolumeName\":\"*\"}}},{\"Effect\":\"Allow\",\"Action\":\"ec2:DeleteVolume\",\"Resource\":\"arn:aws:ec2:*:*:volume/*\",\"Condition\":{\"StringLike\":{\"aws:ResourceTag/ebs.csi.aws.com/cluster\":\"true\"}}},{\"Effect\":\"Allow\",\"Action\":\"ec2:DeleteVolume\",\"Resource\":\"arn:aws:ec2:*:*:volume/*\",\"Condition\":{\"StringLike\":{\"aws:ResourceTag/CSIVolumeName\":\"*\"}}},{\"Effect\":\"Allow\",\"Action\":\"ec2:DeleteVolume\",\"Resource\":\"arn:aws:ec2:*:*:volume/*\",\"Condition\":{\"StringLike\":{\"ec2:ResourceTag/kubernetes.io/created-for/pvc/name\":\"*\"}}},{\"Effect\":\"Allow\",\"Action\":\"ec2:CreateSnapshot\",\"Resource\":\"arn:aws:ec2:*:*:snapshot/*\",\"Condition\":{\"StringLike\":{\"aws:RequestTag/CSIVolumeSnapshotName\":\"*\"}}},{\"Effect\":\"Allow\",\"Action\":\"ec2:CreateSnapshot\",\"Resource\":\"arn:aws:ec2:*:*:snapshot/*\",\"Condition\":{\"StringLike\":{\"aws:RequestTag/ebs.csi.aws.com/cluster\":\"true\"}}},{\"Effect\":\"Allow\",\"Action\":\"ec2:DeleteSnapshot\",\"Resource\":\"arn:aws:ec2:*:*:snapshot/*\",\"Condition\":{\"StringLike\":{\"aws:ResourceTag/CSIVolumeSnapshotName\":\"*\"}}},{\"Effect\":\"Allow\",\"Action\":\"ec2:DeleteSnapshot\",\"Resource\":\"arn:aws:ec2:*:*:snapshot/*\",\"Condition\":{\"StringLike\":{\"aws:ResourceTag/ebs.csi.aws.com/cluster\":\"true\"}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "ec2:DescribeAvailabilityZones", + "ec2:DescribeInstances", + "ec2:DescribeSnapshots", + "ec2:DescribeTags", + "ec2:DescribeVolumes", + "ec2:DescribeVolumesModifications" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "" + }, + { + "actions": [ + "ec2:CreateSnapshot", + "ec2:ModifyVolume" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:*:*:volume/*" + ], + "sid": "" + }, + { + "actions": [ + "ec2:AttachVolume", + "ec2:DetachVolume" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:*:*:instance/*", + "arn:aws:ec2:*:*:volume/*" + ], + "sid": "" + }, + { + "actions": [ + "ec2:CreateVolume", + "ec2:EnableFastSnapshotRestores" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:*:*:snapshot/*" + ], + "sid": "" + }, + { + "actions": [ + "ec2:CreateTags" + ], + "condition": [ + { + "test": "StringEquals", + "values": [ + "CreateVolume", + "CreateSnapshot" + ], + "variable": "ec2:CreateAction" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:*:*:snapshot/*", + "arn:aws:ec2:*:*:volume/*" + ], + "sid": "" + }, + { + "actions": [ + "ec2:DeleteTags" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:*:*:snapshot/*", + "arn:aws:ec2:*:*:volume/*" + ], + "sid": "" + }, + { + "actions": [ + "ec2:CreateVolume" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "true" + ], + "variable": "aws:RequestTag/ebs.csi.aws.com/cluster" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:*:*:volume/*" + ], + "sid": "" + }, + { + "actions": [ + "ec2:CreateVolume" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "*" + ], + "variable": "aws:RequestTag/CSIVolumeName" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:*:*:volume/*" + ], + "sid": "" + }, + { + "actions": [ + "ec2:DeleteVolume" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "true" + ], + "variable": "aws:ResourceTag/ebs.csi.aws.com/cluster" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:*:*:volume/*" + ], + "sid": "" + }, + { + "actions": [ + "ec2:DeleteVolume" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "*" + ], + "variable": "aws:ResourceTag/CSIVolumeName" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:*:*:volume/*" + ], + "sid": "" + }, + { + "actions": [ + "ec2:DeleteVolume" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "*" + ], + "variable": "ec2:ResourceTag/kubernetes.io/created-for/pvc/name" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:*:*:volume/*" + ], + "sid": "" + }, + { + "actions": [ + "ec2:CreateSnapshot" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "*" + ], + "variable": "aws:RequestTag/CSIVolumeSnapshotName" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:*:*:snapshot/*" + ], + "sid": "" + }, + { + "actions": [ + "ec2:CreateSnapshot" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "true" + ], + "variable": "aws:RequestTag/ebs.csi.aws.com/cluster" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:*:*:snapshot/*" + ], + "sid": "" + }, + { + "actions": [ + "ec2:DeleteSnapshot" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "*" + ], + "variable": "aws:ResourceTag/CSIVolumeSnapshotName" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:*:*:snapshot/*" + ], + "sid": "" + }, + { + "actions": [ + "ec2:DeleteSnapshot" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "true" + ], + "variable": "aws:ResourceTag/ebs.csi.aws.com/cluster" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:*:*:snapshot/*" + ], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "efs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "external_dns", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "external_secrets", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "fsx_lustre_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "fsx_openzfs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "karpenter_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "load_balancer_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "load_balancer_controller_targetgroup_only", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "mountpoint_s3_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "node_termination_handler", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "id": "214328930", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRoleWithWebIdentity\",\n \"Principal\": {\n \"Federated\": \"arn:aws:iam::027423573553:oidc-provider/oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D\"\n },\n \"Condition\": {\n \"StringEquals\": {\n \"oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D:aud\": \"sts.amazonaws.com\",\n \"oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D:sub\": \"system:serviceaccount:kube-system:ebs-csi-controller-sa\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRoleWithWebIdentity\",\"Principal\":{\"Federated\":\"arn:aws:iam::027423573553:oidc-provider/oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D\"},\"Condition\":{\"StringEquals\":{\"oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D:aud\":\"sts.amazonaws.com\",\"oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D:sub\":\"system:serviceaccount:kube-system:ebs-csi-controller-sa\"}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRoleWithWebIdentity" + ], + "condition": [ + { + "test": "StringEquals", + "values": [ + "sts.amazonaws.com" + ], + "variable": "oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D:aud" + }, + { + "test": "StringEquals", + "values": [ + "system:serviceaccount:kube-system:ebs-csi-controller-sa" + ], + "variable": "oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D:sub" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "arn:aws:iam::027423573553:oidc-provider/oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D" + ], + "type": "Federated" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "velero", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "vpc_cni", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "data", + "type": "aws_partition", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "dns_suffix": "amazonaws.com", + "id": "aws", + "partition": "aws", + "reverse_dns_prefix": "com.amazonaws" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "data", + "type": "aws_region", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "description": "US West (Oregon)", + "endpoint": "ec2.us-west-2.amazonaws.com", + "id": "us-west-2", + "name": "us-west-2" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "amazon_managed_service_prometheus", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "appmesh_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "appmesh_envoy_proxy", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "aws_gateway_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "cert_manager", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "cluster_autoscaler", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "ebs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::027423573553:policy/AmazonEKS_EBS_CSI_Policy-20260318201827555200000005", + "attachment_count": 1, + "description": "Provides permissions to manage EBS volumes via the container storage interface driver", + "id": "arn:aws:iam::027423573553:policy/AmazonEKS_EBS_CSI_Policy-20260318201827555200000005", + "name": "AmazonEKS_EBS_CSI_Policy-20260318201827555200000005", + "name_prefix": "AmazonEKS_EBS_CSI_Policy-", + "path": "/", + "policy": "{\"Statement\":[{\"Action\":[\"ec2:DescribeVolumesModifications\",\"ec2:DescribeVolumes\",\"ec2:DescribeTags\",\"ec2:DescribeSnapshots\",\"ec2:DescribeInstances\",\"ec2:DescribeAvailabilityZones\"],\"Effect\":\"Allow\",\"Resource\":\"*\"},{\"Action\":[\"ec2:ModifyVolume\",\"ec2:CreateSnapshot\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:*:*:volume/*\"},{\"Action\":[\"ec2:DetachVolume\",\"ec2:AttachVolume\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:ec2:*:*:volume/*\",\"arn:aws:ec2:*:*:instance/*\"]},{\"Action\":[\"ec2:EnableFastSnapshotRestores\",\"ec2:CreateVolume\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:*:*:snapshot/*\"},{\"Action\":\"ec2:CreateTags\",\"Condition\":{\"StringEquals\":{\"ec2:CreateAction\":[\"CreateVolume\",\"CreateSnapshot\"]}},\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:ec2:*:*:volume/*\",\"arn:aws:ec2:*:*:snapshot/*\"]},{\"Action\":\"ec2:DeleteTags\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:ec2:*:*:volume/*\",\"arn:aws:ec2:*:*:snapshot/*\"]},{\"Action\":\"ec2:CreateVolume\",\"Condition\":{\"StringLike\":{\"aws:RequestTag/ebs.csi.aws.com/cluster\":\"true\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:*:*:volume/*\"},{\"Action\":\"ec2:CreateVolume\",\"Condition\":{\"StringLike\":{\"aws:RequestTag/CSIVolumeName\":\"*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:*:*:volume/*\"},{\"Action\":\"ec2:DeleteVolume\",\"Condition\":{\"StringLike\":{\"aws:ResourceTag/ebs.csi.aws.com/cluster\":\"true\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:*:*:volume/*\"},{\"Action\":\"ec2:DeleteVolume\",\"Condition\":{\"StringLike\":{\"aws:ResourceTag/CSIVolumeName\":\"*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:*:*:volume/*\"},{\"Action\":\"ec2:DeleteVolume\",\"Condition\":{\"StringLike\":{\"ec2:ResourceTag/kubernetes.io/created-for/pvc/name\":\"*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:*:*:volume/*\"},{\"Action\":\"ec2:CreateSnapshot\",\"Condition\":{\"StringLike\":{\"aws:RequestTag/CSIVolumeSnapshotName\":\"*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:*:*:snapshot/*\"},{\"Action\":\"ec2:CreateSnapshot\",\"Condition\":{\"StringLike\":{\"aws:RequestTag/ebs.csi.aws.com/cluster\":\"true\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:*:*:snapshot/*\"},{\"Action\":\"ec2:DeleteSnapshot\",\"Condition\":{\"StringLike\":{\"aws:ResourceTag/CSIVolumeSnapshotName\":\"*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:*:*:snapshot/*\"},{\"Action\":\"ec2:DeleteSnapshot\",\"Condition\":{\"StringLike\":{\"aws:ResourceTag/ebs.csi.aws.com/cluster\":\"true\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:*:*:snapshot/*\"}],\"Version\":\"2012-10-17\"}", + "policy_id": "ANPAQMYUSQYYQZREZBTEK", + "tags": {}, + "tags_all": { + "ManagedBy": "terraform", + "Project": "observability-stack" + } + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.ebs_csi_irsa.data.aws_iam_policy_document.ebs_csi", + "module.ebs_csi_irsa.data.aws_partition.current" + ] + } + ] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "efs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "external_dns", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "external_secrets", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "fsx_lustre_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "fsx_openzfs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "karpenter_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "load_balancer_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "load_balancer_controller_targetgroup_only", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "mountpoint_s3_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "node_termination_handler", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "velero", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "vpc_cni", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_role", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::027423573553:role/observability-stack-ebs-csi", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRoleWithWebIdentity\",\"Condition\":{\"StringEquals\":{\"oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D:aud\":\"sts.amazonaws.com\",\"oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D:sub\":\"system:serviceaccount:kube-system:ebs-csi-controller-sa\"}},\"Effect\":\"Allow\",\"Principal\":{\"Federated\":\"arn:aws:iam::027423573553:oidc-provider/oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-03-18T20:28:14Z", + "description": "", + "force_detach_policies": true, + "id": "observability-stack-ebs-csi", + "inline_policy": [], + "managed_policy_arns": [ + "arn:aws:iam::027423573553:policy/AmazonEKS_EBS_CSI_Policy-20260318201827555200000005" + ], + "max_session_duration": 3600, + "name": "observability-stack-ebs-csi", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": {}, + "tags_all": { + "ManagedBy": "terraform", + "Project": "observability-stack" + }, + "unique_id": "AROAQMYUSQYY7QHA4MYZO" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_availability_zones.available", + "module.ebs_csi_irsa.data.aws_caller_identity.current", + "module.ebs_csi_irsa.data.aws_iam_policy_document.this", + "module.ebs_csi_irsa.data.aws_partition.current", + "module.eks.aws_cloudwatch_log_group.this", + "module.eks.aws_eks_cluster.this", + "module.eks.aws_iam_openid_connect_provider.oidc_provider", + "module.eks.aws_iam_policy.cni_ipv6_policy", + "module.eks.aws_iam_role.eks_auto", + "module.eks.aws_iam_role.this", + "module.eks.aws_iam_role_policy_attachment.this", + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.eks.aws_security_group_rule.cluster", + "module.eks.aws_security_group_rule.node", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_policy_document.cni_ipv6_policy", + "module.eks.data.aws_iam_policy_document.node_assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.data.aws_partition.current", + "module.eks.data.tls_certificate.this", + "module.eks.module.kms.aws_kms_external_key.this", + "module.eks.module.kms.aws_kms_key.this", + "module.eks.module.kms.aws_kms_replica_external_key.this", + "module.eks.module.kms.aws_kms_replica_key.this", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "amazon_cloudwatch_observability", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "amazon_managed_service_prometheus", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "appmesh_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "appmesh_envoy_proxy", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "aws_gateway_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "cert_manager", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "cluster_autoscaler", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "ebs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "id": "observability-stack-ebs-csi-20260318202815382700000015", + "policy_arn": "arn:aws:iam::027423573553:policy/AmazonEKS_EBS_CSI_Policy-20260318201827555200000005", + "role": "observability-stack-ebs-csi" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_availability_zones.available", + "module.ebs_csi_irsa.aws_iam_policy.ebs_csi", + "module.ebs_csi_irsa.aws_iam_role.this", + "module.ebs_csi_irsa.data.aws_caller_identity.current", + "module.ebs_csi_irsa.data.aws_iam_policy_document.ebs_csi", + "module.ebs_csi_irsa.data.aws_iam_policy_document.this", + "module.ebs_csi_irsa.data.aws_partition.current", + "module.eks.aws_cloudwatch_log_group.this", + "module.eks.aws_eks_cluster.this", + "module.eks.aws_iam_openid_connect_provider.oidc_provider", + "module.eks.aws_iam_policy.cni_ipv6_policy", + "module.eks.aws_iam_role.eks_auto", + "module.eks.aws_iam_role.this", + "module.eks.aws_iam_role_policy_attachment.this", + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.eks.aws_security_group_rule.cluster", + "module.eks.aws_security_group_rule.node", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_policy_document.cni_ipv6_policy", + "module.eks.data.aws_iam_policy_document.node_assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.data.aws_partition.current", + "module.eks.data.tls_certificate.this", + "module.eks.module.kms.aws_kms_external_key.this", + "module.eks.module.kms.aws_kms_key.this", + "module.eks.module.kms.aws_kms_replica_external_key.this", + "module.eks.module.kms.aws_kms_replica_key.this", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "efs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "external_dns", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "external_secrets", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "fsx_lustre_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "fsx_openzfs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "karpenter_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "load_balancer_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "load_balancer_controller_targetgroup_only", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "mountpoint_s3_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "node_termination_handler", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "velero", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.ebs_csi_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "vpc_cni", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.eks", + "mode": "data", + "type": "aws_caller_identity", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "account_id": "027423573553", + "arn": "arn:aws:sts::027423573553:assumed-role/Admin/kylhouns-Isengard", + "id": "027423573553", + "user_id": "AROAQMYUSQYYXMMZMGMPD:kylhouns-Isengard" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.eks", + "mode": "data", + "type": "aws_eks_addon_version", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": "aws-ebs-csi-driver", + "schema_version": 0, + "attributes": { + "addon_name": "aws-ebs-csi-driver", + "id": "aws-ebs-csi-driver", + "kubernetes_version": "1.32", + "most_recent": false, + "version": "v1.56.0-eksbuild.1" + }, + "sensitive_attributes": [] + }, + { + "index_key": "coredns", + "schema_version": 0, + "attributes": { + "addon_name": "coredns", + "id": "coredns", + "kubernetes_version": "1.32", + "most_recent": false, + "version": "v1.11.4-eksbuild.2" + }, + "sensitive_attributes": [] + }, + { + "index_key": "eks-pod-identity-agent", + "schema_version": 0, + "attributes": { + "addon_name": "eks-pod-identity-agent", + "id": "eks-pod-identity-agent", + "kubernetes_version": "1.32", + "most_recent": false, + "version": "v1.3.10-eksbuild.2" + }, + "sensitive_attributes": [] + }, + { + "index_key": "kube-proxy", + "schema_version": 0, + "attributes": { + "addon_name": "kube-proxy", + "id": "kube-proxy", + "kubernetes_version": "1.32", + "most_recent": false, + "version": "v1.32.6-eksbuild.12" + }, + "sensitive_attributes": [] + }, + { + "index_key": "vpc-cni", + "schema_version": 0, + "attributes": { + "addon_name": "vpc-cni", + "id": "vpc-cni", + "kubernetes_version": "1.32", + "most_recent": false, + "version": "v1.20.4-eksbuild.2" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.eks", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "assume_role_policy", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "id": "2830595799", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"EKSClusterAssumeRole\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sts:TagSession\",\n \"sts:AssumeRole\"\n ],\n \"Principal\": {\n \"Service\": \"eks.amazonaws.com\"\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"EKSClusterAssumeRole\",\"Effect\":\"Allow\",\"Action\":[\"sts:TagSession\",\"sts:AssumeRole\"],\"Principal\":{\"Service\":\"eks.amazonaws.com\"}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole", + "sts:TagSession" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "eks.amazonaws.com" + ], + "type": "Service" + } + ], + "resources": [], + "sid": "EKSClusterAssumeRole" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.eks", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cni_ipv6_policy", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.eks", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "custom", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "id": "513122117", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"Compute\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:RunInstances\",\n \"ec2:CreateLaunchTemplate\",\n \"ec2:CreateFleet\"\n ],\n \"Resource\": \"*\",\n \"Condition\": {\n \"StringEquals\": {\n \"aws:RequestTag/eks:eks-cluster-name\": \"${aws:PrincipalTag/eks:eks-cluster-name}\"\n },\n \"StringLike\": {\n \"aws:RequestTag/eks:kubernetes-node-class-name\": \"*\",\n \"aws:RequestTag/eks:kubernetes-node-pool-name\": \"*\"\n }\n }\n },\n {\n \"Sid\": \"Storage\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:CreateVolume\",\n \"ec2:CreateSnapshot\"\n ],\n \"Resource\": [\n \"arn:aws:ec2:*:*:volume/*\",\n \"arn:aws:ec2:*:*:snapshot/*\"\n ],\n \"Condition\": {\n \"StringEquals\": {\n \"aws:RequestTag/eks:eks-cluster-name\": \"${aws:PrincipalTag/eks:eks-cluster-name}\"\n }\n }\n },\n {\n \"Sid\": \"Networking\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterface\",\n \"Resource\": \"*\",\n \"Condition\": {\n \"StringEquals\": {\n \"aws:RequestTag/eks:eks-cluster-name\": \"${aws:PrincipalTag/eks:eks-cluster-name}\",\n \"aws:RequestTag/eks:kubernetes-cni-node-name\": \"*\"\n }\n }\n },\n {\n \"Sid\": \"LoadBalancer\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"elasticloadbalancing:CreateTargetGroup\",\n \"elasticloadbalancing:CreateRule\",\n \"elasticloadbalancing:CreateLoadBalancer\",\n \"elasticloadbalancing:CreateListener\",\n \"ec2:CreateSecurityGroup\"\n ],\n \"Resource\": \"*\",\n \"Condition\": {\n \"StringEquals\": {\n \"aws:RequestTag/eks:eks-cluster-name\": \"${aws:PrincipalTag/eks:eks-cluster-name}\"\n }\n }\n },\n {\n \"Sid\": \"ShieldProtection\",\n \"Effect\": \"Allow\",\n \"Action\": \"shield:CreateProtection\",\n \"Resource\": \"*\",\n \"Condition\": {\n \"StringEquals\": {\n \"aws:RequestTag/eks:eks-cluster-name\": \"${aws:PrincipalTag/eks:eks-cluster-name}\"\n }\n }\n },\n {\n \"Sid\": \"ShieldTagResource\",\n \"Effect\": \"Allow\",\n \"Action\": \"shield:TagResource\",\n \"Resource\": \"arn:aws:shield::*:protection/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"aws:RequestTag/eks:eks-cluster-name\": \"${aws:PrincipalTag/eks:eks-cluster-name}\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"Compute\",\"Effect\":\"Allow\",\"Action\":[\"ec2:RunInstances\",\"ec2:CreateLaunchTemplate\",\"ec2:CreateFleet\"],\"Resource\":\"*\",\"Condition\":{\"StringEquals\":{\"aws:RequestTag/eks:eks-cluster-name\":\"${aws:PrincipalTag/eks:eks-cluster-name}\"},\"StringLike\":{\"aws:RequestTag/eks:kubernetes-node-class-name\":\"*\",\"aws:RequestTag/eks:kubernetes-node-pool-name\":\"*\"}}},{\"Sid\":\"Storage\",\"Effect\":\"Allow\",\"Action\":[\"ec2:CreateVolume\",\"ec2:CreateSnapshot\"],\"Resource\":[\"arn:aws:ec2:*:*:volume/*\",\"arn:aws:ec2:*:*:snapshot/*\"],\"Condition\":{\"StringEquals\":{\"aws:RequestTag/eks:eks-cluster-name\":\"${aws:PrincipalTag/eks:eks-cluster-name}\"}}},{\"Sid\":\"Networking\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterface\",\"Resource\":\"*\",\"Condition\":{\"StringEquals\":{\"aws:RequestTag/eks:eks-cluster-name\":\"${aws:PrincipalTag/eks:eks-cluster-name}\",\"aws:RequestTag/eks:kubernetes-cni-node-name\":\"*\"}}},{\"Sid\":\"LoadBalancer\",\"Effect\":\"Allow\",\"Action\":[\"elasticloadbalancing:CreateTargetGroup\",\"elasticloadbalancing:CreateRule\",\"elasticloadbalancing:CreateLoadBalancer\",\"elasticloadbalancing:CreateListener\",\"ec2:CreateSecurityGroup\"],\"Resource\":\"*\",\"Condition\":{\"StringEquals\":{\"aws:RequestTag/eks:eks-cluster-name\":\"${aws:PrincipalTag/eks:eks-cluster-name}\"}}},{\"Sid\":\"ShieldProtection\",\"Effect\":\"Allow\",\"Action\":\"shield:CreateProtection\",\"Resource\":\"*\",\"Condition\":{\"StringEquals\":{\"aws:RequestTag/eks:eks-cluster-name\":\"${aws:PrincipalTag/eks:eks-cluster-name}\"}}},{\"Sid\":\"ShieldTagResource\",\"Effect\":\"Allow\",\"Action\":\"shield:TagResource\",\"Resource\":\"arn:aws:shield::*:protection/*\",\"Condition\":{\"StringEquals\":{\"aws:RequestTag/eks:eks-cluster-name\":\"${aws:PrincipalTag/eks:eks-cluster-name}\"}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "ec2:CreateFleet", + "ec2:CreateLaunchTemplate", + "ec2:RunInstances" + ], + "condition": [ + { + "test": "StringEquals", + "values": [ + "${aws:PrincipalTag/eks:eks-cluster-name}" + ], + "variable": "aws:RequestTag/eks:eks-cluster-name" + }, + { + "test": "StringLike", + "values": [ + "*" + ], + "variable": "aws:RequestTag/eks:kubernetes-node-class-name" + }, + { + "test": "StringLike", + "values": [ + "*" + ], + "variable": "aws:RequestTag/eks:kubernetes-node-pool-name" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "Compute" + }, + { + "actions": [ + "ec2:CreateSnapshot", + "ec2:CreateVolume" + ], + "condition": [ + { + "test": "StringEquals", + "values": [ + "${aws:PrincipalTag/eks:eks-cluster-name}" + ], + "variable": "aws:RequestTag/eks:eks-cluster-name" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:*:*:snapshot/*", + "arn:aws:ec2:*:*:volume/*" + ], + "sid": "Storage" + }, + { + "actions": [ + "ec2:CreateNetworkInterface" + ], + "condition": [ + { + "test": "StringEquals", + "values": [ + "${aws:PrincipalTag/eks:eks-cluster-name}" + ], + "variable": "aws:RequestTag/eks:eks-cluster-name" + }, + { + "test": "StringEquals", + "values": [ + "*" + ], + "variable": "aws:RequestTag/eks:kubernetes-cni-node-name" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "Networking" + }, + { + "actions": [ + "ec2:CreateSecurityGroup", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateRule", + "elasticloadbalancing:CreateTargetGroup" + ], + "condition": [ + { + "test": "StringEquals", + "values": [ + "${aws:PrincipalTag/eks:eks-cluster-name}" + ], + "variable": "aws:RequestTag/eks:eks-cluster-name" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "LoadBalancer" + }, + { + "actions": [ + "shield:CreateProtection" + ], + "condition": [ + { + "test": "StringEquals", + "values": [ + "${aws:PrincipalTag/eks:eks-cluster-name}" + ], + "variable": "aws:RequestTag/eks:eks-cluster-name" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "ShieldProtection" + }, + { + "actions": [ + "shield:TagResource" + ], + "condition": [ + { + "test": "StringEquals", + "values": [ + "${aws:PrincipalTag/eks:eks-cluster-name}" + ], + "variable": "aws:RequestTag/eks:eks-cluster-name" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:shield::*:protection/*" + ], + "sid": "ShieldTagResource" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.eks", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "node_assume_role_policy", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.eks", + "mode": "data", + "type": "aws_iam_session_context", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:sts::027423573553:assumed-role/Admin/kylhouns-Isengard", + "id": "arn:aws:sts::027423573553:assumed-role/Admin/kylhouns-Isengard", + "issuer_arn": "arn:aws:iam::027423573553:role/Admin", + "issuer_id": "AROAQMYUSQYYXMMZMGMPD", + "issuer_name": "Admin", + "session_name": "kylhouns-Isengard" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.eks", + "mode": "data", + "type": "aws_partition", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "dns_suffix": "amazonaws.com", + "id": "aws", + "partition": "aws", + "reverse_dns_prefix": "com.amazonaws" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.eks", + "mode": "data", + "type": "tls_certificate", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/tls\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "certificates": [ + { + "cert_pem": "-----BEGIN CERTIFICATE-----\nMIIEkjCCA3qgAwIBAgITBn+USionzfP6wq4rAfkI7rnExjANBgkqhkiG9w0BAQsF\nADCBmDELMAkGA1UEBhMCVVMxEDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNj\nb3R0c2RhbGUxJTAjBgNVBAoTHFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4x\nOzA5BgNVBAMTMlN0YXJmaWVsZCBTZXJ2aWNlcyBSb290IENlcnRpZmljYXRlIEF1\ndGhvcml0eSAtIEcyMB4XDTE1MDUyNTEyMDAwMFoXDTM3MTIzMTAxMDAwMFowOTEL\nMAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv\nb3QgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALJ4gHHKeNXj\nca9HgFB0fW7Y14h29Jlo91ghYPl0hAEvrAIthtOgQ3pOsqTQNroBvo3bSMgHFzZM\n9O6II8c+6zf1tRn4SWiw3te5djgdYZ6k/oI2peVKVuRF4fn9tBb6dNqcmzU5L/qw\nIFAGbHrQgLKm+a/sRxmPUDgH3KKHOVj4utWp+UhnMJbulHheb4mjUcAwhmahRWa6\nVOujw5H5SNz/0egwLX0tdHA114gk957EWW67c4cX8jJGKLhD+rcdqsq08p8kDi1L\n93FcXmn/6pUCyziKrlA4b9v7LWIbxcceVOF34GfID5yHI9Y/QCB/IIDEgEw+OyQm\njgSubJrIqg0CAwEAAaOCATEwggEtMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/\nBAQDAgGGMB0GA1UdDgQWBBSEGMyFNOy8DJSULghZnMeyEE4KCDAfBgNVHSMEGDAW\ngBScXwDfqgHXMCs4iKK4bUqc8hGRgzB4BggrBgEFBQcBAQRsMGowLgYIKwYBBQUH\nMAGGImh0dHA6Ly9vY3NwLnJvb3RnMi5hbWF6b250cnVzdC5jb20wOAYIKwYBBQUH\nMAKGLGh0dHA6Ly9jcnQucm9vdGcyLmFtYXpvbnRydXN0LmNvbS9yb290ZzIuY2Vy\nMD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly9jcmwucm9vdGcyLmFtYXpvbnRydXN0\nLmNvbS9yb290ZzIuY3JsMBEGA1UdIAQKMAgwBgYEVR0gADANBgkqhkiG9w0BAQsF\nAAOCAQEAYjdCXLwQtT6LLOkMm2xF4gcAevnFWAu5CIw+7bMlPLVvUOTNNWqnkzSW\nMiGpSESrnO09tKpzbeR/FoCJbM8oAxiDR3mjEH4wW6w7sGDgd9QIpuEdfF7Au/ma\neyKdpwAJfqxGF4PcnCZXmTA5YpaP7dreqsXMGz7KQ2hsVxa81Q4gLv7/wmpdLqBK\nbRRYh5TmOTFffHPLkIhqhBGWJ6bt2YFGpn6jcgAKUj6DiAdjd4lpFw85hdKrCEVN\n0FE6/V1dN2RMfjCyVSRCnTawXZwXgWHxyvkQAiSr6w10kY17RSlQOYiypok1JR4U\nakcjMS9cmvqtmg5iUaQqqcT5NJ0hGA==\n-----END CERTIFICATE-----\n", + "is_ca": true, + "issuer": "CN=Starfield Services Root Certificate Authority - G2,O=Starfield Technologies\\, Inc.,L=Scottsdale,ST=Arizona,C=US", + "max_path_length": -1, + "not_after": "2037-12-31T01:00:00Z", + "not_before": "2015-05-25T12:00:00Z", + "public_key_algorithm": "RSA", + "serial_number": "144918191876577076464031512351042010504348870", + "sha1_fingerprint": "06b25927c42a721631c1efd9431e648fa62e1e39", + "signature_algorithm": "SHA256-RSA", + "subject": "CN=Amazon Root CA 1,O=Amazon,C=US", + "version": 3 + }, + { + "cert_pem": "-----BEGIN CERTIFICATE-----\nMIIEXjCCA0agAwIBAgITB3MSTyqVLj7Rili9uF0bwM5fJzANBgkqhkiG9w0BAQsF\nADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6\nb24gUm9vdCBDQSAxMB4XDTIyMDgyMzIyMjYzNVoXDTMwMDgyMzIyMjYzNVowPDEL\nMAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEcMBoGA1UEAxMTQW1hem9uIFJT\nQSAyMDQ4IE0wNDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM3pVR6A\nlQOp4xe776FdePXyejgA38mYx1ou9/jrpV6Sfn+/oqBKgwhY6ePsQHHQayWBJdBn\nv4Wz363qRI4XUh9swBFJ11TnZ3LqOMvHmWq2+loA0QPtOfXdJ2fHBLrBrngtJ/GB\n0p5olAVYrSZgvQGP16Rf8ddtNyxEEhYm3HuhmNi+vSeAq1tLYJPAvRCXonTpWdSD\nxY6hvdmxlqTYi82AtBXSfpGQ58HHM0hw0C6aQakghrwWi5fGslLOqzpimNMIsT7c\nqa0GJx6JfKqJqmQQNplO2h8n9ZsFJgBowof01ppdoLAWg6caMOM0om/VILKaa30F\n9W/r8Qjah7ltGVkCAwEAAaOCAVowggFWMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYD\nVR0PAQH/BAQDAgGGMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAdBgNV\nHQ4EFgQUH1KSYVaCVH+BZtgdPQqqMlyH3QgwHwYDVR0jBBgwFoAUhBjMhTTsvAyU\nlC4IWZzHshBOCggwewYIKwYBBQUHAQEEbzBtMC8GCCsGAQUFBzABhiNodHRwOi8v\nb2NzcC5yb290Y2ExLmFtYXpvbnRydXN0LmNvbTA6BggrBgEFBQcwAoYuaHR0cDov\nL2NydC5yb290Y2ExLmFtYXpvbnRydXN0LmNvbS9yb290Y2ExLmNlcjA/BgNVHR8E\nODA2MDSgMqAwhi5odHRwOi8vY3JsLnJvb3RjYTEuYW1hem9udHJ1c3QuY29tL3Jv\nb3RjYTEuY3JsMBMGA1UdIAQMMAowCAYGZ4EMAQIBMA0GCSqGSIb3DQEBCwUAA4IB\nAQA+1O5UsAaNuW3lHzJtpNGwBnZd9QEYFtxpiAnIaV4qApnGS9OCw5ZPwie7YSlD\nZF5yyFPsFhUC2Q9uJHY/CRV1b5hIiGH0+6+w5PgKiY1MWuWT8VAaJjFxvuhM7a/e\nfN2TIw1Wd6WCl6YRisunjQOrSP+unqC8A540JNyZ1JOE3jVqat3OZBGgMvihdj2w\nY23EpwesrKiQzkHzmvSH67PVW4ycbPy08HVZnBxZ5NrlGG9bwXR3fNTaz+c+Ej6c\n5AnwI3qkOFgSkg3Y75cdFz6pO/olK+e3AqygAcv0WjzmkDPuBjssuZjCHMC56oH3\nGJkV29Di2j5prHJbwZjG1inU\n-----END CERTIFICATE-----\n", + "is_ca": true, + "issuer": "CN=Amazon Root CA 1,O=Amazon,C=US", + "max_path_length": 0, + "not_after": "2030-08-23T22:26:35Z", + "not_before": "2022-08-23T22:26:35Z", + "public_key_algorithm": "RSA", + "serial_number": "166129359584585245282080020727744908546170663", + "sha1_fingerprint": "e7b8b5a6743ce1b2f17b041de59558a41472d70c", + "signature_algorithm": "SHA256-RSA", + "subject": "CN=Amazon RSA 2048 M04,O=Amazon,C=US", + "version": 3 + }, + { + "cert_pem": "-----BEGIN CERTIFICATE-----\nMIIF3DCCBMSgAwIBAgIQDQV9d1Pckakw9/avbXdLkzANBgkqhkiG9w0BAQsFADA8\nMQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRwwGgYDVQQDExNBbWF6b24g\nUlNBIDIwNDggTTA0MB4XDTI2MDIwMTAwMDAwMFoXDTI3MDMwMjIzNTk1OVowKDEm\nMCQGA1UEAwwdKi5la3MudXMtd2VzdC0yLmFtYXpvbmF3cy5jb20wggEiMA0GCSqG\nSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDUrtybLxNh6kicFXKyZ80JadVMf81AAUe7\n+uNNPUoirB6E/eJOMYFcpOMRegRr1+Bj5GkIllgpQkblG5FML0JpoV0U7KKJ/KLq\nXcaoMso2Z1enLUrXsp09jWaLC0JAvWVfQ1zywX7l7gDv1VYQUjEMnHjtmIRBDOvI\n6yfIeQ/vamYC4wBUTgYuNDt2M1W6dHlWFSckQj1gAoQJeTF7MFvTxNphD/UuJ+/z\ngVCg5EX5zY4J1m0WdYNYHBO1k86FgkBwPpfl3CoTk+dr/HtNtecEoyXVcar3kGdP\n6XaEQjTpSmnLMwiUdXeaO6y/ycjucTlEdCJymTlQi7oCrrFQkb89AgMBAAGjggLs\nMIIC6DAfBgNVHSMEGDAWgBQfUpJhVoJUf4Fm2B09CqoyXIfdCDAdBgNVHQ4EFgQU\nS6cIgvI3+EyOstSdOMBsAFpq+PAwKAYDVR0RBCEwH4IdKi5la3MudXMtd2VzdC0y\nLmFtYXpvbmF3cy5jb20wEwYDVR0gBAwwCjAIBgZngQwBAgEwDgYDVR0PAQH/BAQD\nAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMDsGA1UdHwQ0MDIwMKAuoCyGKmh0dHA6\nLy9jcmwucjJtMDQuYW1hem9udHJ1c3QuY29tL3IybTA0LmNybDB1BggrBgEFBQcB\nAQRpMGcwLQYIKwYBBQUHMAGGIWh0dHA6Ly9vY3NwLnIybTA0LmFtYXpvbnRydXN0\nLmNvbTA2BggrBgEFBQcwAoYqaHR0cDovL2NydC5yMm0wNC5hbWF6b250cnVzdC5j\nb20vcjJtMDQuY2VyMAwGA1UdEwEB/wQCMAAwggF+BgorBgEEAdZ5AgQCBIIBbgSC\nAWoBaAB2AExj3JjlnB2riPYeij3ero+rRKM3e1+blMP7oZz8wb4mAAABnBadJOkA\nAAQDAEcwRQIhAMefEVRWmXr8K/0vkj+gjXXQTbRXw9CQqnmdz1aetKPiAiBzBfYw\nh9vz9Qhgsn1IUqNClM6X0+0X6ewSuA3gbbFVpAB3AByfaCzp+vBFaVD4G5aKh93b\nMhDYTObIsuOCUkrEz1mfAAABnBadJOgAAAQDAEgwRgIhAJuTj3hURPwYbgM5Ws/z\ngyzFrm6zqJhNm/9URF9ChtSeAiEAzMZR7unXCg/JwSBEztJvfTHX6kmG8E8WPpr1\niaOxvHcAdQBgTJqven93XwHUBvySDciZ6wscffjJUhv6+hd3O5eLyQAAAZwWnSXI\nAAAEAwBGMEQCIFP2mgbUhpxZWXI00D4UfkBddoqj4KYmlFKI3yO7OvEeAiA72WEB\n0YRrnqH9RiMQCRPmpx0Jeb9W0meF5XD3XhuqljANBgkqhkiG9w0BAQsFAAOCAQEA\nyU9ItZOuE69q8N2HTgB3Mn6KVOTbFjvHEAXZg9j+wxLBzvpUmQmkvtg/bw5coOYZ\nkLtI4Lwl0qPMgVB5jWjVMq2CR2wpebMUxfTektNV+831n1Nd0v/J0k1Pw9IX7BNz\nstV5kMcw1UmcWZjRK4DFqiv5hHLl/+Pm/Kkszk80ljNihZhlTRUWA0iQPHrkiwMe\nj1NbGk5QWJUCI9X0OfmfvwKILD383irwt+b78J78clcg4GjixGz8MU4v8GzQwc6x\nGMzH9ui7E7TktJ4XwjAS6ST4Qm+zmZdpnYwnTpSmbF2PztIF2G7zW0Bm1RyjXnun\nCEEHt64OMliW3oLkjdFKbg==\n-----END CERTIFICATE-----\n", + "is_ca": false, + "issuer": "CN=Amazon RSA 2048 M04,O=Amazon,C=US", + "max_path_length": -1, + "not_after": "2027-03-02T23:59:59Z", + "not_before": "2026-02-01T00:00:00Z", + "public_key_algorithm": "RSA", + "serial_number": "17308470184802283501806495047878134675", + "sha1_fingerprint": "a84554e91b2f69f6035a36b183074726b12ffa9c", + "signature_algorithm": "SHA256-RSA", + "subject": "CN=*.eks.us-west-2.amazonaws.com", + "version": 3 + } + ], + "content": null, + "id": "f97f646c2cd14cc0db0f757f0fccc96abbbe2af5", + "url": "https://oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D", + "verify_chain": true + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.eks", + "mode": "managed", + "type": "aws_cloudwatch_log_group", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:logs:us-west-2:027423573553:log-group:/aws/eks/observability-stack/cluster", + "id": "/aws/eks/observability-stack/cluster", + "kms_key_id": "", + "log_group_class": "STANDARD", + "name": "/aws/eks/observability-stack/cluster", + "name_prefix": "", + "retention_in_days": 90, + "skip_destroy": false, + "tags": { + "Name": "/aws/eks/observability-stack/cluster" + }, + "tags_all": { + "ManagedBy": "terraform", + "Name": "/aws/eks/observability-stack/cluster", + "Project": "observability-stack" + } + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "create_before_destroy": true + } + ] + }, + { + "module": "module.eks", + "mode": "managed", + "type": "aws_ec2_tag", + "name": "cluster_primary_security_group", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.eks", + "mode": "managed", + "type": "aws_eks_access_entry", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": "cluster_creator", + "schema_version": 0, + "attributes": { + "access_entry_arn": "arn:aws:eks:us-west-2:027423573553:access-entry/observability-stack/role/027423573553/Admin/9ece8151-38a7-c96c-85de-6283191ef290", + "cluster_name": "observability-stack", + "created_at": "2026-03-18T20:28:13Z", + "id": "observability-stack:arn:aws:iam::027423573553:role/Admin", + "kubernetes_groups": [], + "modified_at": "2026-03-18T20:28:13Z", + "principal_arn": "arn:aws:iam::027423573553:role/Admin", + "tags": {}, + "tags_all": { + "ManagedBy": "terraform", + "Project": "observability-stack" + }, + "timeouts": null, + "type": "STANDARD", + "user_name": "arn:aws:sts::027423573553:assumed-role/Admin/{{SessionName}}" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwfX0=", + "dependencies": [ + "data.aws_availability_zones.available", + "module.eks.aws_cloudwatch_log_group.this", + "module.eks.aws_eks_cluster.this", + "module.eks.aws_iam_policy.cni_ipv6_policy", + "module.eks.aws_iam_role.eks_auto", + "module.eks.aws_iam_role.this", + "module.eks.aws_iam_role_policy_attachment.this", + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.eks.aws_security_group_rule.cluster", + "module.eks.aws_security_group_rule.node", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_policy_document.cni_ipv6_policy", + "module.eks.data.aws_iam_policy_document.node_assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.data.aws_partition.current", + "module.eks.module.kms.aws_kms_external_key.this", + "module.eks.module.kms.aws_kms_key.this", + "module.eks.module.kms.aws_kms_replica_external_key.this", + "module.eks.module.kms.aws_kms_replica_key.this", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.eks", + "mode": "managed", + "type": "aws_eks_access_policy_association", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": "cluster_creator_admin", + "schema_version": 0, + "attributes": { + "access_scope": [ + { + "namespaces": [], + "type": "cluster" + } + ], + "associated_at": "2026-03-18 20:28:14.487 +0000 UTC", + "cluster_name": "observability-stack", + "id": "observability-stack#arn:aws:iam::027423573553:role/Admin#arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy", + "modified_at": "2026-03-18 20:28:14.487 +0000 UTC", + "policy_arn": "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy", + "principal_arn": "arn:aws:iam::027423573553:role/Admin", + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwfX0=", + "dependencies": [ + "data.aws_availability_zones.available", + "module.eks.aws_cloudwatch_log_group.this", + "module.eks.aws_eks_access_entry.this", + "module.eks.aws_eks_cluster.this", + "module.eks.aws_iam_policy.cni_ipv6_policy", + "module.eks.aws_iam_role.eks_auto", + "module.eks.aws_iam_role.this", + "module.eks.aws_iam_role_policy_attachment.this", + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.eks.aws_security_group_rule.cluster", + "module.eks.aws_security_group_rule.node", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_policy_document.cni_ipv6_policy", + "module.eks.data.aws_iam_policy_document.node_assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.data.aws_partition.current", + "module.eks.module.kms.aws_kms_external_key.this", + "module.eks.module.kms.aws_kms_key.this", + "module.eks.module.kms.aws_kms_replica_external_key.this", + "module.eks.module.kms.aws_kms_replica_key.this", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.eks", + "mode": "managed", + "type": "aws_eks_addon", + "name": "before_compute", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.eks", + "mode": "managed", + "type": "aws_eks_addon", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": "aws-ebs-csi-driver", + "schema_version": 0, + "attributes": { + "addon_name": "aws-ebs-csi-driver", + "addon_version": "v1.56.0-eksbuild.1", + "arn": "arn:aws:eks:us-west-2:027423573553:addon/observability-stack/aws-ebs-csi-driver/1cce8155-62e0-1e46-d21c-1448c01c8528", + "cluster_name": "observability-stack", + "configuration_values": "", + "created_at": "2026-03-18T20:37:19Z", + "id": "observability-stack:aws-ebs-csi-driver", + "modified_at": "2026-03-18T20:38:26Z", + "pod_identity_association": [], + "preserve": true, + "resolve_conflicts": null, + "resolve_conflicts_on_create": "OVERWRITE", + "resolve_conflicts_on_update": "OVERWRITE", + "service_account_role_arn": "arn:aws:iam::027423573553:role/observability-stack-ebs-csi", + "tags": {}, + "tags_all": { + "ManagedBy": "terraform", + "Project": "observability-stack" + }, + "timeouts": { + "create": null, + "delete": null, + "update": null + } + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjI0MDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_availability_zones.available", + "module.ebs_csi_irsa.aws_iam_role.this", + "module.ebs_csi_irsa.data.aws_caller_identity.current", + "module.ebs_csi_irsa.data.aws_iam_policy_document.this", + "module.ebs_csi_irsa.data.aws_partition.current", + "module.eks.aws_cloudwatch_log_group.this", + "module.eks.aws_eks_cluster.this", + "module.eks.aws_iam_openid_connect_provider.oidc_provider", + "module.eks.aws_iam_policy.cni_ipv6_policy", + "module.eks.aws_iam_role.eks_auto", + "module.eks.aws_iam_role.this", + "module.eks.aws_iam_role_policy_attachment.this", + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.eks.aws_security_group_rule.cluster", + "module.eks.aws_security_group_rule.node", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_eks_addon_version.this", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_policy_document.cni_ipv6_policy", + "module.eks.data.aws_iam_policy_document.node_assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.data.aws_partition.current", + "module.eks.data.tls_certificate.this", + "module.eks.module.eks_managed_node_group.aws_autoscaling_schedule.this", + "module.eks.module.eks_managed_node_group.aws_eks_node_group.this", + "module.eks.module.eks_managed_node_group.aws_iam_role.this", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy.this", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy_attachment.additional", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy_attachment.this", + "module.eks.module.eks_managed_node_group.aws_launch_template.this", + "module.eks.module.eks_managed_node_group.aws_placement_group.this", + "module.eks.module.eks_managed_node_group.data.aws_caller_identity.current", + "module.eks.module.eks_managed_node_group.data.aws_ec2_instance_type.this", + "module.eks.module.eks_managed_node_group.data.aws_ec2_instance_type_offerings.this", + "module.eks.module.eks_managed_node_group.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.eks_managed_node_group.data.aws_iam_policy_document.role", + "module.eks.module.eks_managed_node_group.data.aws_partition.current", + "module.eks.module.eks_managed_node_group.data.aws_ssm_parameter.ami", + "module.eks.module.eks_managed_node_group.data.aws_subnets.placement_group", + "module.eks.module.eks_managed_node_group.module.user_data.data.cloudinit_config.al2023_eks_managed_node_group", + "module.eks.module.eks_managed_node_group.module.user_data.data.cloudinit_config.linux_eks_managed_node_group", + "module.eks.module.eks_managed_node_group.module.user_data.null_resource.validate_cluster_service_cidr", + "module.eks.module.fargate_profile.aws_eks_fargate_profile.this", + "module.eks.module.fargate_profile.aws_iam_role.this", + "module.eks.module.fargate_profile.aws_iam_role_policy.this", + "module.eks.module.fargate_profile.aws_iam_role_policy_attachment.additional", + "module.eks.module.fargate_profile.aws_iam_role_policy_attachment.this", + "module.eks.module.fargate_profile.data.aws_caller_identity.current", + "module.eks.module.fargate_profile.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.fargate_profile.data.aws_iam_policy_document.role", + "module.eks.module.fargate_profile.data.aws_partition.current", + "module.eks.module.fargate_profile.data.aws_region.current", + "module.eks.module.kms.aws_kms_external_key.this", + "module.eks.module.kms.aws_kms_key.this", + "module.eks.module.kms.aws_kms_replica_external_key.this", + "module.eks.module.kms.aws_kms_replica_key.this", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current", + "module.eks.module.self_managed_node_group.aws_autoscaling_group.this", + "module.eks.module.self_managed_node_group.aws_autoscaling_schedule.this", + "module.eks.module.self_managed_node_group.aws_eks_access_entry.this", + "module.eks.module.self_managed_node_group.aws_iam_instance_profile.this", + "module.eks.module.self_managed_node_group.aws_iam_role.this", + "module.eks.module.self_managed_node_group.aws_iam_role_policy.this", + "module.eks.module.self_managed_node_group.aws_iam_role_policy_attachment.additional", + "module.eks.module.self_managed_node_group.aws_iam_role_policy_attachment.this", + "module.eks.module.self_managed_node_group.aws_launch_template.this", + "module.eks.module.self_managed_node_group.aws_placement_group.this", + "module.eks.module.self_managed_node_group.data.aws_caller_identity.current", + "module.eks.module.self_managed_node_group.data.aws_ec2_instance_type.this", + "module.eks.module.self_managed_node_group.data.aws_ec2_instance_type_offerings.this", + "module.eks.module.self_managed_node_group.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.self_managed_node_group.data.aws_iam_policy_document.role", + "module.eks.module.self_managed_node_group.data.aws_partition.current", + "module.eks.module.self_managed_node_group.data.aws_ssm_parameter.ami", + "module.eks.module.self_managed_node_group.data.aws_subnets.placement_group", + "module.eks.module.self_managed_node_group.module.user_data.data.cloudinit_config.al2023_eks_managed_node_group", + "module.eks.module.self_managed_node_group.module.user_data.data.cloudinit_config.linux_eks_managed_node_group", + "module.eks.module.self_managed_node_group.module.user_data.null_resource.validate_cluster_service_cidr", + "module.eks.time_sleep.this", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": "coredns", + "schema_version": 0, + "attributes": { + "addon_name": "coredns", + "addon_version": "v1.11.4-eksbuild.2", + "arn": "arn:aws:eks:us-west-2:027423573553:addon/observability-stack/coredns/d6ce8155-62e5-5137-dfcc-62867816d439", + "cluster_name": "observability-stack", + "configuration_values": "", + "created_at": "2026-03-18T20:37:19Z", + "id": "observability-stack:coredns", + "modified_at": "2026-03-18T20:37:31Z", + "pod_identity_association": [], + "preserve": true, + "resolve_conflicts": null, + "resolve_conflicts_on_create": "OVERWRITE", + "resolve_conflicts_on_update": "OVERWRITE", + "service_account_role_arn": "", + "tags": {}, + "tags_all": { + "ManagedBy": "terraform", + "Project": "observability-stack" + }, + "timeouts": { + "create": null, + "delete": null, + "update": null + } + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjI0MDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_availability_zones.available", + "module.ebs_csi_irsa.aws_iam_role.this", + "module.ebs_csi_irsa.data.aws_caller_identity.current", + "module.ebs_csi_irsa.data.aws_iam_policy_document.this", + "module.ebs_csi_irsa.data.aws_partition.current", + "module.eks.aws_cloudwatch_log_group.this", + "module.eks.aws_eks_cluster.this", + "module.eks.aws_iam_openid_connect_provider.oidc_provider", + "module.eks.aws_iam_policy.cni_ipv6_policy", + "module.eks.aws_iam_role.eks_auto", + "module.eks.aws_iam_role.this", + "module.eks.aws_iam_role_policy_attachment.this", + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.eks.aws_security_group_rule.cluster", + "module.eks.aws_security_group_rule.node", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_eks_addon_version.this", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_policy_document.cni_ipv6_policy", + "module.eks.data.aws_iam_policy_document.node_assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.data.aws_partition.current", + "module.eks.data.tls_certificate.this", + "module.eks.module.eks_managed_node_group.aws_autoscaling_schedule.this", + "module.eks.module.eks_managed_node_group.aws_eks_node_group.this", + "module.eks.module.eks_managed_node_group.aws_iam_role.this", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy.this", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy_attachment.additional", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy_attachment.this", + "module.eks.module.eks_managed_node_group.aws_launch_template.this", + "module.eks.module.eks_managed_node_group.aws_placement_group.this", + "module.eks.module.eks_managed_node_group.data.aws_caller_identity.current", + "module.eks.module.eks_managed_node_group.data.aws_ec2_instance_type.this", + "module.eks.module.eks_managed_node_group.data.aws_ec2_instance_type_offerings.this", + "module.eks.module.eks_managed_node_group.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.eks_managed_node_group.data.aws_iam_policy_document.role", + "module.eks.module.eks_managed_node_group.data.aws_partition.current", + "module.eks.module.eks_managed_node_group.data.aws_ssm_parameter.ami", + "module.eks.module.eks_managed_node_group.data.aws_subnets.placement_group", + "module.eks.module.eks_managed_node_group.module.user_data.data.cloudinit_config.al2023_eks_managed_node_group", + "module.eks.module.eks_managed_node_group.module.user_data.data.cloudinit_config.linux_eks_managed_node_group", + "module.eks.module.eks_managed_node_group.module.user_data.null_resource.validate_cluster_service_cidr", + "module.eks.module.fargate_profile.aws_eks_fargate_profile.this", + "module.eks.module.fargate_profile.aws_iam_role.this", + "module.eks.module.fargate_profile.aws_iam_role_policy.this", + "module.eks.module.fargate_profile.aws_iam_role_policy_attachment.additional", + "module.eks.module.fargate_profile.aws_iam_role_policy_attachment.this", + "module.eks.module.fargate_profile.data.aws_caller_identity.current", + "module.eks.module.fargate_profile.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.fargate_profile.data.aws_iam_policy_document.role", + "module.eks.module.fargate_profile.data.aws_partition.current", + "module.eks.module.fargate_profile.data.aws_region.current", + "module.eks.module.kms.aws_kms_external_key.this", + "module.eks.module.kms.aws_kms_key.this", + "module.eks.module.kms.aws_kms_replica_external_key.this", + "module.eks.module.kms.aws_kms_replica_key.this", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current", + "module.eks.module.self_managed_node_group.aws_autoscaling_group.this", + "module.eks.module.self_managed_node_group.aws_autoscaling_schedule.this", + "module.eks.module.self_managed_node_group.aws_eks_access_entry.this", + "module.eks.module.self_managed_node_group.aws_iam_instance_profile.this", + "module.eks.module.self_managed_node_group.aws_iam_role.this", + "module.eks.module.self_managed_node_group.aws_iam_role_policy.this", + "module.eks.module.self_managed_node_group.aws_iam_role_policy_attachment.additional", + "module.eks.module.self_managed_node_group.aws_iam_role_policy_attachment.this", + "module.eks.module.self_managed_node_group.aws_launch_template.this", + "module.eks.module.self_managed_node_group.aws_placement_group.this", + "module.eks.module.self_managed_node_group.data.aws_caller_identity.current", + "module.eks.module.self_managed_node_group.data.aws_ec2_instance_type.this", + "module.eks.module.self_managed_node_group.data.aws_ec2_instance_type_offerings.this", + "module.eks.module.self_managed_node_group.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.self_managed_node_group.data.aws_iam_policy_document.role", + "module.eks.module.self_managed_node_group.data.aws_partition.current", + "module.eks.module.self_managed_node_group.data.aws_ssm_parameter.ami", + "module.eks.module.self_managed_node_group.data.aws_subnets.placement_group", + "module.eks.module.self_managed_node_group.module.user_data.data.cloudinit_config.al2023_eks_managed_node_group", + "module.eks.module.self_managed_node_group.module.user_data.data.cloudinit_config.linux_eks_managed_node_group", + "module.eks.module.self_managed_node_group.module.user_data.null_resource.validate_cluster_service_cidr", + "module.eks.time_sleep.this", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": "eks-pod-identity-agent", + "schema_version": 0, + "attributes": { + "addon_name": "eks-pod-identity-agent", + "addon_version": "v1.3.10-eksbuild.2", + "arn": "arn:aws:eks:us-west-2:027423573553:addon/observability-stack/eks-pod-identity-agent/96ce8155-62df-5974-2a55-aef2242ebd35", + "cluster_name": "observability-stack", + "configuration_values": "", + "created_at": "2026-03-18T20:37:19Z", + "id": "observability-stack:eks-pod-identity-agent", + "modified_at": "2026-03-18T20:37:55Z", + "pod_identity_association": [], + "preserve": true, + "resolve_conflicts": null, + "resolve_conflicts_on_create": "OVERWRITE", + "resolve_conflicts_on_update": "OVERWRITE", + "service_account_role_arn": "", + "tags": {}, + "tags_all": { + "ManagedBy": "terraform", + "Project": "observability-stack" + }, + "timeouts": { + "create": null, + "delete": null, + "update": null + } + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjI0MDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_availability_zones.available", + "module.ebs_csi_irsa.aws_iam_role.this", + "module.ebs_csi_irsa.data.aws_caller_identity.current", + "module.ebs_csi_irsa.data.aws_iam_policy_document.this", + "module.ebs_csi_irsa.data.aws_partition.current", + "module.eks.aws_cloudwatch_log_group.this", + "module.eks.aws_eks_cluster.this", + "module.eks.aws_iam_openid_connect_provider.oidc_provider", + "module.eks.aws_iam_policy.cni_ipv6_policy", + "module.eks.aws_iam_role.eks_auto", + "module.eks.aws_iam_role.this", + "module.eks.aws_iam_role_policy_attachment.this", + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.eks.aws_security_group_rule.cluster", + "module.eks.aws_security_group_rule.node", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_eks_addon_version.this", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_policy_document.cni_ipv6_policy", + "module.eks.data.aws_iam_policy_document.node_assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.data.aws_partition.current", + "module.eks.data.tls_certificate.this", + "module.eks.module.eks_managed_node_group.aws_autoscaling_schedule.this", + "module.eks.module.eks_managed_node_group.aws_eks_node_group.this", + "module.eks.module.eks_managed_node_group.aws_iam_role.this", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy.this", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy_attachment.additional", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy_attachment.this", + "module.eks.module.eks_managed_node_group.aws_launch_template.this", + "module.eks.module.eks_managed_node_group.aws_placement_group.this", + "module.eks.module.eks_managed_node_group.data.aws_caller_identity.current", + "module.eks.module.eks_managed_node_group.data.aws_ec2_instance_type.this", + "module.eks.module.eks_managed_node_group.data.aws_ec2_instance_type_offerings.this", + "module.eks.module.eks_managed_node_group.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.eks_managed_node_group.data.aws_iam_policy_document.role", + "module.eks.module.eks_managed_node_group.data.aws_partition.current", + "module.eks.module.eks_managed_node_group.data.aws_ssm_parameter.ami", + "module.eks.module.eks_managed_node_group.data.aws_subnets.placement_group", + "module.eks.module.eks_managed_node_group.module.user_data.data.cloudinit_config.al2023_eks_managed_node_group", + "module.eks.module.eks_managed_node_group.module.user_data.data.cloudinit_config.linux_eks_managed_node_group", + "module.eks.module.eks_managed_node_group.module.user_data.null_resource.validate_cluster_service_cidr", + "module.eks.module.fargate_profile.aws_eks_fargate_profile.this", + "module.eks.module.fargate_profile.aws_iam_role.this", + "module.eks.module.fargate_profile.aws_iam_role_policy.this", + "module.eks.module.fargate_profile.aws_iam_role_policy_attachment.additional", + "module.eks.module.fargate_profile.aws_iam_role_policy_attachment.this", + "module.eks.module.fargate_profile.data.aws_caller_identity.current", + "module.eks.module.fargate_profile.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.fargate_profile.data.aws_iam_policy_document.role", + "module.eks.module.fargate_profile.data.aws_partition.current", + "module.eks.module.fargate_profile.data.aws_region.current", + "module.eks.module.kms.aws_kms_external_key.this", + "module.eks.module.kms.aws_kms_key.this", + "module.eks.module.kms.aws_kms_replica_external_key.this", + "module.eks.module.kms.aws_kms_replica_key.this", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current", + "module.eks.module.self_managed_node_group.aws_autoscaling_group.this", + "module.eks.module.self_managed_node_group.aws_autoscaling_schedule.this", + "module.eks.module.self_managed_node_group.aws_eks_access_entry.this", + "module.eks.module.self_managed_node_group.aws_iam_instance_profile.this", + "module.eks.module.self_managed_node_group.aws_iam_role.this", + "module.eks.module.self_managed_node_group.aws_iam_role_policy.this", + "module.eks.module.self_managed_node_group.aws_iam_role_policy_attachment.additional", + "module.eks.module.self_managed_node_group.aws_iam_role_policy_attachment.this", + "module.eks.module.self_managed_node_group.aws_launch_template.this", + "module.eks.module.self_managed_node_group.aws_placement_group.this", + "module.eks.module.self_managed_node_group.data.aws_caller_identity.current", + "module.eks.module.self_managed_node_group.data.aws_ec2_instance_type.this", + "module.eks.module.self_managed_node_group.data.aws_ec2_instance_type_offerings.this", + "module.eks.module.self_managed_node_group.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.self_managed_node_group.data.aws_iam_policy_document.role", + "module.eks.module.self_managed_node_group.data.aws_partition.current", + "module.eks.module.self_managed_node_group.data.aws_ssm_parameter.ami", + "module.eks.module.self_managed_node_group.data.aws_subnets.placement_group", + "module.eks.module.self_managed_node_group.module.user_data.data.cloudinit_config.al2023_eks_managed_node_group", + "module.eks.module.self_managed_node_group.module.user_data.data.cloudinit_config.linux_eks_managed_node_group", + "module.eks.module.self_managed_node_group.module.user_data.null_resource.validate_cluster_service_cidr", + "module.eks.time_sleep.this", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": "kube-proxy", + "schema_version": 0, + "attributes": { + "addon_name": "kube-proxy", + "addon_version": "v1.32.6-eksbuild.12", + "arn": "arn:aws:eks:us-west-2:027423573553:addon/observability-stack/kube-proxy/0cce8155-62dd-23e7-cb36-895daa5ca9b6", + "cluster_name": "observability-stack", + "configuration_values": "", + "created_at": "2026-03-18T20:37:19Z", + "id": "observability-stack:kube-proxy", + "modified_at": "2026-03-18T20:37:55Z", + "pod_identity_association": [], + "preserve": true, + "resolve_conflicts": null, + "resolve_conflicts_on_create": "OVERWRITE", + "resolve_conflicts_on_update": "OVERWRITE", + "service_account_role_arn": "", + "tags": {}, + "tags_all": { + "ManagedBy": "terraform", + "Project": "observability-stack" + }, + "timeouts": { + "create": null, + "delete": null, + "update": null + } + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjI0MDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_availability_zones.available", + "module.ebs_csi_irsa.aws_iam_role.this", + "module.ebs_csi_irsa.data.aws_caller_identity.current", + "module.ebs_csi_irsa.data.aws_iam_policy_document.this", + "module.ebs_csi_irsa.data.aws_partition.current", + "module.eks.aws_cloudwatch_log_group.this", + "module.eks.aws_eks_cluster.this", + "module.eks.aws_iam_openid_connect_provider.oidc_provider", + "module.eks.aws_iam_policy.cni_ipv6_policy", + "module.eks.aws_iam_role.eks_auto", + "module.eks.aws_iam_role.this", + "module.eks.aws_iam_role_policy_attachment.this", + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.eks.aws_security_group_rule.cluster", + "module.eks.aws_security_group_rule.node", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_eks_addon_version.this", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_policy_document.cni_ipv6_policy", + "module.eks.data.aws_iam_policy_document.node_assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.data.aws_partition.current", + "module.eks.data.tls_certificate.this", + "module.eks.module.eks_managed_node_group.aws_autoscaling_schedule.this", + "module.eks.module.eks_managed_node_group.aws_eks_node_group.this", + "module.eks.module.eks_managed_node_group.aws_iam_role.this", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy.this", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy_attachment.additional", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy_attachment.this", + "module.eks.module.eks_managed_node_group.aws_launch_template.this", + "module.eks.module.eks_managed_node_group.aws_placement_group.this", + "module.eks.module.eks_managed_node_group.data.aws_caller_identity.current", + "module.eks.module.eks_managed_node_group.data.aws_ec2_instance_type.this", + "module.eks.module.eks_managed_node_group.data.aws_ec2_instance_type_offerings.this", + "module.eks.module.eks_managed_node_group.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.eks_managed_node_group.data.aws_iam_policy_document.role", + "module.eks.module.eks_managed_node_group.data.aws_partition.current", + "module.eks.module.eks_managed_node_group.data.aws_ssm_parameter.ami", + "module.eks.module.eks_managed_node_group.data.aws_subnets.placement_group", + "module.eks.module.eks_managed_node_group.module.user_data.data.cloudinit_config.al2023_eks_managed_node_group", + "module.eks.module.eks_managed_node_group.module.user_data.data.cloudinit_config.linux_eks_managed_node_group", + "module.eks.module.eks_managed_node_group.module.user_data.null_resource.validate_cluster_service_cidr", + "module.eks.module.fargate_profile.aws_eks_fargate_profile.this", + "module.eks.module.fargate_profile.aws_iam_role.this", + "module.eks.module.fargate_profile.aws_iam_role_policy.this", + "module.eks.module.fargate_profile.aws_iam_role_policy_attachment.additional", + "module.eks.module.fargate_profile.aws_iam_role_policy_attachment.this", + "module.eks.module.fargate_profile.data.aws_caller_identity.current", + "module.eks.module.fargate_profile.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.fargate_profile.data.aws_iam_policy_document.role", + "module.eks.module.fargate_profile.data.aws_partition.current", + "module.eks.module.fargate_profile.data.aws_region.current", + "module.eks.module.kms.aws_kms_external_key.this", + "module.eks.module.kms.aws_kms_key.this", + "module.eks.module.kms.aws_kms_replica_external_key.this", + "module.eks.module.kms.aws_kms_replica_key.this", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current", + "module.eks.module.self_managed_node_group.aws_autoscaling_group.this", + "module.eks.module.self_managed_node_group.aws_autoscaling_schedule.this", + "module.eks.module.self_managed_node_group.aws_eks_access_entry.this", + "module.eks.module.self_managed_node_group.aws_iam_instance_profile.this", + "module.eks.module.self_managed_node_group.aws_iam_role.this", + "module.eks.module.self_managed_node_group.aws_iam_role_policy.this", + "module.eks.module.self_managed_node_group.aws_iam_role_policy_attachment.additional", + "module.eks.module.self_managed_node_group.aws_iam_role_policy_attachment.this", + "module.eks.module.self_managed_node_group.aws_launch_template.this", + "module.eks.module.self_managed_node_group.aws_placement_group.this", + "module.eks.module.self_managed_node_group.data.aws_caller_identity.current", + "module.eks.module.self_managed_node_group.data.aws_ec2_instance_type.this", + "module.eks.module.self_managed_node_group.data.aws_ec2_instance_type_offerings.this", + "module.eks.module.self_managed_node_group.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.self_managed_node_group.data.aws_iam_policy_document.role", + "module.eks.module.self_managed_node_group.data.aws_partition.current", + "module.eks.module.self_managed_node_group.data.aws_ssm_parameter.ami", + "module.eks.module.self_managed_node_group.data.aws_subnets.placement_group", + "module.eks.module.self_managed_node_group.module.user_data.data.cloudinit_config.al2023_eks_managed_node_group", + "module.eks.module.self_managed_node_group.module.user_data.data.cloudinit_config.linux_eks_managed_node_group", + "module.eks.module.self_managed_node_group.module.user_data.null_resource.validate_cluster_service_cidr", + "module.eks.time_sleep.this", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": "vpc-cni", + "schema_version": 0, + "attributes": { + "addon_name": "vpc-cni", + "addon_version": "v1.20.4-eksbuild.2", + "arn": "arn:aws:eks:us-west-2:027423573553:addon/observability-stack/vpc-cni/16ce8155-62df-b015-0f55-9d22607b5bf1", + "cluster_name": "observability-stack", + "configuration_values": "", + "created_at": "2026-03-18T20:37:19Z", + "id": "observability-stack:vpc-cni", + "modified_at": "2026-03-18T20:37:56Z", + "pod_identity_association": [], + "preserve": true, + "resolve_conflicts": null, + "resolve_conflicts_on_create": "OVERWRITE", + "resolve_conflicts_on_update": "OVERWRITE", + "service_account_role_arn": "", + "tags": {}, + "tags_all": { + "ManagedBy": "terraform", + "Project": "observability-stack" + }, + "timeouts": { + "create": null, + "delete": null, + "update": null + } + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjI0MDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_availability_zones.available", + "module.ebs_csi_irsa.aws_iam_role.this", + "module.ebs_csi_irsa.data.aws_caller_identity.current", + "module.ebs_csi_irsa.data.aws_iam_policy_document.this", + "module.ebs_csi_irsa.data.aws_partition.current", + "module.eks.aws_cloudwatch_log_group.this", + "module.eks.aws_eks_cluster.this", + "module.eks.aws_iam_openid_connect_provider.oidc_provider", + "module.eks.aws_iam_policy.cni_ipv6_policy", + "module.eks.aws_iam_role.eks_auto", + "module.eks.aws_iam_role.this", + "module.eks.aws_iam_role_policy_attachment.this", + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.eks.aws_security_group_rule.cluster", + "module.eks.aws_security_group_rule.node", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_eks_addon_version.this", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_policy_document.cni_ipv6_policy", + "module.eks.data.aws_iam_policy_document.node_assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.data.aws_partition.current", + "module.eks.data.tls_certificate.this", + "module.eks.module.eks_managed_node_group.aws_autoscaling_schedule.this", + "module.eks.module.eks_managed_node_group.aws_eks_node_group.this", + "module.eks.module.eks_managed_node_group.aws_iam_role.this", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy.this", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy_attachment.additional", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy_attachment.this", + "module.eks.module.eks_managed_node_group.aws_launch_template.this", + "module.eks.module.eks_managed_node_group.aws_placement_group.this", + "module.eks.module.eks_managed_node_group.data.aws_caller_identity.current", + "module.eks.module.eks_managed_node_group.data.aws_ec2_instance_type.this", + "module.eks.module.eks_managed_node_group.data.aws_ec2_instance_type_offerings.this", + "module.eks.module.eks_managed_node_group.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.eks_managed_node_group.data.aws_iam_policy_document.role", + "module.eks.module.eks_managed_node_group.data.aws_partition.current", + "module.eks.module.eks_managed_node_group.data.aws_ssm_parameter.ami", + "module.eks.module.eks_managed_node_group.data.aws_subnets.placement_group", + "module.eks.module.eks_managed_node_group.module.user_data.data.cloudinit_config.al2023_eks_managed_node_group", + "module.eks.module.eks_managed_node_group.module.user_data.data.cloudinit_config.linux_eks_managed_node_group", + "module.eks.module.eks_managed_node_group.module.user_data.null_resource.validate_cluster_service_cidr", + "module.eks.module.fargate_profile.aws_eks_fargate_profile.this", + "module.eks.module.fargate_profile.aws_iam_role.this", + "module.eks.module.fargate_profile.aws_iam_role_policy.this", + "module.eks.module.fargate_profile.aws_iam_role_policy_attachment.additional", + "module.eks.module.fargate_profile.aws_iam_role_policy_attachment.this", + "module.eks.module.fargate_profile.data.aws_caller_identity.current", + "module.eks.module.fargate_profile.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.fargate_profile.data.aws_iam_policy_document.role", + "module.eks.module.fargate_profile.data.aws_partition.current", + "module.eks.module.fargate_profile.data.aws_region.current", + "module.eks.module.kms.aws_kms_external_key.this", + "module.eks.module.kms.aws_kms_key.this", + "module.eks.module.kms.aws_kms_replica_external_key.this", + "module.eks.module.kms.aws_kms_replica_key.this", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current", + "module.eks.module.self_managed_node_group.aws_autoscaling_group.this", + "module.eks.module.self_managed_node_group.aws_autoscaling_schedule.this", + "module.eks.module.self_managed_node_group.aws_eks_access_entry.this", + "module.eks.module.self_managed_node_group.aws_iam_instance_profile.this", + "module.eks.module.self_managed_node_group.aws_iam_role.this", + "module.eks.module.self_managed_node_group.aws_iam_role_policy.this", + "module.eks.module.self_managed_node_group.aws_iam_role_policy_attachment.additional", + "module.eks.module.self_managed_node_group.aws_iam_role_policy_attachment.this", + "module.eks.module.self_managed_node_group.aws_launch_template.this", + "module.eks.module.self_managed_node_group.aws_placement_group.this", + "module.eks.module.self_managed_node_group.data.aws_caller_identity.current", + "module.eks.module.self_managed_node_group.data.aws_ec2_instance_type.this", + "module.eks.module.self_managed_node_group.data.aws_ec2_instance_type_offerings.this", + "module.eks.module.self_managed_node_group.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.self_managed_node_group.data.aws_iam_policy_document.role", + "module.eks.module.self_managed_node_group.data.aws_partition.current", + "module.eks.module.self_managed_node_group.data.aws_ssm_parameter.ami", + "module.eks.module.self_managed_node_group.data.aws_subnets.placement_group", + "module.eks.module.self_managed_node_group.module.user_data.data.cloudinit_config.al2023_eks_managed_node_group", + "module.eks.module.self_managed_node_group.module.user_data.data.cloudinit_config.linux_eks_managed_node_group", + "module.eks.module.self_managed_node_group.module.user_data.null_resource.validate_cluster_service_cidr", + "module.eks.time_sleep.this", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.eks", + "mode": "managed", + "type": "aws_eks_cluster", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 1, + "attributes": { + "access_config": [ + { + "authentication_mode": "API_AND_CONFIG_MAP", + "bootstrap_cluster_creator_admin_permissions": false + } + ], + "arn": "arn:aws:eks:us-west-2:027423573553:cluster/observability-stack", + "bootstrap_self_managed_addons": true, + "certificate_authority": [ + { + "data": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURCVENDQWUyZ0F3SUJBZ0lJWUw3U21MRHdoQ1F3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TmpBek1UZ3lNREU1TURsYUZ3MHpOakF6TVRVeU1ESTBNRGxhTUJVeApFekFSQmdOVkJBTVRDbXQxWW1WeWJtVjBaWE13Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLCkFvSUJBUURoYjF3aHBHSVlpa3U5TUV6Mnl3ck5DMnNHNUVnL3M3Tkd1Q1BaYzREOWxaclk0dE9ldmhyZFcvTWoKTHp3T21POXBTMnVxd2FFWnBDTmxMVWdDMXV6Z3JJRi9abFRqU2x6c0VJaldYbFRUQWE1N05FWmpLR28xeFJKTwpBbFovZy9TaHVXcFd2NFU1UHNLN2NlK3lwbDI0NVIzRzVrVnFWbkkwdkRwMjU1YVg1V0N6bnpSUm5yNU9WVE5QCndmMGh3bC9QUEpvYVRLYkhKcGk5QWRmMVIrbFdWZlFEdmhKckcwN2JBSGNncWhGYnNLd0JkWVlaK2JWYzdCVmQKd0dnb2NqYU4xMkI1SnlnKzBjUmFHN3NEdmdibk9zQ2hMbVdJeWRwL0tEVldSOGwwWU54dkdBR1c4R1I3S3AyUApLTHV1R290bVRyZ2xMN1pZZkJVRjQxU1o0VVBOQWdNQkFBR2pXVEJYTUE0R0ExVWREd0VCL3dRRUF3SUNwREFQCkJnTlZIUk1CQWY4RUJUQURBUUgvTUIwR0ExVWREZ1FXQkJRQlc3RG1rSGpQUmkycWpLR01uM2I2cnRWTThqQVYKQmdOVkhSRUVEakFNZ2dwcmRXSmxjbTVsZEdWek1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQlBBS3NwcjFCdwo4Tk9qTWVGTTZ6NEpsQ0pSaVRBNWVxd01XdDZpcGk0a2xnT0l0WDEwV2M4S25IeEpIME9uNG9CdVo5d1NsQmRTCmxocHk4elpmSXkreVVKVFRuMC9SdWlmbVdtd2NpNXlFZzZjWmpzclh1TVA2Nm5aektZNzdVVitVb05INzlKNlYKdlBOeVBQaVo4N1ZIdzJ2SVFGWFFjczIyQ09LQnB1YUNpWGZONUROdnpiaHJLWGQ3d3NaMTRLY1NEaTVBK3hrTwpGWHRTRFVHS09IdXNZcEQ5L3R3NkZ1d0lMYWY4WFFBb2toK0Z3aHc5OWltU0hzVlkrKys2dGQycVhKT0J6TjVNCnlVUi9lWGhTZGdpazFTSTFNZVZ5c1d2RDFJejlhNFNsUFJlZG1jSGRxdC9tejRLUFFaMjBGUmVsN0NWQVEyOHEKaDlTeXV3UFRPcWc2Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K" + } + ], + "cluster_id": null, + "compute_config": [], + "created_at": "2026-03-18T20:18:51Z", + "enabled_cluster_log_types": [ + "api", + "audit", + "authenticator" + ], + "encryption_config": [ + { + "provider": [ + { + "key_arn": "arn:aws:kms:us-west-2:027423573553:key/12030f91-64b4-40e3-b7ad-77340e0391c7" + } + ], + "resources": [ + "secrets" + ] + } + ], + "endpoint": "https://91D986DCB52447A7F312838B02FCD62D.gr7.us-west-2.eks.amazonaws.com", + "force_update_version": null, + "id": "observability-stack", + "identity": [ + { + "oidc": [ + { + "issuer": "https://oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D" + } + ] + } + ], + "kubernetes_network_config": [ + { + "elastic_load_balancing": [ + { + "enabled": false + } + ], + "ip_family": "ipv4", + "service_ipv4_cidr": "172.20.0.0/16", + "service_ipv6_cidr": "" + } + ], + "name": "observability-stack", + "outpost_config": [], + "platform_version": "eks.39", + "remote_network_config": [], + "role_arn": "arn:aws:iam::027423573553:role/observability-stack-cluster-20260318201827553600000003", + "status": "ACTIVE", + "storage_config": [], + "tags": { + "terraform-aws-modules": "eks" + }, + "tags_all": { + "ManagedBy": "terraform", + "Project": "observability-stack", + "terraform-aws-modules": "eks" + }, + "timeouts": { + "create": null, + "delete": null, + "update": null + }, + "upgrade_policy": [ + { + "support_type": "EXTENDED" + } + ], + "version": "1.32", + "vpc_config": [ + { + "cluster_security_group_id": "sg-03c8bf9784bd7274e", + "endpoint_private_access": true, + "endpoint_public_access": true, + "public_access_cidrs": [ + "0.0.0.0/0" + ], + "security_group_ids": [ + "sg-0a09a2f98bc2b809b" + ], + "subnet_ids": [ + "subnet-00bc6b6598ab09efb", + "subnet-06f8647c688701ae4", + "subnet-0862c3b38b530fd97" + ], + "vpc_id": "vpc-03aefcf5aa0581d7a" + } + ], + "zonal_shift_config": [] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjkwMDAwMDAwMDAwMCwidXBkYXRlIjozNjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=", + "dependencies": [ + "data.aws_availability_zones.available", + "module.eks.aws_cloudwatch_log_group.this", + "module.eks.aws_iam_policy.cni_ipv6_policy", + "module.eks.aws_iam_role.eks_auto", + "module.eks.aws_iam_role.this", + "module.eks.aws_iam_role_policy_attachment.this", + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.eks.aws_security_group_rule.cluster", + "module.eks.aws_security_group_rule.node", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_policy_document.cni_ipv6_policy", + "module.eks.data.aws_iam_policy_document.node_assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.data.aws_partition.current", + "module.eks.module.kms.aws_kms_external_key.this", + "module.eks.module.kms.aws_kms_key.this", + "module.eks.module.kms.aws_kms_replica_external_key.this", + "module.eks.module.kms.aws_kms_replica_key.this", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ], + "create_before_destroy": true + } + ] + }, + { + "module": "module.eks", + "mode": "managed", + "type": "aws_eks_identity_provider_config", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.eks", + "mode": "managed", + "type": "aws_iam_openid_connect_provider", + "name": "oidc_provider", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::027423573553:oidc-provider/oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D", + "client_id_list": [ + "sts.amazonaws.com" + ], + "id": "arn:aws:iam::027423573553:oidc-provider/oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D", + "tags": { + "Name": "observability-stack-eks-irsa" + }, + "tags_all": { + "ManagedBy": "terraform", + "Name": "observability-stack-eks-irsa", + "Project": "observability-stack" + }, + "thumbprint_list": [ + "06b25927c42a721631c1efd9431e648fa62e1e39" + ], + "url": "oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_availability_zones.available", + "module.eks.aws_cloudwatch_log_group.this", + "module.eks.aws_eks_cluster.this", + "module.eks.aws_iam_policy.cni_ipv6_policy", + "module.eks.aws_iam_role.eks_auto", + "module.eks.aws_iam_role.this", + "module.eks.aws_iam_role_policy_attachment.this", + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.eks.aws_security_group_rule.cluster", + "module.eks.aws_security_group_rule.node", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_policy_document.cni_ipv6_policy", + "module.eks.data.aws_iam_policy_document.node_assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.data.aws_partition.current", + "module.eks.data.tls_certificate.this", + "module.eks.module.kms.aws_kms_external_key.this", + "module.eks.module.kms.aws_kms_key.this", + "module.eks.module.kms.aws_kms_replica_external_key.this", + "module.eks.module.kms.aws_kms_replica_key.this", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.eks", + "mode": "managed", + "type": "aws_iam_policy", + "name": "cluster_encryption", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::027423573553:policy/observability-stack-cluster-ClusterEncryption20260318201849029100000012", + "attachment_count": 1, + "description": "Cluster encryption policy to allow cluster role to utilize CMK provided", + "id": "arn:aws:iam::027423573553:policy/observability-stack-cluster-ClusterEncryption20260318201849029100000012", + "name": "observability-stack-cluster-ClusterEncryption20260318201849029100000012", + "name_prefix": "observability-stack-cluster-ClusterEncryption", + "path": "/", + "policy": "{\"Statement\":[{\"Action\":[\"kms:Encrypt\",\"kms:Decrypt\",\"kms:ListGrants\",\"kms:DescribeKey\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:kms:us-west-2:027423573553:key/12030f91-64b4-40e3-b7ad-77340e0391c7\"}],\"Version\":\"2012-10-17\"}", + "policy_id": "ANPAQMYUSQYYRD4S5DT4O", + "tags": {}, + "tags_all": { + "ManagedBy": "terraform", + "Project": "observability-stack" + } + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.eks.aws_iam_role.this", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.module.kms.aws_kms_external_key.this", + "module.eks.module.kms.aws_kms_key.this", + "module.eks.module.kms.aws_kms_replica_external_key.this", + "module.eks.module.kms.aws_kms_replica_key.this", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current" + ] + } + ] + }, + { + "module": "module.eks", + "mode": "managed", + "type": "aws_iam_policy", + "name": "cni_ipv6_policy", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.eks", + "mode": "managed", + "type": "aws_iam_policy", + "name": "custom", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::027423573553:policy/observability-stack-cluster-20260318201827554500000004", + "attachment_count": 1, + "description": "", + "id": "arn:aws:iam::027423573553:policy/observability-stack-cluster-20260318201827554500000004", + "name": "observability-stack-cluster-20260318201827554500000004", + "name_prefix": "observability-stack-cluster-", + "path": "/", + "policy": "{\"Statement\":[{\"Action\":[\"ec2:RunInstances\",\"ec2:CreateLaunchTemplate\",\"ec2:CreateFleet\"],\"Condition\":{\"StringEquals\":{\"aws:RequestTag/eks:eks-cluster-name\":\"${aws:PrincipalTag/eks:eks-cluster-name}\"},\"StringLike\":{\"aws:RequestTag/eks:kubernetes-node-class-name\":\"*\",\"aws:RequestTag/eks:kubernetes-node-pool-name\":\"*\"}},\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"Compute\"},{\"Action\":[\"ec2:CreateVolume\",\"ec2:CreateSnapshot\"],\"Condition\":{\"StringEquals\":{\"aws:RequestTag/eks:eks-cluster-name\":\"${aws:PrincipalTag/eks:eks-cluster-name}\"}},\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:ec2:*:*:volume/*\",\"arn:aws:ec2:*:*:snapshot/*\"],\"Sid\":\"Storage\"},{\"Action\":\"ec2:CreateNetworkInterface\",\"Condition\":{\"StringEquals\":{\"aws:RequestTag/eks:eks-cluster-name\":\"${aws:PrincipalTag/eks:eks-cluster-name}\",\"aws:RequestTag/eks:kubernetes-cni-node-name\":\"*\"}},\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"Networking\"},{\"Action\":[\"elasticloadbalancing:CreateTargetGroup\",\"elasticloadbalancing:CreateRule\",\"elasticloadbalancing:CreateLoadBalancer\",\"elasticloadbalancing:CreateListener\",\"ec2:CreateSecurityGroup\"],\"Condition\":{\"StringEquals\":{\"aws:RequestTag/eks:eks-cluster-name\":\"${aws:PrincipalTag/eks:eks-cluster-name}\"}},\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"LoadBalancer\"},{\"Action\":\"shield:CreateProtection\",\"Condition\":{\"StringEquals\":{\"aws:RequestTag/eks:eks-cluster-name\":\"${aws:PrincipalTag/eks:eks-cluster-name}\"}},\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"ShieldProtection\"},{\"Action\":\"shield:TagResource\",\"Condition\":{\"StringEquals\":{\"aws:RequestTag/eks:eks-cluster-name\":\"${aws:PrincipalTag/eks:eks-cluster-name}\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:shield::*:protection/*\",\"Sid\":\"ShieldTagResource\"}],\"Version\":\"2012-10-17\"}", + "policy_id": "ANPAQMYUSQYYXYGTCPHVU", + "tags": {}, + "tags_all": { + "ManagedBy": "terraform", + "Project": "observability-stack" + } + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.eks.data.aws_iam_policy_document.custom", + "module.eks.data.aws_partition.current" + ] + } + ] + }, + { + "module": "module.eks", + "mode": "managed", + "type": "aws_iam_role", + "name": "eks_auto", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.eks", + "mode": "managed", + "type": "aws_iam_role", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::027423573553:role/observability-stack-cluster-20260318201827553600000003", + "assume_role_policy": "{\"Statement\":[{\"Action\":[\"sts:TagSession\",\"sts:AssumeRole\"],\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"eks.amazonaws.com\"},\"Sid\":\"EKSClusterAssumeRole\"}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-03-18T20:18:27Z", + "description": "", + "force_detach_policies": true, + "id": "observability-stack-cluster-20260318201827553600000003", + "inline_policy": [], + "managed_policy_arns": [ + "arn:aws:iam::027423573553:policy/observability-stack-cluster-20260318201827554500000004", + "arn:aws:iam::027423573553:policy/observability-stack-cluster-ClusterEncryption20260318201849029100000012", + "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", + "arn:aws:iam::aws:policy/AmazonEKSVPCResourceController" + ], + "max_session_duration": 3600, + "name": "observability-stack-cluster-20260318201827553600000003", + "name_prefix": "observability-stack-cluster-", + "path": "/", + "permissions_boundary": "", + "tags": {}, + "tags_all": { + "ManagedBy": "terraform", + "Project": "observability-stack" + }, + "unique_id": "AROAQMYUSQYY2A3WUUJBE" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.eks.data.aws_iam_policy_document.assume_role_policy" + ], + "create_before_destroy": true + } + ] + }, + { + "module": "module.eks", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "additional", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.eks", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "cluster_encryption", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "id": "observability-stack-cluster-20260318201827553600000003-20260318201849597300000013", + "policy_arn": "arn:aws:iam::027423573553:policy/observability-stack-cluster-ClusterEncryption20260318201849029100000012", + "role": "observability-stack-cluster-20260318201827553600000003" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.eks.aws_iam_policy.cluster_encryption", + "module.eks.aws_iam_role.this", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.module.kms.aws_kms_external_key.this", + "module.eks.module.kms.aws_kms_key.this", + "module.eks.module.kms.aws_kms_replica_external_key.this", + "module.eks.module.kms.aws_kms_replica_key.this", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current" + ] + } + ] + }, + { + "module": "module.eks", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "custom", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "id": "observability-stack-cluster-20260318201827553600000003-20260318201828429000000008", + "policy_arn": "arn:aws:iam::027423573553:policy/observability-stack-cluster-20260318201827554500000004", + "role": "observability-stack-cluster-20260318201827553600000003" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.eks.aws_iam_policy.custom", + "module.eks.aws_iam_role.this", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_policy_document.custom", + "module.eks.data.aws_partition.current" + ] + } + ] + }, + { + "module": "module.eks", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "eks_auto", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.eks", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "eks_auto_additional", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.eks", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": "AmazonEKSClusterPolicy", + "schema_version": 0, + "attributes": { + "id": "observability-stack-cluster-20260318201827553600000003-2026031820182842940000000a", + "policy_arn": "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", + "role": "observability-stack-cluster-20260318201827553600000003" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.eks.aws_iam_role.this", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_partition.current" + ], + "create_before_destroy": true + }, + { + "index_key": "AmazonEKSVPCResourceController", + "schema_version": 0, + "attributes": { + "id": "observability-stack-cluster-20260318201827553600000003-2026031820182843710000000c", + "policy_arn": "arn:aws:iam::aws:policy/AmazonEKSVPCResourceController", + "role": "observability-stack-cluster-20260318201827553600000003" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.eks.aws_iam_role.this", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_partition.current" + ], + "create_before_destroy": true + } + ] + }, + { + "module": "module.eks", + "mode": "managed", + "type": "aws_security_group", + "name": "cluster", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-west-2:027423573553:security-group/sg-0a09a2f98bc2b809b", + "description": "EKS cluster security group", + "egress": [], + "id": "sg-0a09a2f98bc2b809b", + "ingress": [ + { + "cidr_blocks": [], + "description": "Node groups to cluster API", + "from_port": 443, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [ + "sg-039c64dc512590a29" + ], + "self": false, + "to_port": 443 + } + ], + "name": "observability-stack-cluster-2026031820184013550000000f", + "name_prefix": "observability-stack-cluster-", + "owner_id": "027423573553", + "revoke_rules_on_delete": false, + "tags": { + "Name": "observability-stack-cluster" + }, + "tags_all": { + "ManagedBy": "terraform", + "Name": "observability-stack-cluster", + "Project": "observability-stack" + }, + "timeouts": null, + "vpc_id": "vpc-03aefcf5aa0581d7a" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6OTAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=", + "dependencies": [ + "module.vpc.aws_vpc.this" + ], + "create_before_destroy": true + } + ] + }, + { + "module": "module.eks", + "mode": "managed", + "type": "aws_security_group", + "name": "node", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-west-2:027423573553:security-group/sg-039c64dc512590a29", + "description": "EKS node shared security group", + "egress": [ + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "Allow all egress", + "from_port": 0, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "-1", + "security_groups": [], + "self": false, + "to_port": 0 + } + ], + "id": "sg-039c64dc512590a29", + "ingress": [ + { + "cidr_blocks": [], + "description": "Cluster API to node 4443/tcp webhook", + "from_port": 4443, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [ + "sg-0a09a2f98bc2b809b" + ], + "self": false, + "to_port": 4443 + }, + { + "cidr_blocks": [], + "description": "Cluster API to node 6443/tcp webhook", + "from_port": 6443, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [ + "sg-0a09a2f98bc2b809b" + ], + "self": false, + "to_port": 6443 + }, + { + "cidr_blocks": [], + "description": "Cluster API to node 8443/tcp webhook", + "from_port": 8443, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [ + "sg-0a09a2f98bc2b809b" + ], + "self": false, + "to_port": 8443 + }, + { + "cidr_blocks": [], + "description": "Cluster API to node 9443/tcp webhook", + "from_port": 9443, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [ + "sg-0a09a2f98bc2b809b" + ], + "self": false, + "to_port": 9443 + }, + { + "cidr_blocks": [], + "description": "Cluster API to node groups", + "from_port": 443, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [ + "sg-0a09a2f98bc2b809b" + ], + "self": false, + "to_port": 443 + }, + { + "cidr_blocks": [], + "description": "Cluster API to node kubelets", + "from_port": 10250, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [ + "sg-0a09a2f98bc2b809b" + ], + "self": false, + "to_port": 10250 + }, + { + "cidr_blocks": [], + "description": "Node to node CoreDNS UDP", + "from_port": 53, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "udp", + "security_groups": [], + "self": true, + "to_port": 53 + }, + { + "cidr_blocks": [], + "description": "Node to node CoreDNS", + "from_port": 53, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": true, + "to_port": 53 + }, + { + "cidr_blocks": [], + "description": "Node to node ingress on ephemeral ports", + "from_port": 1025, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": true, + "to_port": 65535 + }, + { + "cidr_blocks": [], + "description": "elbv2.k8s.aws/targetGroupBinding=shared", + "from_port": 5601, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [ + "sg-0ec86f6fd3561da11" + ], + "self": false, + "to_port": 5601 + } + ], + "name": "observability-stack-node-20260318201840542000000010", + "name_prefix": "observability-stack-node-", + "owner_id": "027423573553", + "revoke_rules_on_delete": false, + "tags": { + "Name": "observability-stack-node", + "kubernetes.io/cluster/observability-stack": "owned" + }, + "tags_all": { + "ManagedBy": "terraform", + "Name": "observability-stack-node", + "Project": "observability-stack", + "kubernetes.io/cluster/observability-stack": "owned" + }, + "timeouts": null, + "vpc_id": "vpc-03aefcf5aa0581d7a" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6OTAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=", + "dependencies": [ + "module.vpc.aws_vpc.this" + ], + "create_before_destroy": true + } + ] + }, + { + "module": "module.eks", + "mode": "managed", + "type": "aws_security_group_rule", + "name": "cluster", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": "ingress_nodes_443", + "schema_version": 2, + "attributes": { + "cidr_blocks": null, + "description": "Node groups to cluster API", + "from_port": 443, + "id": "sgrule-1181540205", + "ipv6_cidr_blocks": null, + "prefix_list_ids": null, + "protocol": "tcp", + "security_group_id": "sg-0a09a2f98bc2b809b", + "security_group_rule_id": "sgr-077f102466a349635", + "self": false, + "source_security_group_id": "sg-039c64dc512590a29", + "timeouts": null, + "to_port": 443, + "type": "ingress" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==", + "dependencies": [ + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.vpc.aws_vpc.this" + ], + "create_before_destroy": true + } + ] + }, + { + "module": "module.eks", + "mode": "managed", + "type": "aws_security_group_rule", + "name": "node", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": "egress_all", + "schema_version": 2, + "attributes": { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "Allow all egress", + "from_port": 0, + "id": "sgrule-2475787203", + "ipv6_cidr_blocks": null, + "prefix_list_ids": [], + "protocol": "-1", + "security_group_id": "sg-039c64dc512590a29", + "security_group_rule_id": "sgr-0f66e35b51a2a9856", + "self": false, + "source_security_group_id": null, + "timeouts": null, + "to_port": 0, + "type": "egress" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==", + "dependencies": [ + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.vpc.aws_vpc.this" + ], + "create_before_destroy": true + }, + { + "index_key": "ingress_cluster_443", + "schema_version": 2, + "attributes": { + "cidr_blocks": null, + "description": "Cluster API to node groups", + "from_port": 443, + "id": "sgrule-1462776238", + "ipv6_cidr_blocks": null, + "prefix_list_ids": [], + "protocol": "tcp", + "security_group_id": "sg-039c64dc512590a29", + "security_group_rule_id": "sgr-092cf55f6b375a4e1", + "self": false, + "source_security_group_id": "sg-0a09a2f98bc2b809b", + "timeouts": null, + "to_port": 443, + "type": "ingress" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==", + "dependencies": [ + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.vpc.aws_vpc.this" + ], + "create_before_destroy": true + }, + { + "index_key": "ingress_cluster_4443_webhook", + "schema_version": 2, + "attributes": { + "cidr_blocks": null, + "description": "Cluster API to node 4443/tcp webhook", + "from_port": 4443, + "id": "sgrule-2443875299", + "ipv6_cidr_blocks": null, + "prefix_list_ids": [], + "protocol": "tcp", + "security_group_id": "sg-039c64dc512590a29", + "security_group_rule_id": "sgr-008e3e3dfb3bfde5c", + "self": false, + "source_security_group_id": "sg-0a09a2f98bc2b809b", + "timeouts": null, + "to_port": 4443, + "type": "ingress" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==", + "dependencies": [ + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.vpc.aws_vpc.this" + ], + "create_before_destroy": true + }, + { + "index_key": "ingress_cluster_6443_webhook", + "schema_version": 2, + "attributes": { + "cidr_blocks": null, + "description": "Cluster API to node 6443/tcp webhook", + "from_port": 6443, + "id": "sgrule-1491661941", + "ipv6_cidr_blocks": null, + "prefix_list_ids": [], + "protocol": "tcp", + "security_group_id": "sg-039c64dc512590a29", + "security_group_rule_id": "sgr-05878181195aef7de", + "self": false, + "source_security_group_id": "sg-0a09a2f98bc2b809b", + "timeouts": null, + "to_port": 6443, + "type": "ingress" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==", + "dependencies": [ + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.vpc.aws_vpc.this" + ], + "create_before_destroy": true + }, + { + "index_key": "ingress_cluster_8443_webhook", + "schema_version": 2, + "attributes": { + "cidr_blocks": null, + "description": "Cluster API to node 8443/tcp webhook", + "from_port": 8443, + "id": "sgrule-1270138964", + "ipv6_cidr_blocks": null, + "prefix_list_ids": [], + "protocol": "tcp", + "security_group_id": "sg-039c64dc512590a29", + "security_group_rule_id": "sgr-0af67159d3876a1de", + "self": false, + "source_security_group_id": "sg-0a09a2f98bc2b809b", + "timeouts": null, + "to_port": 8443, + "type": "ingress" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==", + "dependencies": [ + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.vpc.aws_vpc.this" + ], + "create_before_destroy": true + }, + { + "index_key": "ingress_cluster_9443_webhook", + "schema_version": 2, + "attributes": { + "cidr_blocks": null, + "description": "Cluster API to node 9443/tcp webhook", + "from_port": 9443, + "id": "sgrule-789969823", + "ipv6_cidr_blocks": null, + "prefix_list_ids": [], + "protocol": "tcp", + "security_group_id": "sg-039c64dc512590a29", + "security_group_rule_id": "sgr-05c24769847b5b659", + "self": false, + "source_security_group_id": "sg-0a09a2f98bc2b809b", + "timeouts": null, + "to_port": 9443, + "type": "ingress" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==", + "dependencies": [ + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.vpc.aws_vpc.this" + ], + "create_before_destroy": true + }, + { + "index_key": "ingress_cluster_kubelet", + "schema_version": 2, + "attributes": { + "cidr_blocks": null, + "description": "Cluster API to node kubelets", + "from_port": 10250, + "id": "sgrule-1472426363", + "ipv6_cidr_blocks": null, + "prefix_list_ids": [], + "protocol": "tcp", + "security_group_id": "sg-039c64dc512590a29", + "security_group_rule_id": "sgr-0ffdd1c8be8659dce", + "self": false, + "source_security_group_id": "sg-0a09a2f98bc2b809b", + "timeouts": null, + "to_port": 10250, + "type": "ingress" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==", + "dependencies": [ + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.vpc.aws_vpc.this" + ], + "create_before_destroy": true + }, + { + "index_key": "ingress_nodes_ephemeral", + "schema_version": 2, + "attributes": { + "cidr_blocks": null, + "description": "Node to node ingress on ephemeral ports", + "from_port": 1025, + "id": "sgrule-1052607575", + "ipv6_cidr_blocks": null, + "prefix_list_ids": [], + "protocol": "tcp", + "security_group_id": "sg-039c64dc512590a29", + "security_group_rule_id": "sgr-0357697388ca61651", + "self": true, + "source_security_group_id": null, + "timeouts": null, + "to_port": 65535, + "type": "ingress" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==", + "dependencies": [ + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.vpc.aws_vpc.this" + ], + "create_before_destroy": true + }, + { + "index_key": "ingress_self_coredns_tcp", + "schema_version": 2, + "attributes": { + "cidr_blocks": null, + "description": "Node to node CoreDNS", + "from_port": 53, + "id": "sgrule-2727722642", + "ipv6_cidr_blocks": null, + "prefix_list_ids": [], + "protocol": "tcp", + "security_group_id": "sg-039c64dc512590a29", + "security_group_rule_id": "sgr-079d67d71e8513449", + "self": true, + "source_security_group_id": null, + "timeouts": null, + "to_port": 53, + "type": "ingress" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==", + "dependencies": [ + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.vpc.aws_vpc.this" + ], + "create_before_destroy": true + }, + { + "index_key": "ingress_self_coredns_udp", + "schema_version": 2, + "attributes": { + "cidr_blocks": null, + "description": "Node to node CoreDNS UDP", + "from_port": 53, + "id": "sgrule-113909117", + "ipv6_cidr_blocks": null, + "prefix_list_ids": [], + "protocol": "udp", + "security_group_id": "sg-039c64dc512590a29", + "security_group_rule_id": "sgr-03497585c3ae5e8b8", + "self": true, + "source_security_group_id": null, + "timeouts": null, + "to_port": 53, + "type": "ingress" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==", + "dependencies": [ + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.vpc.aws_vpc.this" + ], + "create_before_destroy": true + } + ] + }, + { + "module": "module.eks", + "mode": "managed", + "type": "time_sleep", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/time\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "create_duration": "30s", + "destroy_duration": null, + "id": "2026-03-18T20:28:43Z", + "triggers": { + "cluster_certificate_authority_data": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURCVENDQWUyZ0F3SUJBZ0lJWUw3U21MRHdoQ1F3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TmpBek1UZ3lNREU1TURsYUZ3MHpOakF6TVRVeU1ESTBNRGxhTUJVeApFekFSQmdOVkJBTVRDbXQxWW1WeWJtVjBaWE13Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLCkFvSUJBUURoYjF3aHBHSVlpa3U5TUV6Mnl3ck5DMnNHNUVnL3M3Tkd1Q1BaYzREOWxaclk0dE9ldmhyZFcvTWoKTHp3T21POXBTMnVxd2FFWnBDTmxMVWdDMXV6Z3JJRi9abFRqU2x6c0VJaldYbFRUQWE1N05FWmpLR28xeFJKTwpBbFovZy9TaHVXcFd2NFU1UHNLN2NlK3lwbDI0NVIzRzVrVnFWbkkwdkRwMjU1YVg1V0N6bnpSUm5yNU9WVE5QCndmMGh3bC9QUEpvYVRLYkhKcGk5QWRmMVIrbFdWZlFEdmhKckcwN2JBSGNncWhGYnNLd0JkWVlaK2JWYzdCVmQKd0dnb2NqYU4xMkI1SnlnKzBjUmFHN3NEdmdibk9zQ2hMbVdJeWRwL0tEVldSOGwwWU54dkdBR1c4R1I3S3AyUApLTHV1R290bVRyZ2xMN1pZZkJVRjQxU1o0VVBOQWdNQkFBR2pXVEJYTUE0R0ExVWREd0VCL3dRRUF3SUNwREFQCkJnTlZIUk1CQWY4RUJUQURBUUgvTUIwR0ExVWREZ1FXQkJRQlc3RG1rSGpQUmkycWpLR01uM2I2cnRWTThqQVYKQmdOVkhSRUVEakFNZ2dwcmRXSmxjbTVsZEdWek1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQlBBS3NwcjFCdwo4Tk9qTWVGTTZ6NEpsQ0pSaVRBNWVxd01XdDZpcGk0a2xnT0l0WDEwV2M4S25IeEpIME9uNG9CdVo5d1NsQmRTCmxocHk4elpmSXkreVVKVFRuMC9SdWlmbVdtd2NpNXlFZzZjWmpzclh1TVA2Nm5aektZNzdVVitVb05INzlKNlYKdlBOeVBQaVo4N1ZIdzJ2SVFGWFFjczIyQ09LQnB1YUNpWGZONUROdnpiaHJLWGQ3d3NaMTRLY1NEaTVBK3hrTwpGWHRTRFVHS09IdXNZcEQ5L3R3NkZ1d0lMYWY4WFFBb2toK0Z3aHc5OWltU0hzVlkrKys2dGQycVhKT0J6TjVNCnlVUi9lWGhTZGdpazFTSTFNZVZ5c1d2RDFJejlhNFNsUFJlZG1jSGRxdC9tejRLUFFaMjBGUmVsN0NWQVEyOHEKaDlTeXV3UFRPcWc2Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K", + "cluster_endpoint": "https://91D986DCB52447A7F312838B02FCD62D.gr7.us-west-2.eks.amazonaws.com", + "cluster_name": "observability-stack", + "cluster_service_cidr": "172.20.0.0/16", + "cluster_version": "1.32" + } + }, + "sensitive_attributes": [], + "dependencies": [ + "data.aws_availability_zones.available", + "module.eks.aws_cloudwatch_log_group.this", + "module.eks.aws_eks_cluster.this", + "module.eks.aws_iam_policy.cni_ipv6_policy", + "module.eks.aws_iam_role.eks_auto", + "module.eks.aws_iam_role.this", + "module.eks.aws_iam_role_policy_attachment.this", + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.eks.aws_security_group_rule.cluster", + "module.eks.aws_security_group_rule.node", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_policy_document.cni_ipv6_policy", + "module.eks.data.aws_iam_policy_document.node_assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.data.aws_partition.current", + "module.eks.module.kms.aws_kms_external_key.this", + "module.eks.module.kms.aws_kms_key.this", + "module.eks.module.kms.aws_kms_replica_external_key.this", + "module.eks.module.kms.aws_kms_replica_key.this", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ], + "create_before_destroy": true + } + ] + }, + { + "module": "module.eks.module.eks_managed_node_group[\"default\"]", + "mode": "data", + "type": "aws_caller_identity", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "account_id": "027423573553", + "arn": "arn:aws:sts::027423573553:assumed-role/Admin/kylhouns-Isengard", + "id": "027423573553", + "user_id": "AROAQMYUSQYYXMMZMGMPD:kylhouns-Isengard" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.eks.module.eks_managed_node_group[\"default\"]", + "mode": "data", + "type": "aws_ec2_instance_type", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.eks.module.eks_managed_node_group[\"default\"]", + "mode": "data", + "type": "aws_ec2_instance_type_offerings", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.eks.module.eks_managed_node_group[\"default\"]", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "assume_role_policy", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "id": "2560088296", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"EKSNodeAssumeRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"ec2.amazonaws.com\"\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"EKSNodeAssumeRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "ec2.amazonaws.com" + ], + "type": "Service" + } + ], + "resources": [], + "sid": "EKSNodeAssumeRole" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.eks.module.eks_managed_node_group[\"default\"]", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.eks.module.eks_managed_node_group[\"default\"]", + "mode": "data", + "type": "aws_partition", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "dns_suffix": "amazonaws.com", + "id": "aws", + "partition": "aws", + "reverse_dns_prefix": "com.amazonaws" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.eks.module.eks_managed_node_group[\"default\"]", + "mode": "data", + "type": "aws_ssm_parameter", + "name": "ami", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.eks.module.eks_managed_node_group[\"default\"]", + "mode": "data", + "type": "aws_subnets", + "name": "placement_group", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.eks.module.eks_managed_node_group[\"default\"]", + "mode": "managed", + "type": "aws_autoscaling_schedule", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.eks.module.eks_managed_node_group[\"default\"]", + "mode": "managed", + "type": "aws_eks_node_group", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "ami_type": "AL2023_x86_64_STANDARD", + "arn": "arn:aws:eks:us-west-2:027423573553:nodegroup/observability-stack/default-20260318202959387700000001/f6ce8152-07c7-695e-68a7-64098e591426", + "capacity_type": "ON_DEMAND", + "cluster_name": "observability-stack", + "disk_size": 0, + "force_update_version": null, + "id": "observability-stack:default-20260318202959387700000001", + "instance_types": [ + "m5.xlarge" + ], + "labels": {}, + "launch_template": [ + { + "id": "lt-0fde2bc540de7fc87", + "name": "default-20260318202843435600000017", + "version": "1" + } + ], + "node_group_name": "default-20260318202959387700000001", + "node_group_name_prefix": "default-", + "node_repair_config": [], + "node_role_arn": "arn:aws:iam::027423573553:role/default-eks-node-group-20260318201827553200000002", + "release_version": "1.32.12-20260304", + "remote_access": [], + "resources": [ + { + "autoscaling_groups": [ + { + "name": "eks-default-20260318202959387700000001-f6ce8152-07c7-695e-68a7-64098e591426" + } + ], + "remote_access_security_group_id": "" + } + ], + "scaling_config": [ + { + "desired_size": 4, + "max_size": 5, + "min_size": 4 + } + ], + "status": "ACTIVE", + "subnet_ids": [ + "subnet-00bc6b6598ab09efb", + "subnet-06f8647c688701ae4", + "subnet-0862c3b38b530fd97" + ], + "tags": { + "Name": "default" + }, + "tags_all": { + "ManagedBy": "terraform", + "Name": "default", + "Project": "observability-stack" + }, + "taint": [], + "timeouts": { + "create": null, + "delete": null, + "update": null + }, + "update_config": [ + { + "max_unavailable": 0, + "max_unavailable_percentage": 33 + } + ], + "version": "1.32" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozNjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInVwZGF0ZSI6MzYwMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_availability_zones.available", + "module.eks.aws_cloudwatch_log_group.this", + "module.eks.aws_eks_cluster.this", + "module.eks.aws_iam_policy.cni_ipv6_policy", + "module.eks.aws_iam_role.eks_auto", + "module.eks.aws_iam_role.this", + "module.eks.aws_iam_role_policy_attachment.this", + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.eks.aws_security_group_rule.cluster", + "module.eks.aws_security_group_rule.node", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_policy_document.cni_ipv6_policy", + "module.eks.data.aws_iam_policy_document.node_assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.data.aws_partition.current", + "module.eks.module.eks_managed_node_group.aws_iam_role.this", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy_attachment.additional", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy_attachment.this", + "module.eks.module.eks_managed_node_group.aws_launch_template.this", + "module.eks.module.eks_managed_node_group.aws_placement_group.this", + "module.eks.module.eks_managed_node_group.data.aws_caller_identity.current", + "module.eks.module.eks_managed_node_group.data.aws_ec2_instance_type.this", + "module.eks.module.eks_managed_node_group.data.aws_ec2_instance_type_offerings.this", + "module.eks.module.eks_managed_node_group.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.eks_managed_node_group.data.aws_partition.current", + "module.eks.module.eks_managed_node_group.data.aws_ssm_parameter.ami", + "module.eks.module.eks_managed_node_group.data.aws_subnets.placement_group", + "module.eks.module.eks_managed_node_group.module.user_data.data.cloudinit_config.al2023_eks_managed_node_group", + "module.eks.module.eks_managed_node_group.module.user_data.data.cloudinit_config.linux_eks_managed_node_group", + "module.eks.module.kms.aws_kms_external_key.this", + "module.eks.module.kms.aws_kms_key.this", + "module.eks.module.kms.aws_kms_replica_external_key.this", + "module.eks.module.kms.aws_kms_replica_key.this", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current", + "module.eks.time_sleep.this", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ], + "create_before_destroy": true + } + ] + }, + { + "module": "module.eks.module.eks_managed_node_group[\"default\"]", + "mode": "managed", + "type": "aws_iam_role", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::027423573553:role/default-eks-node-group-20260318201827553200000002", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"},\"Sid\":\"EKSNodeAssumeRole\"}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-03-18T20:18:27Z", + "description": "EKS managed node group IAM role", + "force_detach_policies": true, + "id": "default-eks-node-group-20260318201827553200000002", + "inline_policy": [], + "managed_policy_arns": [ + "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", + "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", + "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy" + ], + "max_session_duration": 3600, + "name": "default-eks-node-group-20260318201827553200000002", + "name_prefix": "default-eks-node-group-", + "path": "/", + "permissions_boundary": "", + "tags": {}, + "tags_all": { + "ManagedBy": "terraform", + "Project": "observability-stack" + }, + "unique_id": "AROAQMYUSQYYRLTLVDS3L" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_availability_zones.available", + "module.eks.module.eks_managed_node_group.data.aws_iam_policy_document.assume_role_policy", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ], + "create_before_destroy": true + } + ] + }, + { + "module": "module.eks.module.eks_managed_node_group[\"default\"]", + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.eks.module.eks_managed_node_group[\"default\"]", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "additional", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.eks.module.eks_managed_node_group[\"default\"]", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": "AmazonEC2ContainerRegistryReadOnly", + "schema_version": 0, + "attributes": { + "id": "default-eks-node-group-20260318201827553200000002-20260318201828420400000007", + "policy_arn": "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", + "role": "default-eks-node-group-20260318201827553200000002" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_availability_zones.available", + "module.eks.module.eks_managed_node_group.aws_iam_role.this", + "module.eks.module.eks_managed_node_group.data.aws_caller_identity.current", + "module.eks.module.eks_managed_node_group.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.eks_managed_node_group.data.aws_partition.current", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ], + "create_before_destroy": true + }, + { + "index_key": "AmazonEKSWorkerNodePolicy", + "schema_version": 0, + "attributes": { + "id": "default-eks-node-group-20260318201827553200000002-20260318201828429000000009", + "policy_arn": "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", + "role": "default-eks-node-group-20260318201827553200000002" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_availability_zones.available", + "module.eks.module.eks_managed_node_group.aws_iam_role.this", + "module.eks.module.eks_managed_node_group.data.aws_caller_identity.current", + "module.eks.module.eks_managed_node_group.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.eks_managed_node_group.data.aws_partition.current", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ], + "create_before_destroy": true + }, + { + "index_key": "AmazonEKS_CNI_Policy", + "schema_version": 0, + "attributes": { + "id": "default-eks-node-group-20260318201827553200000002-2026031820182843710000000b", + "policy_arn": "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", + "role": "default-eks-node-group-20260318201827553200000002" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_availability_zones.available", + "module.eks.module.eks_managed_node_group.aws_iam_role.this", + "module.eks.module.eks_managed_node_group.data.aws_caller_identity.current", + "module.eks.module.eks_managed_node_group.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.eks_managed_node_group.data.aws_partition.current", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ], + "create_before_destroy": true + } + ] + }, + { + "module": "module.eks.module.eks_managed_node_group[\"default\"]", + "mode": "managed", + "type": "aws_launch_template", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:ec2:us-west-2:027423573553:launch-template/lt-0fde2bc540de7fc87", + "block_device_mappings": [], + "capacity_reservation_specification": [], + "cpu_options": [], + "credit_specification": [], + "default_version": 1, + "description": "Custom launch template for default EKS managed node group", + "disable_api_stop": false, + "disable_api_termination": false, + "ebs_optimized": "", + "elastic_gpu_specifications": [], + "elastic_inference_accelerator": [], + "enclave_options": [], + "hibernation_options": [], + "iam_instance_profile": [], + "id": "lt-0fde2bc540de7fc87", + "image_id": "", + "instance_initiated_shutdown_behavior": "", + "instance_market_options": [], + "instance_requirements": [], + "instance_type": "", + "kernel_id": "", + "key_name": "", + "latest_version": 1, + "license_specification": [], + "maintenance_options": [], + "metadata_options": [ + { + "http_endpoint": "enabled", + "http_protocol_ipv6": "", + "http_put_response_hop_limit": 2, + "http_tokens": "required", + "instance_metadata_tags": "" + } + ], + "monitoring": [ + { + "enabled": true + } + ], + "name": "default-20260318202843435600000017", + "name_prefix": "default-", + "network_interfaces": [], + "placement": [], + "private_dns_name_options": [], + "ram_disk_id": "", + "security_group_names": [], + "tag_specifications": [ + { + "resource_type": "instance", + "tags": { + "Name": "default" + } + }, + { + "resource_type": "network-interface", + "tags": { + "Name": "default" + } + }, + { + "resource_type": "volume", + "tags": { + "Name": "default" + } + } + ], + "tags": {}, + "tags_all": { + "ManagedBy": "terraform", + "Project": "observability-stack" + }, + "update_default_version": true, + "user_data": "", + "vpc_security_group_ids": [ + "sg-039c64dc512590a29" + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_availability_zones.available", + "module.eks.aws_cloudwatch_log_group.this", + "module.eks.aws_eks_cluster.this", + "module.eks.aws_iam_policy.cni_ipv6_policy", + "module.eks.aws_iam_role.eks_auto", + "module.eks.aws_iam_role.this", + "module.eks.aws_iam_role_policy_attachment.this", + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.eks.aws_security_group_rule.cluster", + "module.eks.aws_security_group_rule.node", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_policy_document.cni_ipv6_policy", + "module.eks.data.aws_iam_policy_document.node_assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.data.aws_partition.current", + "module.eks.module.eks_managed_node_group.aws_iam_role.this", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy_attachment.additional", + "module.eks.module.eks_managed_node_group.aws_iam_role_policy_attachment.this", + "module.eks.module.eks_managed_node_group.aws_placement_group.this", + "module.eks.module.eks_managed_node_group.data.aws_caller_identity.current", + "module.eks.module.eks_managed_node_group.data.aws_ec2_instance_type.this", + "module.eks.module.eks_managed_node_group.data.aws_iam_policy_document.assume_role_policy", + "module.eks.module.eks_managed_node_group.data.aws_partition.current", + "module.eks.module.eks_managed_node_group.module.user_data.data.cloudinit_config.al2023_eks_managed_node_group", + "module.eks.module.eks_managed_node_group.module.user_data.data.cloudinit_config.linux_eks_managed_node_group", + "module.eks.module.kms.aws_kms_external_key.this", + "module.eks.module.kms.aws_kms_key.this", + "module.eks.module.kms.aws_kms_replica_external_key.this", + "module.eks.module.kms.aws_kms_replica_key.this", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current", + "module.eks.time_sleep.this", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ], + "create_before_destroy": true + } + ] + }, + { + "module": "module.eks.module.eks_managed_node_group[\"default\"]", + "mode": "managed", + "type": "aws_placement_group", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.eks.module.eks_managed_node_group[\"default\"].module.user_data", + "mode": "data", + "type": "cloudinit_config", + "name": "al2023_eks_managed_node_group", + "provider": "provider[\"registry.terraform.io/hashicorp/cloudinit\"]", + "instances": [] + }, + { + "module": "module.eks.module.eks_managed_node_group[\"default\"].module.user_data", + "mode": "data", + "type": "cloudinit_config", + "name": "linux_eks_managed_node_group", + "provider": "provider[\"registry.terraform.io/hashicorp/cloudinit\"]", + "instances": [] + }, + { + "module": "module.eks.module.eks_managed_node_group[\"default\"].module.user_data", + "mode": "managed", + "type": "null_resource", + "name": "validate_cluster_service_cidr", + "provider": "provider[\"registry.terraform.io/hashicorp/null\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "6383338662689565081", + "triggers": null + }, + "sensitive_attributes": [], + "dependencies": [ + "data.aws_availability_zones.available", + "module.eks.aws_cloudwatch_log_group.this", + "module.eks.aws_eks_cluster.this", + "module.eks.aws_iam_policy.cni_ipv6_policy", + "module.eks.aws_iam_role.eks_auto", + "module.eks.aws_iam_role.this", + "module.eks.aws_iam_role_policy_attachment.this", + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.eks.aws_security_group_rule.cluster", + "module.eks.aws_security_group_rule.node", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_policy_document.cni_ipv6_policy", + "module.eks.data.aws_iam_policy_document.node_assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.data.aws_partition.current", + "module.eks.module.kms.aws_kms_external_key.this", + "module.eks.module.kms.aws_kms_key.this", + "module.eks.module.kms.aws_kms_replica_external_key.this", + "module.eks.module.kms.aws_kms_replica_key.this", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current", + "module.eks.time_sleep.this", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.eks.module.kms", + "mode": "data", + "type": "aws_caller_identity", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "account_id": "027423573553", + "arn": "arn:aws:sts::027423573553:assumed-role/Admin/kylhouns-Isengard", + "id": "027423573553", + "user_id": "AROAQMYUSQYYXMMZMGMPD:kylhouns-Isengard" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.eks.module.kms", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "id": "1923360655", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"Default\",\n \"Effect\": \"Allow\",\n \"Action\": \"kms:*\",\n \"Resource\": \"*\",\n \"Principal\": {\n \"AWS\": \"arn:aws:iam::027423573553:root\"\n }\n },\n {\n \"Sid\": \"KeyAdministration\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"kms:Update*\",\n \"kms:UntagResource\",\n \"kms:TagResource\",\n \"kms:ScheduleKeyDeletion\",\n \"kms:Revoke*\",\n \"kms:ReplicateKey\",\n \"kms:Put*\",\n \"kms:List*\",\n \"kms:ImportKeyMaterial\",\n \"kms:Get*\",\n \"kms:Enable*\",\n \"kms:Disable*\",\n \"kms:Describe*\",\n \"kms:Delete*\",\n \"kms:Create*\",\n \"kms:CancelKeyDeletion\"\n ],\n \"Resource\": \"*\",\n \"Principal\": {\n \"AWS\": \"arn:aws:iam::027423573553:role/Admin\"\n }\n },\n {\n \"Sid\": \"KeyUsage\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"kms:ReEncrypt*\",\n \"kms:GenerateDataKey*\",\n \"kms:Encrypt\",\n \"kms:DescribeKey\",\n \"kms:Decrypt\"\n ],\n \"Resource\": \"*\",\n \"Principal\": {\n \"AWS\": \"arn:aws:iam::027423573553:role/observability-stack-cluster-20260318201827553600000003\"\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"Default\",\"Effect\":\"Allow\",\"Action\":\"kms:*\",\"Resource\":\"*\",\"Principal\":{\"AWS\":\"arn:aws:iam::027423573553:root\"}},{\"Sid\":\"KeyAdministration\",\"Effect\":\"Allow\",\"Action\":[\"kms:Update*\",\"kms:UntagResource\",\"kms:TagResource\",\"kms:ScheduleKeyDeletion\",\"kms:Revoke*\",\"kms:ReplicateKey\",\"kms:Put*\",\"kms:List*\",\"kms:ImportKeyMaterial\",\"kms:Get*\",\"kms:Enable*\",\"kms:Disable*\",\"kms:Describe*\",\"kms:Delete*\",\"kms:Create*\",\"kms:CancelKeyDeletion\"],\"Resource\":\"*\",\"Principal\":{\"AWS\":\"arn:aws:iam::027423573553:role/Admin\"}},{\"Sid\":\"KeyUsage\",\"Effect\":\"Allow\",\"Action\":[\"kms:ReEncrypt*\",\"kms:GenerateDataKey*\",\"kms:Encrypt\",\"kms:DescribeKey\",\"kms:Decrypt\"],\"Resource\":\"*\",\"Principal\":{\"AWS\":\"arn:aws:iam::027423573553:role/observability-stack-cluster-20260318201827553600000003\"}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "kms:*" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "arn:aws:iam::027423573553:root" + ], + "type": "AWS" + } + ], + "resources": [ + "*" + ], + "sid": "Default" + }, + { + "actions": [ + "kms:CancelKeyDeletion", + "kms:Create*", + "kms:Delete*", + "kms:Describe*", + "kms:Disable*", + "kms:Enable*", + "kms:Get*", + "kms:ImportKeyMaterial", + "kms:List*", + "kms:Put*", + "kms:ReplicateKey", + "kms:Revoke*", + "kms:ScheduleKeyDeletion", + "kms:TagResource", + "kms:UntagResource", + "kms:Update*" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "arn:aws:iam::027423573553:role/Admin" + ], + "type": "AWS" + } + ], + "resources": [ + "*" + ], + "sid": "KeyAdministration" + }, + { + "actions": [ + "kms:Decrypt", + "kms:DescribeKey", + "kms:Encrypt", + "kms:GenerateDataKey*", + "kms:ReEncrypt*" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "arn:aws:iam::027423573553:role/observability-stack-cluster-20260318201827553600000003" + ], + "type": "AWS" + } + ], + "resources": [ + "*" + ], + "sid": "KeyUsage" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.eks.module.kms", + "mode": "data", + "type": "aws_partition", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "dns_suffix": "amazonaws.com", + "id": "aws", + "partition": "aws", + "reverse_dns_prefix": "com.amazonaws" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.eks.module.kms", + "mode": "managed", + "type": "aws_kms_alias", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": "cluster", + "schema_version": 0, + "attributes": { + "arn": "arn:aws:kms:us-west-2:027423573553:alias/eks/observability-stack", + "id": "alias/eks/observability-stack", + "name": "alias/eks/observability-stack", + "name_prefix": "", + "target_key_arn": "arn:aws:kms:us-west-2:027423573553:key/12030f91-64b4-40e3-b7ad-77340e0391c7", + "target_key_id": "12030f91-64b4-40e3-b7ad-77340e0391c7" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.eks.aws_iam_role.this", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.module.kms.aws_kms_external_key.this", + "module.eks.module.kms.aws_kms_key.this", + "module.eks.module.kms.aws_kms_replica_external_key.this", + "module.eks.module.kms.aws_kms_replica_key.this", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current" + ] + } + ] + }, + { + "module": "module.eks.module.kms", + "mode": "managed", + "type": "aws_kms_external_key", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.eks.module.kms", + "mode": "managed", + "type": "aws_kms_grant", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.eks.module.kms", + "mode": "managed", + "type": "aws_kms_key", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:kms:us-west-2:027423573553:key/12030f91-64b4-40e3-b7ad-77340e0391c7", + "bypass_policy_lockout_safety_check": false, + "custom_key_store_id": "", + "customer_master_key_spec": "SYMMETRIC_DEFAULT", + "deletion_window_in_days": null, + "description": "observability-stack cluster encryption key", + "enable_key_rotation": true, + "id": "12030f91-64b4-40e3-b7ad-77340e0391c7", + "is_enabled": true, + "key_id": "12030f91-64b4-40e3-b7ad-77340e0391c7", + "key_usage": "ENCRYPT_DECRYPT", + "multi_region": false, + "policy": "{\"Statement\":[{\"Action\":\"kms:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::027423573553:root\"},\"Resource\":\"*\",\"Sid\":\"Default\"},{\"Action\":[\"kms:Update*\",\"kms:UntagResource\",\"kms:TagResource\",\"kms:ScheduleKeyDeletion\",\"kms:Revoke*\",\"kms:ReplicateKey\",\"kms:Put*\",\"kms:List*\",\"kms:ImportKeyMaterial\",\"kms:Get*\",\"kms:Enable*\",\"kms:Disable*\",\"kms:Describe*\",\"kms:Delete*\",\"kms:Create*\",\"kms:CancelKeyDeletion\"],\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::027423573553:role/Admin\"},\"Resource\":\"*\",\"Sid\":\"KeyAdministration\"},{\"Action\":[\"kms:ReEncrypt*\",\"kms:GenerateDataKey*\",\"kms:Encrypt\",\"kms:DescribeKey\",\"kms:Decrypt\"],\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::027423573553:role/observability-stack-cluster-20260318201827553600000003\"},\"Resource\":\"*\",\"Sid\":\"KeyUsage\"}],\"Version\":\"2012-10-17\"}", + "rotation_period_in_days": 365, + "tags": { + "terraform-aws-modules": "eks" + }, + "tags_all": { + "ManagedBy": "terraform", + "Project": "observability-stack", + "terraform-aws-modules": "eks" + }, + "timeouts": null, + "xks_key_id": "" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDB9fQ==", + "dependencies": [ + "module.eks.aws_iam_role.this", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current" + ], + "create_before_destroy": true + } + ] + }, + { + "module": "module.eks.module.kms", + "mode": "managed", + "type": "aws_kms_replica_external_key", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.eks.module.kms", + "mode": "managed", + "type": "aws_kms_replica_key", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "data", + "type": "aws_caller_identity", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "account_id": "027423573553", + "arn": "arn:aws:sts::027423573553:assumed-role/Admin/kylhouns-Isengard", + "id": "027423573553", + "user_id": "AROAQMYUSQYYXMMZMGMPD:kylhouns-Isengard" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.external_dns_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "amazon_managed_service_prometheus", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "appmesh_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "appmesh_envoy_proxy", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "aws_gateway_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cert_manager", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cluster_autoscaler", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "ebs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "efs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "external_dns", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "id": "3951040301", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"route53:ChangeResourceRecordSets\",\n \"Resource\": \"arn:aws:route53:::hostedzone/Z07457962NTDHS0A6G9LH\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"route53:ListTagsForResources\",\n \"route53:ListResourceRecordSets\",\n \"route53:ListHostedZones\"\n ],\n \"Resource\": \"*\"\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"route53:ChangeResourceRecordSets\",\"Resource\":\"arn:aws:route53:::hostedzone/Z07457962NTDHS0A6G9LH\"},{\"Effect\":\"Allow\",\"Action\":[\"route53:ListTagsForResources\",\"route53:ListResourceRecordSets\",\"route53:ListHostedZones\"],\"Resource\":\"*\"}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "route53:ChangeResourceRecordSets" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:route53:::hostedzone/Z07457962NTDHS0A6G9LH" + ], + "sid": "" + }, + { + "actions": [ + "route53:ListHostedZones", + "route53:ListResourceRecordSets", + "route53:ListTagsForResources" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.external_dns_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "external_secrets", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "fsx_lustre_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "fsx_openzfs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "karpenter_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "load_balancer_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "load_balancer_controller_targetgroup_only", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "mountpoint_s3_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "node_termination_handler", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "id": "3909713710", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRoleWithWebIdentity\",\n \"Principal\": {\n \"Federated\": \"arn:aws:iam::027423573553:oidc-provider/oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D\"\n },\n \"Condition\": {\n \"StringEquals\": {\n \"oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D:aud\": \"sts.amazonaws.com\",\n \"oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D:sub\": \"system:serviceaccount:kube-system:external-dns\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRoleWithWebIdentity\",\"Principal\":{\"Federated\":\"arn:aws:iam::027423573553:oidc-provider/oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D\"},\"Condition\":{\"StringEquals\":{\"oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D:aud\":\"sts.amazonaws.com\",\"oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D:sub\":\"system:serviceaccount:kube-system:external-dns\"}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRoleWithWebIdentity" + ], + "condition": [ + { + "test": "StringEquals", + "values": [ + "sts.amazonaws.com" + ], + "variable": "oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D:aud" + }, + { + "test": "StringEquals", + "values": [ + "system:serviceaccount:kube-system:external-dns" + ], + "variable": "oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D:sub" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "arn:aws:iam::027423573553:oidc-provider/oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D" + ], + "type": "Federated" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.external_dns_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "velero", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "vpc_cni", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "data", + "type": "aws_partition", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "dns_suffix": "amazonaws.com", + "id": "aws", + "partition": "aws", + "reverse_dns_prefix": "com.amazonaws" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.external_dns_irsa", + "mode": "data", + "type": "aws_region", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "description": "US West (Oregon)", + "endpoint": "ec2.us-west-2.amazonaws.com", + "id": "us-west-2", + "name": "us-west-2" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "amazon_managed_service_prometheus", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "appmesh_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "appmesh_envoy_proxy", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "aws_gateway_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "cert_manager", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "cluster_autoscaler", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "ebs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "efs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "external_dns", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::027423573553:policy/AmazonEKS_External_DNS_Policy-20260318201827552700000001", + "attachment_count": 1, + "description": "External DNS policy to allow management of Route53 hosted zone records", + "id": "arn:aws:iam::027423573553:policy/AmazonEKS_External_DNS_Policy-20260318201827552700000001", + "name": "AmazonEKS_External_DNS_Policy-20260318201827552700000001", + "name_prefix": "AmazonEKS_External_DNS_Policy-", + "path": "/", + "policy": "{\"Statement\":[{\"Action\":\"route53:ChangeResourceRecordSets\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:route53:::hostedzone/Z07457962NTDHS0A6G9LH\"},{\"Action\":[\"route53:ListTagsForResources\",\"route53:ListResourceRecordSets\",\"route53:ListHostedZones\"],\"Effect\":\"Allow\",\"Resource\":\"*\"}],\"Version\":\"2012-10-17\"}", + "policy_id": "ANPAQMYUSQYYZYFDNRBWO", + "tags": {}, + "tags_all": { + "ManagedBy": "terraform", + "Project": "observability-stack" + } + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.external_dns_irsa.data.aws_iam_policy_document.external_dns" + ] + } + ] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "external_secrets", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "fsx_lustre_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "fsx_openzfs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "karpenter_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "load_balancer_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "load_balancer_controller_targetgroup_only", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "mountpoint_s3_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "node_termination_handler", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "velero", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "vpc_cni", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_role", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::027423573553:role/observability-stack-external-dns", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRoleWithWebIdentity\",\"Condition\":{\"StringEquals\":{\"oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D:aud\":\"sts.amazonaws.com\",\"oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D:sub\":\"system:serviceaccount:kube-system:external-dns\"}},\"Effect\":\"Allow\",\"Principal\":{\"Federated\":\"arn:aws:iam::027423573553:oidc-provider/oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-03-18T20:28:14Z", + "description": "", + "force_detach_policies": true, + "id": "observability-stack-external-dns", + "inline_policy": [], + "managed_policy_arns": [ + "arn:aws:iam::027423573553:policy/AmazonEKS_External_DNS_Policy-20260318201827552700000001" + ], + "max_session_duration": 3600, + "name": "observability-stack-external-dns", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": {}, + "tags_all": { + "ManagedBy": "terraform", + "Project": "observability-stack" + }, + "unique_id": "AROAQMYUSQYY43FP3UD6E" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_availability_zones.available", + "module.eks.aws_cloudwatch_log_group.this", + "module.eks.aws_eks_cluster.this", + "module.eks.aws_iam_openid_connect_provider.oidc_provider", + "module.eks.aws_iam_policy.cni_ipv6_policy", + "module.eks.aws_iam_role.eks_auto", + "module.eks.aws_iam_role.this", + "module.eks.aws_iam_role_policy_attachment.this", + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.eks.aws_security_group_rule.cluster", + "module.eks.aws_security_group_rule.node", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_policy_document.cni_ipv6_policy", + "module.eks.data.aws_iam_policy_document.node_assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.data.aws_partition.current", + "module.eks.data.tls_certificate.this", + "module.eks.module.kms.aws_kms_external_key.this", + "module.eks.module.kms.aws_kms_key.this", + "module.eks.module.kms.aws_kms_replica_external_key.this", + "module.eks.module.kms.aws_kms_replica_key.this", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current", + "module.external_dns_irsa.data.aws_caller_identity.current", + "module.external_dns_irsa.data.aws_iam_policy_document.this", + "module.external_dns_irsa.data.aws_partition.current", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "amazon_cloudwatch_observability", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "amazon_managed_service_prometheus", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "appmesh_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "appmesh_envoy_proxy", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "aws_gateway_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "cert_manager", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "cluster_autoscaler", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "ebs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "efs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "external_dns", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "id": "observability-stack-external-dns-20260318202815339800000014", + "policy_arn": "arn:aws:iam::027423573553:policy/AmazonEKS_External_DNS_Policy-20260318201827552700000001", + "role": "observability-stack-external-dns" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_availability_zones.available", + "module.eks.aws_cloudwatch_log_group.this", + "module.eks.aws_eks_cluster.this", + "module.eks.aws_iam_openid_connect_provider.oidc_provider", + "module.eks.aws_iam_policy.cni_ipv6_policy", + "module.eks.aws_iam_role.eks_auto", + "module.eks.aws_iam_role.this", + "module.eks.aws_iam_role_policy_attachment.this", + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.eks.aws_security_group_rule.cluster", + "module.eks.aws_security_group_rule.node", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_policy_document.cni_ipv6_policy", + "module.eks.data.aws_iam_policy_document.node_assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.data.aws_partition.current", + "module.eks.data.tls_certificate.this", + "module.eks.module.kms.aws_kms_external_key.this", + "module.eks.module.kms.aws_kms_key.this", + "module.eks.module.kms.aws_kms_replica_external_key.this", + "module.eks.module.kms.aws_kms_replica_key.this", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current", + "module.external_dns_irsa.aws_iam_policy.external_dns", + "module.external_dns_irsa.aws_iam_role.this", + "module.external_dns_irsa.data.aws_caller_identity.current", + "module.external_dns_irsa.data.aws_iam_policy_document.external_dns", + "module.external_dns_irsa.data.aws_iam_policy_document.this", + "module.external_dns_irsa.data.aws_partition.current", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "external_secrets", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "fsx_lustre_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "fsx_openzfs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "karpenter_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "load_balancer_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "load_balancer_controller_targetgroup_only", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "mountpoint_s3_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "node_termination_handler", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "velero", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.external_dns_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "vpc_cni", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "data", + "type": "aws_caller_identity", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "account_id": "027423573553", + "arn": "arn:aws:sts::027423573553:assumed-role/Admin/kylhouns-Isengard", + "id": "027423573553", + "user_id": "AROAQMYUSQYYXMMZMGMPD:kylhouns-Isengard" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.lb_controller_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "amazon_managed_service_prometheus", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "appmesh_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "appmesh_envoy_proxy", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "aws_gateway_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cert_manager", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cluster_autoscaler", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "ebs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "efs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "external_dns", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "external_secrets", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "fsx_lustre_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "fsx_openzfs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "karpenter_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "load_balancer_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "id": "1541424006", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"iam:CreateServiceLinkedRole\",\n \"Resource\": \"*\",\n \"Condition\": {\n \"StringEquals\": {\n \"iam:AWSServiceName\": \"elasticloadbalancing.amazonaws.com\"\n }\n }\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"elasticloadbalancing:DescribeTrustStores\",\n \"elasticloadbalancing:DescribeTargetHealth\",\n \"elasticloadbalancing:DescribeTargetGroups\",\n \"elasticloadbalancing:DescribeTargetGroupAttributes\",\n \"elasticloadbalancing:DescribeTags\",\n \"elasticloadbalancing:DescribeSSLPolicies\",\n \"elasticloadbalancing:DescribeRules\",\n \"elasticloadbalancing:DescribeLoadBalancers\",\n \"elasticloadbalancing:DescribeLoadBalancerAttributes\",\n \"elasticloadbalancing:DescribeListeners\",\n \"elasticloadbalancing:DescribeListenerCertificates\",\n \"elasticloadbalancing:DescribeListenerAttributes\",\n \"elasticloadbalancing:DescribeCapacityReservation\",\n \"ec2:GetSecurityGroupsForVpc\",\n \"ec2:GetCoipPoolUsage\",\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeVpcPeeringConnections\",\n \"ec2:DescribeTags\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeRouteTables\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeIpamPools\",\n \"ec2:DescribeInternetGateways\",\n \"ec2:DescribeInstances\",\n \"ec2:DescribeCoipPools\",\n \"ec2:DescribeAvailabilityZones\",\n \"ec2:DescribeAddresses\",\n \"ec2:DescribeAccountAttributes\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"wafv2:GetWebACLForResource\",\n \"wafv2:GetWebACL\",\n \"wafv2:DisassociateWebACL\",\n \"wafv2:AssociateWebACL\",\n \"waf-regional:GetWebACLForResource\",\n \"waf-regional:GetWebACL\",\n \"waf-regional:DisassociateWebACL\",\n \"waf-regional:AssociateWebACL\",\n \"shield:GetSubscriptionState\",\n \"shield:DescribeProtection\",\n \"shield:DeleteProtection\",\n \"shield:CreateProtection\",\n \"iam:ListServerCertificates\",\n \"iam:GetServerCertificate\",\n \"cognito-idp:DescribeUserPoolClient\",\n \"acm:ListCertificates\",\n \"acm:DescribeCertificate\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:RevokeSecurityGroupIngress\",\n \"ec2:AuthorizeSecurityGroupIngress\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateSecurityGroup\",\n \"Resource\": \"*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateTags\",\n \"Resource\": \"arn:aws:ec2:*:*:security-group/*\",\n \"Condition\": {\n \"Null\": {\n \"aws:RequestTag/elbv2.k8s.aws/cluster\": \"false\"\n },\n \"StringEquals\": {\n \"ec2:CreateAction\": \"CreateSecurityGroup\"\n }\n }\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DeleteTags\",\n \"ec2:CreateTags\"\n ],\n \"Resource\": \"arn:aws:ec2:*:*:security-group/*\",\n \"Condition\": {\n \"Null\": {\n \"aws:RequestTag/elbv2.k8s.aws/cluster\": \"true\",\n \"aws:ResourceTag/elbv2.k8s.aws/cluster\": \"false\"\n }\n }\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:RevokeSecurityGroupIngress\",\n \"ec2:DeleteSecurityGroup\",\n \"ec2:AuthorizeSecurityGroupIngress\"\n ],\n \"Resource\": \"*\",\n \"Condition\": {\n \"Null\": {\n \"aws:ResourceTag/elbv2.k8s.aws/cluster\": \"false\"\n }\n }\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"elasticloadbalancing:CreateTargetGroup\",\n \"elasticloadbalancing:CreateLoadBalancer\"\n ],\n \"Resource\": \"*\",\n \"Condition\": {\n \"Null\": {\n \"aws:RequestTag/elbv2.k8s.aws/cluster\": \"false\"\n }\n }\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"elasticloadbalancing:DeleteRule\",\n \"elasticloadbalancing:DeleteListener\",\n \"elasticloadbalancing:CreateRule\",\n \"elasticloadbalancing:CreateListener\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"elasticloadbalancing:RemoveTags\",\n \"elasticloadbalancing:AddTags\"\n ],\n \"Resource\": [\n \"arn:aws:elasticloadbalancing:*:*:targetgroup/*/*\",\n \"arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*\",\n \"arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*\"\n ],\n \"Condition\": {\n \"Null\": {\n \"aws:RequestTag/elbv2.k8s.aws/cluster\": \"true\",\n \"aws:ResourceTag/elbv2.k8s.aws/cluster\": \"false\"\n }\n }\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"elasticloadbalancing:RemoveTags\",\n \"elasticloadbalancing:AddTags\"\n ],\n \"Resource\": [\n \"arn:aws:elasticloadbalancing:*:*:listener/net/*/*/*\",\n \"arn:aws:elasticloadbalancing:*:*:listener/app/*/*/*\",\n \"arn:aws:elasticloadbalancing:*:*:listener-rule/net/*/*/*\",\n \"arn:aws:elasticloadbalancing:*:*:listener-rule/app/*/*/*\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"elasticloadbalancing:SetSubnets\",\n \"elasticloadbalancing:SetSecurityGroups\",\n \"elasticloadbalancing:SetIpAddressType\",\n \"elasticloadbalancing:ModifyTargetGroupAttributes\",\n \"elasticloadbalancing:ModifyTargetGroup\",\n \"elasticloadbalancing:ModifyLoadBalancerAttributes\",\n \"elasticloadbalancing:ModifyListenerAttributes\",\n \"elasticloadbalancing:ModifyIpPools\",\n \"elasticloadbalancing:ModifyCapacityReservation\",\n \"elasticloadbalancing:DeleteTargetGroup\",\n \"elasticloadbalancing:DeleteLoadBalancer\"\n ],\n \"Resource\": \"*\",\n \"Condition\": {\n \"Null\": {\n \"aws:ResourceTag/elbv2.k8s.aws/cluster\": \"false\"\n }\n }\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"elasticloadbalancing:AddTags\",\n \"Resource\": [\n \"arn:aws:elasticloadbalancing:*:*:targetgroup/*/*\",\n \"arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*\",\n \"arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*\"\n ],\n \"Condition\": {\n \"Null\": {\n \"aws:RequestTag/elbv2.k8s.aws/cluster\": \"false\"\n },\n \"StringEquals\": {\n \"elasticloadbalancing:CreateAction\": [\n \"CreateTargetGroup\",\n \"CreateLoadBalancer\"\n ]\n }\n }\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"elasticloadbalancing:RegisterTargets\",\n \"elasticloadbalancing:DeregisterTargets\"\n ],\n \"Resource\": \"arn:aws:elasticloadbalancing:*:*:targetgroup/*/*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"elasticloadbalancing:SetWebAcl\",\n \"elasticloadbalancing:SetRulePriorities\",\n \"elasticloadbalancing:RemoveListenerCertificates\",\n \"elasticloadbalancing:ModifyRule\",\n \"elasticloadbalancing:ModifyListener\",\n \"elasticloadbalancing:AddListenerCertificates\"\n ],\n \"Resource\": \"*\"\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"iam:CreateServiceLinkedRole\",\"Resource\":\"*\",\"Condition\":{\"StringEquals\":{\"iam:AWSServiceName\":\"elasticloadbalancing.amazonaws.com\"}}},{\"Effect\":\"Allow\",\"Action\":[\"elasticloadbalancing:DescribeTrustStores\",\"elasticloadbalancing:DescribeTargetHealth\",\"elasticloadbalancing:DescribeTargetGroups\",\"elasticloadbalancing:DescribeTargetGroupAttributes\",\"elasticloadbalancing:DescribeTags\",\"elasticloadbalancing:DescribeSSLPolicies\",\"elasticloadbalancing:DescribeRules\",\"elasticloadbalancing:DescribeLoadBalancers\",\"elasticloadbalancing:DescribeLoadBalancerAttributes\",\"elasticloadbalancing:DescribeListeners\",\"elasticloadbalancing:DescribeListenerCertificates\",\"elasticloadbalancing:DescribeListenerAttributes\",\"elasticloadbalancing:DescribeCapacityReservation\",\"ec2:GetSecurityGroupsForVpc\",\"ec2:GetCoipPoolUsage\",\"ec2:DescribeVpcs\",\"ec2:DescribeVpcPeeringConnections\",\"ec2:DescribeTags\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeIpamPools\",\"ec2:DescribeInternetGateways\",\"ec2:DescribeInstances\",\"ec2:DescribeCoipPools\",\"ec2:DescribeAvailabilityZones\",\"ec2:DescribeAddresses\",\"ec2:DescribeAccountAttributes\"],\"Resource\":\"*\"},{\"Effect\":\"Allow\",\"Action\":[\"wafv2:GetWebACLForResource\",\"wafv2:GetWebACL\",\"wafv2:DisassociateWebACL\",\"wafv2:AssociateWebACL\",\"waf-regional:GetWebACLForResource\",\"waf-regional:GetWebACL\",\"waf-regional:DisassociateWebACL\",\"waf-regional:AssociateWebACL\",\"shield:GetSubscriptionState\",\"shield:DescribeProtection\",\"shield:DeleteProtection\",\"shield:CreateProtection\",\"iam:ListServerCertificates\",\"iam:GetServerCertificate\",\"cognito-idp:DescribeUserPoolClient\",\"acm:ListCertificates\",\"acm:DescribeCertificate\"],\"Resource\":\"*\"},{\"Effect\":\"Allow\",\"Action\":[\"ec2:RevokeSecurityGroupIngress\",\"ec2:AuthorizeSecurityGroupIngress\"],\"Resource\":\"*\"},{\"Effect\":\"Allow\",\"Action\":\"ec2:CreateSecurityGroup\",\"Resource\":\"*\"},{\"Effect\":\"Allow\",\"Action\":\"ec2:CreateTags\",\"Resource\":\"arn:aws:ec2:*:*:security-group/*\",\"Condition\":{\"Null\":{\"aws:RequestTag/elbv2.k8s.aws/cluster\":\"false\"},\"StringEquals\":{\"ec2:CreateAction\":\"CreateSecurityGroup\"}}},{\"Effect\":\"Allow\",\"Action\":[\"ec2:DeleteTags\",\"ec2:CreateTags\"],\"Resource\":\"arn:aws:ec2:*:*:security-group/*\",\"Condition\":{\"Null\":{\"aws:RequestTag/elbv2.k8s.aws/cluster\":\"true\",\"aws:ResourceTag/elbv2.k8s.aws/cluster\":\"false\"}}},{\"Effect\":\"Allow\",\"Action\":[\"ec2:RevokeSecurityGroupIngress\",\"ec2:DeleteSecurityGroup\",\"ec2:AuthorizeSecurityGroupIngress\"],\"Resource\":\"*\",\"Condition\":{\"Null\":{\"aws:ResourceTag/elbv2.k8s.aws/cluster\":\"false\"}}},{\"Effect\":\"Allow\",\"Action\":[\"elasticloadbalancing:CreateTargetGroup\",\"elasticloadbalancing:CreateLoadBalancer\"],\"Resource\":\"*\",\"Condition\":{\"Null\":{\"aws:RequestTag/elbv2.k8s.aws/cluster\":\"false\"}}},{\"Effect\":\"Allow\",\"Action\":[\"elasticloadbalancing:DeleteRule\",\"elasticloadbalancing:DeleteListener\",\"elasticloadbalancing:CreateRule\",\"elasticloadbalancing:CreateListener\"],\"Resource\":\"*\"},{\"Effect\":\"Allow\",\"Action\":[\"elasticloadbalancing:RemoveTags\",\"elasticloadbalancing:AddTags\"],\"Resource\":[\"arn:aws:elasticloadbalancing:*:*:targetgroup/*/*\",\"arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*\",\"arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*\"],\"Condition\":{\"Null\":{\"aws:RequestTag/elbv2.k8s.aws/cluster\":\"true\",\"aws:ResourceTag/elbv2.k8s.aws/cluster\":\"false\"}}},{\"Effect\":\"Allow\",\"Action\":[\"elasticloadbalancing:RemoveTags\",\"elasticloadbalancing:AddTags\"],\"Resource\":[\"arn:aws:elasticloadbalancing:*:*:listener/net/*/*/*\",\"arn:aws:elasticloadbalancing:*:*:listener/app/*/*/*\",\"arn:aws:elasticloadbalancing:*:*:listener-rule/net/*/*/*\",\"arn:aws:elasticloadbalancing:*:*:listener-rule/app/*/*/*\"]},{\"Effect\":\"Allow\",\"Action\":[\"elasticloadbalancing:SetSubnets\",\"elasticloadbalancing:SetSecurityGroups\",\"elasticloadbalancing:SetIpAddressType\",\"elasticloadbalancing:ModifyTargetGroupAttributes\",\"elasticloadbalancing:ModifyTargetGroup\",\"elasticloadbalancing:ModifyLoadBalancerAttributes\",\"elasticloadbalancing:ModifyListenerAttributes\",\"elasticloadbalancing:ModifyIpPools\",\"elasticloadbalancing:ModifyCapacityReservation\",\"elasticloadbalancing:DeleteTargetGroup\",\"elasticloadbalancing:DeleteLoadBalancer\"],\"Resource\":\"*\",\"Condition\":{\"Null\":{\"aws:ResourceTag/elbv2.k8s.aws/cluster\":\"false\"}}},{\"Effect\":\"Allow\",\"Action\":\"elasticloadbalancing:AddTags\",\"Resource\":[\"arn:aws:elasticloadbalancing:*:*:targetgroup/*/*\",\"arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*\",\"arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*\"],\"Condition\":{\"Null\":{\"aws:RequestTag/elbv2.k8s.aws/cluster\":\"false\"},\"StringEquals\":{\"elasticloadbalancing:CreateAction\":[\"CreateTargetGroup\",\"CreateLoadBalancer\"]}}},{\"Effect\":\"Allow\",\"Action\":[\"elasticloadbalancing:RegisterTargets\",\"elasticloadbalancing:DeregisterTargets\"],\"Resource\":\"arn:aws:elasticloadbalancing:*:*:targetgroup/*/*\"},{\"Effect\":\"Allow\",\"Action\":[\"elasticloadbalancing:SetWebAcl\",\"elasticloadbalancing:SetRulePriorities\",\"elasticloadbalancing:RemoveListenerCertificates\",\"elasticloadbalancing:ModifyRule\",\"elasticloadbalancing:ModifyListener\",\"elasticloadbalancing:AddListenerCertificates\"],\"Resource\":\"*\"}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "iam:CreateServiceLinkedRole" + ], + "condition": [ + { + "test": "StringEquals", + "values": [ + "elasticloadbalancing.amazonaws.com" + ], + "variable": "iam:AWSServiceName" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "" + }, + { + "actions": [ + "ec2:DescribeAccountAttributes", + "ec2:DescribeAddresses", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeCoipPools", + "ec2:DescribeInstances", + "ec2:DescribeInternetGateways", + "ec2:DescribeIpamPools", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeTags", + "ec2:DescribeVpcPeeringConnections", + "ec2:DescribeVpcs", + "ec2:GetCoipPoolUsage", + "ec2:GetSecurityGroupsForVpc", + "elasticloadbalancing:DescribeCapacityReservation", + "elasticloadbalancing:DescribeListenerAttributes", + "elasticloadbalancing:DescribeListenerCertificates", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeRules", + "elasticloadbalancing:DescribeSSLPolicies", + "elasticloadbalancing:DescribeTags", + "elasticloadbalancing:DescribeTargetGroupAttributes", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:DescribeTrustStores" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "" + }, + { + "actions": [ + "acm:DescribeCertificate", + "acm:ListCertificates", + "cognito-idp:DescribeUserPoolClient", + "iam:GetServerCertificate", + "iam:ListServerCertificates", + "shield:CreateProtection", + "shield:DeleteProtection", + "shield:DescribeProtection", + "shield:GetSubscriptionState", + "waf-regional:AssociateWebACL", + "waf-regional:DisassociateWebACL", + "waf-regional:GetWebACL", + "waf-regional:GetWebACLForResource", + "wafv2:AssociateWebACL", + "wafv2:DisassociateWebACL", + "wafv2:GetWebACL", + "wafv2:GetWebACLForResource" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "" + }, + { + "actions": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:RevokeSecurityGroupIngress" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "" + }, + { + "actions": [ + "ec2:CreateSecurityGroup" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "" + }, + { + "actions": [ + "ec2:CreateTags" + ], + "condition": [ + { + "test": "Null", + "values": [ + "false" + ], + "variable": "aws:RequestTag/elbv2.k8s.aws/cluster" + }, + { + "test": "StringEquals", + "values": [ + "CreateSecurityGroup" + ], + "variable": "ec2:CreateAction" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:*:*:security-group/*" + ], + "sid": "" + }, + { + "actions": [ + "ec2:CreateTags", + "ec2:DeleteTags" + ], + "condition": [ + { + "test": "Null", + "values": [ + "false" + ], + "variable": "aws:ResourceTag/elbv2.k8s.aws/cluster" + }, + { + "test": "Null", + "values": [ + "true" + ], + "variable": "aws:RequestTag/elbv2.k8s.aws/cluster" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:*:*:security-group/*" + ], + "sid": "" + }, + { + "actions": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:DeleteSecurityGroup", + "ec2:RevokeSecurityGroupIngress" + ], + "condition": [ + { + "test": "Null", + "values": [ + "false" + ], + "variable": "aws:ResourceTag/elbv2.k8s.aws/cluster" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "" + }, + { + "actions": [ + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateTargetGroup" + ], + "condition": [ + { + "test": "Null", + "values": [ + "false" + ], + "variable": "aws:RequestTag/elbv2.k8s.aws/cluster" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "" + }, + { + "actions": [ + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateRule", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteRule" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "" + }, + { + "actions": [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:RemoveTags" + ], + "condition": [ + { + "test": "Null", + "values": [ + "false" + ], + "variable": "aws:ResourceTag/elbv2.k8s.aws/cluster" + }, + { + "test": "Null", + "values": [ + "true" + ], + "variable": "aws:RequestTag/elbv2.k8s.aws/cluster" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*", + "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*" + ], + "sid": "" + }, + { + "actions": [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:RemoveTags" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:elasticloadbalancing:*:*:listener-rule/app/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener-rule/net/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener/app/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener/net/*/*/*" + ], + "sid": "" + }, + { + "actions": [ + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:ModifyCapacityReservation", + "elasticloadbalancing:ModifyIpPools", + "elasticloadbalancing:ModifyListenerAttributes", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:ModifyTargetGroupAttributes", + "elasticloadbalancing:SetIpAddressType", + "elasticloadbalancing:SetSecurityGroups", + "elasticloadbalancing:SetSubnets" + ], + "condition": [ + { + "test": "Null", + "values": [ + "false" + ], + "variable": "aws:ResourceTag/elbv2.k8s.aws/cluster" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "" + }, + { + "actions": [ + "elasticloadbalancing:AddTags" + ], + "condition": [ + { + "test": "Null", + "values": [ + "false" + ], + "variable": "aws:RequestTag/elbv2.k8s.aws/cluster" + }, + { + "test": "StringEquals", + "values": [ + "CreateTargetGroup", + "CreateLoadBalancer" + ], + "variable": "elasticloadbalancing:CreateAction" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*", + "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*" + ], + "sid": "" + }, + { + "actions": [ + "elasticloadbalancing:DeregisterTargets", + "elasticloadbalancing:RegisterTargets" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*" + ], + "sid": "" + }, + { + "actions": [ + "elasticloadbalancing:AddListenerCertificates", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyRule", + "elasticloadbalancing:RemoveListenerCertificates", + "elasticloadbalancing:SetRulePriorities", + "elasticloadbalancing:SetWebAcl" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.lb_controller_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "load_balancer_controller_targetgroup_only", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "mountpoint_s3_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "node_termination_handler", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "id": "1290851146", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRoleWithWebIdentity\",\n \"Principal\": {\n \"Federated\": \"arn:aws:iam::027423573553:oidc-provider/oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D\"\n },\n \"Condition\": {\n \"StringEquals\": {\n \"oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D:aud\": \"sts.amazonaws.com\",\n \"oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D:sub\": \"system:serviceaccount:kube-system:aws-load-balancer-controller\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRoleWithWebIdentity\",\"Principal\":{\"Federated\":\"arn:aws:iam::027423573553:oidc-provider/oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D\"},\"Condition\":{\"StringEquals\":{\"oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D:aud\":\"sts.amazonaws.com\",\"oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D:sub\":\"system:serviceaccount:kube-system:aws-load-balancer-controller\"}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRoleWithWebIdentity" + ], + "condition": [ + { + "test": "StringEquals", + "values": [ + "sts.amazonaws.com" + ], + "variable": "oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D:aud" + }, + { + "test": "StringEquals", + "values": [ + "system:serviceaccount:kube-system:aws-load-balancer-controller" + ], + "variable": "oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D:sub" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "arn:aws:iam::027423573553:oidc-provider/oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D" + ], + "type": "Federated" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.lb_controller_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "velero", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "vpc_cni", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "data", + "type": "aws_partition", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "dns_suffix": "amazonaws.com", + "id": "aws", + "partition": "aws", + "reverse_dns_prefix": "com.amazonaws" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.lb_controller_irsa", + "mode": "data", + "type": "aws_region", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "description": "US West (Oregon)", + "endpoint": "ec2.us-west-2.amazonaws.com", + "id": "us-west-2", + "name": "us-west-2" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "amazon_managed_service_prometheus", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "appmesh_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "appmesh_envoy_proxy", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "aws_gateway_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "cert_manager", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "cluster_autoscaler", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "ebs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "efs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "external_dns", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "external_secrets", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "fsx_lustre_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "fsx_openzfs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "karpenter_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "load_balancer_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::027423573553:policy/AmazonEKS_AWS_Load_Balancer_Controller-20260318201827555800000006", + "attachment_count": 1, + "description": "Provides permissions for AWS Load Balancer Controller addon", + "id": "arn:aws:iam::027423573553:policy/AmazonEKS_AWS_Load_Balancer_Controller-20260318201827555800000006", + "name": "AmazonEKS_AWS_Load_Balancer_Controller-20260318201827555800000006", + "name_prefix": "AmazonEKS_AWS_Load_Balancer_Controller-", + "path": "/", + "policy": "{\"Statement\":[{\"Action\":\"iam:CreateServiceLinkedRole\",\"Condition\":{\"StringEquals\":{\"iam:AWSServiceName\":\"elasticloadbalancing.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"*\"},{\"Action\":[\"elasticloadbalancing:DescribeTrustStores\",\"elasticloadbalancing:DescribeTargetHealth\",\"elasticloadbalancing:DescribeTargetGroups\",\"elasticloadbalancing:DescribeTargetGroupAttributes\",\"elasticloadbalancing:DescribeTags\",\"elasticloadbalancing:DescribeSSLPolicies\",\"elasticloadbalancing:DescribeRules\",\"elasticloadbalancing:DescribeLoadBalancers\",\"elasticloadbalancing:DescribeLoadBalancerAttributes\",\"elasticloadbalancing:DescribeListeners\",\"elasticloadbalancing:DescribeListenerCertificates\",\"elasticloadbalancing:DescribeListenerAttributes\",\"elasticloadbalancing:DescribeCapacityReservation\",\"ec2:GetSecurityGroupsForVpc\",\"ec2:GetCoipPoolUsage\",\"ec2:DescribeVpcs\",\"ec2:DescribeVpcPeeringConnections\",\"ec2:DescribeTags\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeIpamPools\",\"ec2:DescribeInternetGateways\",\"ec2:DescribeInstances\",\"ec2:DescribeCoipPools\",\"ec2:DescribeAvailabilityZones\",\"ec2:DescribeAddresses\",\"ec2:DescribeAccountAttributes\"],\"Effect\":\"Allow\",\"Resource\":\"*\"},{\"Action\":[\"wafv2:GetWebACLForResource\",\"wafv2:GetWebACL\",\"wafv2:DisassociateWebACL\",\"wafv2:AssociateWebACL\",\"waf-regional:GetWebACLForResource\",\"waf-regional:GetWebACL\",\"waf-regional:DisassociateWebACL\",\"waf-regional:AssociateWebACL\",\"shield:GetSubscriptionState\",\"shield:DescribeProtection\",\"shield:DeleteProtection\",\"shield:CreateProtection\",\"iam:ListServerCertificates\",\"iam:GetServerCertificate\",\"cognito-idp:DescribeUserPoolClient\",\"acm:ListCertificates\",\"acm:DescribeCertificate\"],\"Effect\":\"Allow\",\"Resource\":\"*\"},{\"Action\":[\"ec2:RevokeSecurityGroupIngress\",\"ec2:AuthorizeSecurityGroupIngress\"],\"Effect\":\"Allow\",\"Resource\":\"*\"},{\"Action\":\"ec2:CreateSecurityGroup\",\"Effect\":\"Allow\",\"Resource\":\"*\"},{\"Action\":\"ec2:CreateTags\",\"Condition\":{\"Null\":{\"aws:RequestTag/elbv2.k8s.aws/cluster\":\"false\"},\"StringEquals\":{\"ec2:CreateAction\":\"CreateSecurityGroup\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:*:*:security-group/*\"},{\"Action\":[\"ec2:DeleteTags\",\"ec2:CreateTags\"],\"Condition\":{\"Null\":{\"aws:RequestTag/elbv2.k8s.aws/cluster\":\"true\",\"aws:ResourceTag/elbv2.k8s.aws/cluster\":\"false\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:*:*:security-group/*\"},{\"Action\":[\"ec2:RevokeSecurityGroupIngress\",\"ec2:DeleteSecurityGroup\",\"ec2:AuthorizeSecurityGroupIngress\"],\"Condition\":{\"Null\":{\"aws:ResourceTag/elbv2.k8s.aws/cluster\":\"false\"}},\"Effect\":\"Allow\",\"Resource\":\"*\"},{\"Action\":[\"elasticloadbalancing:CreateTargetGroup\",\"elasticloadbalancing:CreateLoadBalancer\"],\"Condition\":{\"Null\":{\"aws:RequestTag/elbv2.k8s.aws/cluster\":\"false\"}},\"Effect\":\"Allow\",\"Resource\":\"*\"},{\"Action\":[\"elasticloadbalancing:DeleteRule\",\"elasticloadbalancing:DeleteListener\",\"elasticloadbalancing:CreateRule\",\"elasticloadbalancing:CreateListener\"],\"Effect\":\"Allow\",\"Resource\":\"*\"},{\"Action\":[\"elasticloadbalancing:RemoveTags\",\"elasticloadbalancing:AddTags\"],\"Condition\":{\"Null\":{\"aws:RequestTag/elbv2.k8s.aws/cluster\":\"true\",\"aws:ResourceTag/elbv2.k8s.aws/cluster\":\"false\"}},\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:elasticloadbalancing:*:*:targetgroup/*/*\",\"arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*\",\"arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*\"]},{\"Action\":[\"elasticloadbalancing:RemoveTags\",\"elasticloadbalancing:AddTags\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:elasticloadbalancing:*:*:listener/net/*/*/*\",\"arn:aws:elasticloadbalancing:*:*:listener/app/*/*/*\",\"arn:aws:elasticloadbalancing:*:*:listener-rule/net/*/*/*\",\"arn:aws:elasticloadbalancing:*:*:listener-rule/app/*/*/*\"]},{\"Action\":[\"elasticloadbalancing:SetSubnets\",\"elasticloadbalancing:SetSecurityGroups\",\"elasticloadbalancing:SetIpAddressType\",\"elasticloadbalancing:ModifyTargetGroupAttributes\",\"elasticloadbalancing:ModifyTargetGroup\",\"elasticloadbalancing:ModifyLoadBalancerAttributes\",\"elasticloadbalancing:ModifyListenerAttributes\",\"elasticloadbalancing:ModifyIpPools\",\"elasticloadbalancing:ModifyCapacityReservation\",\"elasticloadbalancing:DeleteTargetGroup\",\"elasticloadbalancing:DeleteLoadBalancer\"],\"Condition\":{\"Null\":{\"aws:ResourceTag/elbv2.k8s.aws/cluster\":\"false\"}},\"Effect\":\"Allow\",\"Resource\":\"*\"},{\"Action\":\"elasticloadbalancing:AddTags\",\"Condition\":{\"Null\":{\"aws:RequestTag/elbv2.k8s.aws/cluster\":\"false\"},\"StringEquals\":{\"elasticloadbalancing:CreateAction\":[\"CreateTargetGroup\",\"CreateLoadBalancer\"]}},\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:elasticloadbalancing:*:*:targetgroup/*/*\",\"arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*\",\"arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*\"]},{\"Action\":[\"elasticloadbalancing:RegisterTargets\",\"elasticloadbalancing:DeregisterTargets\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:elasticloadbalancing:*:*:targetgroup/*/*\"},{\"Action\":[\"elasticloadbalancing:SetWebAcl\",\"elasticloadbalancing:SetRulePriorities\",\"elasticloadbalancing:RemoveListenerCertificates\",\"elasticloadbalancing:ModifyRule\",\"elasticloadbalancing:ModifyListener\",\"elasticloadbalancing:AddListenerCertificates\"],\"Effect\":\"Allow\",\"Resource\":\"*\"}],\"Version\":\"2012-10-17\"}", + "policy_id": "ANPAQMYUSQYYYIABW6PHV", + "tags": {}, + "tags_all": { + "ManagedBy": "terraform", + "Project": "observability-stack" + } + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.lb_controller_irsa.data.aws_iam_policy_document.load_balancer_controller", + "module.lb_controller_irsa.data.aws_partition.current" + ] + } + ] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "load_balancer_controller_targetgroup_only", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "mountpoint_s3_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "node_termination_handler", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "velero", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_policy", + "name": "vpc_cni", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_role", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::027423573553:role/observability-stack-lb-controller", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRoleWithWebIdentity\",\"Condition\":{\"StringEquals\":{\"oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D:aud\":\"sts.amazonaws.com\",\"oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D:sub\":\"system:serviceaccount:kube-system:aws-load-balancer-controller\"}},\"Effect\":\"Allow\",\"Principal\":{\"Federated\":\"arn:aws:iam::027423573553:oidc-provider/oidc.eks.us-west-2.amazonaws.com/id/91D986DCB52447A7F312838B02FCD62D\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-03-18T20:28:14Z", + "description": "", + "force_detach_policies": true, + "id": "observability-stack-lb-controller", + "inline_policy": [], + "managed_policy_arns": [ + "arn:aws:iam::027423573553:policy/AmazonEKS_AWS_Load_Balancer_Controller-20260318201827555800000006" + ], + "max_session_duration": 3600, + "name": "observability-stack-lb-controller", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": {}, + "tags_all": { + "ManagedBy": "terraform", + "Project": "observability-stack" + }, + "unique_id": "AROAQMYUSQYYXI2MJFYL2" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_availability_zones.available", + "module.eks.aws_cloudwatch_log_group.this", + "module.eks.aws_eks_cluster.this", + "module.eks.aws_iam_openid_connect_provider.oidc_provider", + "module.eks.aws_iam_policy.cni_ipv6_policy", + "module.eks.aws_iam_role.eks_auto", + "module.eks.aws_iam_role.this", + "module.eks.aws_iam_role_policy_attachment.this", + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.eks.aws_security_group_rule.cluster", + "module.eks.aws_security_group_rule.node", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_policy_document.cni_ipv6_policy", + "module.eks.data.aws_iam_policy_document.node_assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.data.aws_partition.current", + "module.eks.data.tls_certificate.this", + "module.eks.module.kms.aws_kms_external_key.this", + "module.eks.module.kms.aws_kms_key.this", + "module.eks.module.kms.aws_kms_replica_external_key.this", + "module.eks.module.kms.aws_kms_replica_key.this", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current", + "module.lb_controller_irsa.data.aws_caller_identity.current", + "module.lb_controller_irsa.data.aws_iam_policy_document.this", + "module.lb_controller_irsa.data.aws_partition.current", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "amazon_cloudwatch_observability", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "amazon_managed_service_prometheus", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "appmesh_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "appmesh_envoy_proxy", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "aws_gateway_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "cert_manager", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "cluster_autoscaler", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "ebs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "efs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "external_dns", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "external_secrets", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "fsx_lustre_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "fsx_openzfs_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "karpenter_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "load_balancer_controller", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "id": "observability-stack-lb-controller-20260318202815389000000016", + "policy_arn": "arn:aws:iam::027423573553:policy/AmazonEKS_AWS_Load_Balancer_Controller-20260318201827555800000006", + "role": "observability-stack-lb-controller" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_availability_zones.available", + "module.eks.aws_cloudwatch_log_group.this", + "module.eks.aws_eks_cluster.this", + "module.eks.aws_iam_openid_connect_provider.oidc_provider", + "module.eks.aws_iam_policy.cni_ipv6_policy", + "module.eks.aws_iam_role.eks_auto", + "module.eks.aws_iam_role.this", + "module.eks.aws_iam_role_policy_attachment.this", + "module.eks.aws_security_group.cluster", + "module.eks.aws_security_group.node", + "module.eks.aws_security_group_rule.cluster", + "module.eks.aws_security_group_rule.node", + "module.eks.data.aws_caller_identity.current", + "module.eks.data.aws_iam_policy_document.assume_role_policy", + "module.eks.data.aws_iam_policy_document.cni_ipv6_policy", + "module.eks.data.aws_iam_policy_document.node_assume_role_policy", + "module.eks.data.aws_iam_session_context.current", + "module.eks.data.aws_partition.current", + "module.eks.data.tls_certificate.this", + "module.eks.module.kms.aws_kms_external_key.this", + "module.eks.module.kms.aws_kms_key.this", + "module.eks.module.kms.aws_kms_replica_external_key.this", + "module.eks.module.kms.aws_kms_replica_key.this", + "module.eks.module.kms.data.aws_caller_identity.current", + "module.eks.module.kms.data.aws_iam_policy_document.this", + "module.eks.module.kms.data.aws_partition.current", + "module.lb_controller_irsa.aws_iam_policy.load_balancer_controller", + "module.lb_controller_irsa.aws_iam_role.this", + "module.lb_controller_irsa.data.aws_caller_identity.current", + "module.lb_controller_irsa.data.aws_iam_policy_document.load_balancer_controller", + "module.lb_controller_irsa.data.aws_iam_policy_document.this", + "module.lb_controller_irsa.data.aws_partition.current", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "load_balancer_controller_targetgroup_only", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "mountpoint_s3_csi", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "node_termination_handler", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "velero", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.lb_controller_irsa", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "vpc_cni", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "data", + "type": "aws_caller_identity", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "flow_log_cloudwatch_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "data", + "type": "aws_iam_policy_document", + "name": "vpc_flow_log_cloudwatch", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "data", + "type": "aws_partition", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "data", + "type": "aws_region", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_cloudwatch_log_group", + "name": "flow_log", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_customer_gateway", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_db_subnet_group", + "name": "database", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_default_network_acl", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:ec2:us-west-2:027423573553:network-acl/acl-0a5e114c086bba3b7", + "default_network_acl_id": "acl-0a5e114c086bba3b7", + "egress": [ + { + "action": "allow", + "cidr_block": "", + "from_port": 0, + "icmp_code": 0, + "icmp_type": 0, + "ipv6_cidr_block": "::/0", + "protocol": "-1", + "rule_no": 101, + "to_port": 0 + }, + { + "action": "allow", + "cidr_block": "0.0.0.0/0", + "from_port": 0, + "icmp_code": 0, + "icmp_type": 0, + "ipv6_cidr_block": "", + "protocol": "-1", + "rule_no": 100, + "to_port": 0 + } + ], + "id": "acl-0a5e114c086bba3b7", + "ingress": [ + { + "action": "allow", + "cidr_block": "", + "from_port": 0, + "icmp_code": 0, + "icmp_type": 0, + "ipv6_cidr_block": "::/0", + "protocol": "-1", + "rule_no": 101, + "to_port": 0 + }, + { + "action": "allow", + "cidr_block": "0.0.0.0/0", + "from_port": 0, + "icmp_code": 0, + "icmp_type": 0, + "ipv6_cidr_block": "", + "protocol": "-1", + "rule_no": 100, + "to_port": 0 + } + ], + "owner_id": "027423573553", + "subnet_ids": [ + "subnet-00bc6b6598ab09efb", + "subnet-0103d5dff6443d113", + "subnet-02fb98c38c541a8d8", + "subnet-06f8647c688701ae4", + "subnet-071119bffc7eafe55", + "subnet-0862c3b38b530fd97" + ], + "tags": { + "Name": "observability-stack-default" + }, + "tags_all": { + "ManagedBy": "terraform", + "Name": "observability-stack-default", + "Project": "observability-stack" + }, + "vpc_id": "vpc-03aefcf5aa0581d7a" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.vpc.aws_vpc.this" + ] + } + ] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_default_route_table", + "name": "default", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:ec2:us-west-2:027423573553:route-table/rtb-0eb0c8a9b7385e199", + "default_route_table_id": "rtb-0eb0c8a9b7385e199", + "id": "rtb-0eb0c8a9b7385e199", + "owner_id": "027423573553", + "propagating_vgws": [], + "route": [], + "tags": { + "Name": "observability-stack-default" + }, + "tags_all": { + "ManagedBy": "terraform", + "Name": "observability-stack-default", + "Project": "observability-stack" + }, + "timeouts": { + "create": "5m", + "update": "5m" + }, + "vpc_id": "vpc-03aefcf5aa0581d7a" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsInVwZGF0ZSI6MzAwMDAwMDAwMDAwfX0=", + "dependencies": [ + "module.vpc.aws_vpc.this" + ] + } + ] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_default_security_group", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-west-2:027423573553:security-group/sg-01804f6e1f144e984", + "description": "default VPC security group", + "egress": [], + "id": "sg-01804f6e1f144e984", + "ingress": [], + "name": "default", + "name_prefix": "", + "owner_id": "027423573553", + "revoke_rules_on_delete": false, + "tags": { + "Name": "observability-stack-default" + }, + "tags_all": { + "ManagedBy": "terraform", + "Name": "observability-stack-default", + "Project": "observability-stack" + }, + "vpc_id": "vpc-03aefcf5aa0581d7a" + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "module.vpc.aws_vpc.this" + ] + } + ] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_default_vpc", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_egress_only_internet_gateway", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_eip", + "name": "nat", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "address": null, + "allocation_id": "eipalloc-0113a2632fcf6cddf", + "arn": "arn:aws:ec2:us-west-2:027423573553:elastic-ip/eipalloc-0113a2632fcf6cddf", + "associate_with_private_ip": null, + "association_id": "eipassoc-04a75448822bedae0", + "carrier_ip": "", + "customer_owned_ip": "", + "customer_owned_ipv4_pool": "", + "domain": "vpc", + "id": "eipalloc-0113a2632fcf6cddf", + "instance": "", + "ipam_pool_id": null, + "network_border_group": "us-west-2", + "network_interface": "eni-0d26887f86c85fe3e", + "private_dns": "ip-10-0-120-98.us-west-2.compute.internal", + "private_ip": "10.0.120.98", + "ptr_record": "", + "public_dns": "ec2-52-88-124-67.us-west-2.compute.amazonaws.com", + "public_ip": "52.88.124.67", + "public_ipv4_pool": "amazon", + "tags": { + "Name": "observability-stack-us-west-2a" + }, + "tags_all": { + "ManagedBy": "terraform", + "Name": "observability-stack-us-west-2a", + "Project": "observability-stack" + }, + "timeouts": null, + "vpc": true + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjoxODAwMDAwMDAwMDAsInJlYWQiOjkwMDAwMDAwMDAwMCwidXBkYXRlIjozMDAwMDAwMDAwMDB9fQ==", + "dependencies": [ + "data.aws_availability_zones.available", + "module.vpc.aws_internet_gateway.this", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_elasticache_subnet_group", + "name": "elasticache", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_flow_log", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_iam_policy", + "name": "vpc_flow_log_cloudwatch", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_iam_role", + "name": "vpc_flow_log_cloudwatch", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "vpc_flow_log_cloudwatch", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_internet_gateway", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:ec2:us-west-2:027423573553:internet-gateway/igw-0017951791f30c8cc", + "id": "igw-0017951791f30c8cc", + "owner_id": "027423573553", + "tags": { + "Name": "observability-stack" + }, + "tags_all": { + "ManagedBy": "terraform", + "Name": "observability-stack", + "Project": "observability-stack" + }, + "timeouts": null, + "vpc_id": "vpc-03aefcf5aa0581d7a" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19", + "dependencies": [ + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_nat_gateway", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "allocation_id": "eipalloc-0113a2632fcf6cddf", + "association_id": "eipassoc-04a75448822bedae0", + "connectivity_type": "public", + "id": "nat-0777712a8d4bdb8c1", + "network_interface_id": "eni-0d26887f86c85fe3e", + "private_ip": "10.0.120.98", + "public_ip": "52.88.124.67", + "secondary_allocation_ids": [], + "secondary_private_ip_address_count": 0, + "secondary_private_ip_addresses": [], + "subnet_id": "subnet-0103d5dff6443d113", + "tags": { + "Name": "observability-stack-us-west-2a" + }, + "tags_all": { + "ManagedBy": "terraform", + "Name": "observability-stack-us-west-2a", + "Project": "observability-stack" + }, + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTgwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9fQ==", + "dependencies": [ + "data.aws_availability_zones.available", + "module.vpc.aws_eip.nat", + "module.vpc.aws_internet_gateway.this", + "module.vpc.aws_subnet.public", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_network_acl", + "name": "database", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_network_acl", + "name": "elasticache", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_network_acl", + "name": "intra", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_network_acl", + "name": "outpost", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_network_acl", + "name": "private", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_network_acl", + "name": "public", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_network_acl", + "name": "redshift", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_network_acl_rule", + "name": "database_inbound", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_network_acl_rule", + "name": "database_outbound", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_network_acl_rule", + "name": "elasticache_inbound", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_network_acl_rule", + "name": "elasticache_outbound", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_network_acl_rule", + "name": "intra_inbound", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_network_acl_rule", + "name": "intra_outbound", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_network_acl_rule", + "name": "outpost_inbound", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_network_acl_rule", + "name": "outpost_outbound", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_network_acl_rule", + "name": "private_inbound", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_network_acl_rule", + "name": "private_outbound", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_network_acl_rule", + "name": "public_inbound", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_network_acl_rule", + "name": "public_outbound", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_network_acl_rule", + "name": "redshift_inbound", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_network_acl_rule", + "name": "redshift_outbound", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_redshift_subnet_group", + "name": "redshift", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_route", + "name": "database_dns64_nat_gateway", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_route", + "name": "database_internet_gateway", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_route", + "name": "database_ipv6_egress", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_route", + "name": "database_nat_gateway", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_route", + "name": "private_dns64_nat_gateway", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_route", + "name": "private_ipv6_egress", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_route", + "name": "private_nat_gateway", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "carrier_gateway_id": "", + "core_network_arn": "", + "destination_cidr_block": "0.0.0.0/0", + "destination_ipv6_cidr_block": "", + "destination_prefix_list_id": "", + "egress_only_gateway_id": "", + "gateway_id": "", + "id": "r-rtb-09296e0d036ce62661080289494", + "instance_id": "", + "instance_owner_id": "", + "local_gateway_id": "", + "nat_gateway_id": "nat-0777712a8d4bdb8c1", + "network_interface_id": "", + "origin": "CreateRoute", + "route_table_id": "rtb-09296e0d036ce6266", + "state": "active", + "timeouts": { + "create": "5m", + "delete": null, + "update": null + }, + "transit_gateway_id": "", + "vpc_endpoint_id": "", + "vpc_peering_connection_id": "" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_availability_zones.available", + "module.vpc.aws_eip.nat", + "module.vpc.aws_internet_gateway.this", + "module.vpc.aws_nat_gateway.this", + "module.vpc.aws_route_table.private", + "module.vpc.aws_subnet.public", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_route", + "name": "public_internet_gateway", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "carrier_gateway_id": "", + "core_network_arn": "", + "destination_cidr_block": "0.0.0.0/0", + "destination_ipv6_cidr_block": "", + "destination_prefix_list_id": "", + "egress_only_gateway_id": "", + "gateway_id": "igw-0017951791f30c8cc", + "id": "r-rtb-073d827383549597d1080289494", + "instance_id": "", + "instance_owner_id": "", + "local_gateway_id": "", + "nat_gateway_id": "", + "network_interface_id": "", + "origin": "CreateRoute", + "route_table_id": "rtb-073d827383549597d", + "state": "active", + "timeouts": { + "create": "5m", + "delete": null, + "update": null + }, + "transit_gateway_id": "", + "vpc_endpoint_id": "", + "vpc_peering_connection_id": "" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_availability_zones.available", + "module.vpc.aws_internet_gateway.this", + "module.vpc.aws_route_table.public", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_route", + "name": "public_internet_gateway_ipv6", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_route_table", + "name": "database", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_route_table", + "name": "elasticache", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_route_table", + "name": "intra", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_route_table", + "name": "private", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:ec2:us-west-2:027423573553:route-table/rtb-09296e0d036ce6266", + "id": "rtb-09296e0d036ce6266", + "owner_id": "027423573553", + "propagating_vgws": [], + "route": [ + { + "carrier_gateway_id": "", + "cidr_block": "0.0.0.0/0", + "core_network_arn": "", + "destination_prefix_list_id": "", + "egress_only_gateway_id": "", + "gateway_id": "", + "ipv6_cidr_block": "", + "local_gateway_id": "", + "nat_gateway_id": "nat-0777712a8d4bdb8c1", + "network_interface_id": "", + "transit_gateway_id": "", + "vpc_endpoint_id": "", + "vpc_peering_connection_id": "" + } + ], + "tags": { + "Name": "observability-stack-private" + }, + "tags_all": { + "ManagedBy": "terraform", + "Name": "observability-stack-private", + "Project": "observability-stack" + }, + "timeouts": null, + "vpc_id": "vpc-03aefcf5aa0581d7a" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_availability_zones.available", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_route_table", + "name": "public", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:ec2:us-west-2:027423573553:route-table/rtb-073d827383549597d", + "id": "rtb-073d827383549597d", + "owner_id": "027423573553", + "propagating_vgws": [], + "route": [ + { + "carrier_gateway_id": "", + "cidr_block": "0.0.0.0/0", + "core_network_arn": "", + "destination_prefix_list_id": "", + "egress_only_gateway_id": "", + "gateway_id": "igw-0017951791f30c8cc", + "ipv6_cidr_block": "", + "local_gateway_id": "", + "nat_gateway_id": "", + "network_interface_id": "", + "transit_gateway_id": "", + "vpc_endpoint_id": "", + "vpc_peering_connection_id": "" + } + ], + "tags": { + "Name": "observability-stack-public" + }, + "tags_all": { + "ManagedBy": "terraform", + "Name": "observability-stack-public", + "Project": "observability-stack" + }, + "timeouts": null, + "vpc_id": "vpc-03aefcf5aa0581d7a" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_availability_zones.available", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_route_table", + "name": "redshift", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_route_table_association", + "name": "database", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_route_table_association", + "name": "elasticache", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_route_table_association", + "name": "intra", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_route_table_association", + "name": "outpost", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_route_table_association", + "name": "private", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "gateway_id": "", + "id": "rtbassoc-0733c252b1b4d11ad", + "route_table_id": "rtb-09296e0d036ce6266", + "subnet_id": "subnet-00bc6b6598ab09efb", + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_availability_zones.available", + "module.vpc.aws_route_table.private", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 1, + "schema_version": 0, + "attributes": { + "gateway_id": "", + "id": "rtbassoc-09f0385e55967b756", + "route_table_id": "rtb-09296e0d036ce6266", + "subnet_id": "subnet-0862c3b38b530fd97", + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_availability_zones.available", + "module.vpc.aws_route_table.private", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 2, + "schema_version": 0, + "attributes": { + "gateway_id": "", + "id": "rtbassoc-0ba7f463260829933", + "route_table_id": "rtb-09296e0d036ce6266", + "subnet_id": "subnet-06f8647c688701ae4", + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_availability_zones.available", + "module.vpc.aws_route_table.private", + "module.vpc.aws_subnet.private", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_route_table_association", + "name": "public", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "gateway_id": "", + "id": "rtbassoc-024db46e61ffe0bb4", + "route_table_id": "rtb-073d827383549597d", + "subnet_id": "subnet-0103d5dff6443d113", + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_availability_zones.available", + "module.vpc.aws_route_table.public", + "module.vpc.aws_subnet.public", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 1, + "schema_version": 0, + "attributes": { + "gateway_id": "", + "id": "rtbassoc-0c441f9a8fd6291e5", + "route_table_id": "rtb-073d827383549597d", + "subnet_id": "subnet-071119bffc7eafe55", + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_availability_zones.available", + "module.vpc.aws_route_table.public", + "module.vpc.aws_subnet.public", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 2, + "schema_version": 0, + "attributes": { + "gateway_id": "", + "id": "rtbassoc-0a754d2b6cb30938f", + "route_table_id": "rtb-073d827383549597d", + "subnet_id": "subnet-02fb98c38c541a8d8", + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_availability_zones.available", + "module.vpc.aws_route_table.public", + "module.vpc.aws_subnet.public", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_route_table_association", + "name": "redshift", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_route_table_association", + "name": "redshift_public", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_subnet", + "name": "database", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_subnet", + "name": "elasticache", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_subnet", + "name": "intra", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_subnet", + "name": "outpost", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_subnet", + "name": "private", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-west-2:027423573553:subnet/subnet-00bc6b6598ab09efb", + "assign_ipv6_address_on_creation": false, + "availability_zone": "us-west-2a", + "availability_zone_id": "usw2-az2", + "cidr_block": "10.0.0.0/19", + "customer_owned_ipv4_pool": "", + "enable_dns64": false, + "enable_lni_at_device_index": 0, + "enable_resource_name_dns_a_record_on_launch": false, + "enable_resource_name_dns_aaaa_record_on_launch": false, + "id": "subnet-00bc6b6598ab09efb", + "ipv6_cidr_block": "", + "ipv6_cidr_block_association_id": "", + "ipv6_native": false, + "map_customer_owned_ip_on_launch": false, + "map_public_ip_on_launch": false, + "outpost_arn": "", + "owner_id": "027423573553", + "private_dns_hostname_type_on_launch": "ip-name", + "tags": { + "Name": "observability-stack-private-us-west-2a", + "kubernetes.io/role/internal-elb": "1" + }, + "tags_all": { + "ManagedBy": "terraform", + "Name": "observability-stack-private-us-west-2a", + "Project": "observability-stack", + "kubernetes.io/role/internal-elb": "1" + }, + "timeouts": null, + "vpc_id": "vpc-03aefcf5aa0581d7a" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "data.aws_availability_zones.available", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ], + "create_before_destroy": true + }, + { + "index_key": 1, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-west-2:027423573553:subnet/subnet-0862c3b38b530fd97", + "assign_ipv6_address_on_creation": false, + "availability_zone": "us-west-2b", + "availability_zone_id": "usw2-az1", + "cidr_block": "10.0.32.0/19", + "customer_owned_ipv4_pool": "", + "enable_dns64": false, + "enable_lni_at_device_index": 0, + "enable_resource_name_dns_a_record_on_launch": false, + "enable_resource_name_dns_aaaa_record_on_launch": false, + "id": "subnet-0862c3b38b530fd97", + "ipv6_cidr_block": "", + "ipv6_cidr_block_association_id": "", + "ipv6_native": false, + "map_customer_owned_ip_on_launch": false, + "map_public_ip_on_launch": false, + "outpost_arn": "", + "owner_id": "027423573553", + "private_dns_hostname_type_on_launch": "ip-name", + "tags": { + "Name": "observability-stack-private-us-west-2b", + "kubernetes.io/role/internal-elb": "1" + }, + "tags_all": { + "ManagedBy": "terraform", + "Name": "observability-stack-private-us-west-2b", + "Project": "observability-stack", + "kubernetes.io/role/internal-elb": "1" + }, + "timeouts": null, + "vpc_id": "vpc-03aefcf5aa0581d7a" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "data.aws_availability_zones.available", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ], + "create_before_destroy": true + }, + { + "index_key": 2, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-west-2:027423573553:subnet/subnet-06f8647c688701ae4", + "assign_ipv6_address_on_creation": false, + "availability_zone": "us-west-2c", + "availability_zone_id": "usw2-az3", + "cidr_block": "10.0.64.0/19", + "customer_owned_ipv4_pool": "", + "enable_dns64": false, + "enable_lni_at_device_index": 0, + "enable_resource_name_dns_a_record_on_launch": false, + "enable_resource_name_dns_aaaa_record_on_launch": false, + "id": "subnet-06f8647c688701ae4", + "ipv6_cidr_block": "", + "ipv6_cidr_block_association_id": "", + "ipv6_native": false, + "map_customer_owned_ip_on_launch": false, + "map_public_ip_on_launch": false, + "outpost_arn": "", + "owner_id": "027423573553", + "private_dns_hostname_type_on_launch": "ip-name", + "tags": { + "Name": "observability-stack-private-us-west-2c", + "kubernetes.io/role/internal-elb": "1" + }, + "tags_all": { + "ManagedBy": "terraform", + "Name": "observability-stack-private-us-west-2c", + "Project": "observability-stack", + "kubernetes.io/role/internal-elb": "1" + }, + "timeouts": null, + "vpc_id": "vpc-03aefcf5aa0581d7a" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "data.aws_availability_zones.available", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ], + "create_before_destroy": true + } + ] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_subnet", + "name": "public", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-west-2:027423573553:subnet/subnet-0103d5dff6443d113", + "assign_ipv6_address_on_creation": false, + "availability_zone": "us-west-2a", + "availability_zone_id": "usw2-az2", + "cidr_block": "10.0.96.0/19", + "customer_owned_ipv4_pool": "", + "enable_dns64": false, + "enable_lni_at_device_index": 0, + "enable_resource_name_dns_a_record_on_launch": false, + "enable_resource_name_dns_aaaa_record_on_launch": false, + "id": "subnet-0103d5dff6443d113", + "ipv6_cidr_block": "", + "ipv6_cidr_block_association_id": "", + "ipv6_native": false, + "map_customer_owned_ip_on_launch": false, + "map_public_ip_on_launch": false, + "outpost_arn": "", + "owner_id": "027423573553", + "private_dns_hostname_type_on_launch": "ip-name", + "tags": { + "Name": "observability-stack-public-us-west-2a", + "kubernetes.io/role/elb": "1" + }, + "tags_all": { + "ManagedBy": "terraform", + "Name": "observability-stack-public-us-west-2a", + "Project": "observability-stack", + "kubernetes.io/role/elb": "1" + }, + "timeouts": null, + "vpc_id": "vpc-03aefcf5aa0581d7a" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "data.aws_availability_zones.available", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 1, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-west-2:027423573553:subnet/subnet-071119bffc7eafe55", + "assign_ipv6_address_on_creation": false, + "availability_zone": "us-west-2b", + "availability_zone_id": "usw2-az1", + "cidr_block": "10.0.128.0/19", + "customer_owned_ipv4_pool": "", + "enable_dns64": false, + "enable_lni_at_device_index": 0, + "enable_resource_name_dns_a_record_on_launch": false, + "enable_resource_name_dns_aaaa_record_on_launch": false, + "id": "subnet-071119bffc7eafe55", + "ipv6_cidr_block": "", + "ipv6_cidr_block_association_id": "", + "ipv6_native": false, + "map_customer_owned_ip_on_launch": false, + "map_public_ip_on_launch": false, + "outpost_arn": "", + "owner_id": "027423573553", + "private_dns_hostname_type_on_launch": "ip-name", + "tags": { + "Name": "observability-stack-public-us-west-2b", + "kubernetes.io/role/elb": "1" + }, + "tags_all": { + "ManagedBy": "terraform", + "Name": "observability-stack-public-us-west-2b", + "Project": "observability-stack", + "kubernetes.io/role/elb": "1" + }, + "timeouts": null, + "vpc_id": "vpc-03aefcf5aa0581d7a" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "data.aws_availability_zones.available", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 2, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-west-2:027423573553:subnet/subnet-02fb98c38c541a8d8", + "assign_ipv6_address_on_creation": false, + "availability_zone": "us-west-2c", + "availability_zone_id": "usw2-az3", + "cidr_block": "10.0.160.0/19", + "customer_owned_ipv4_pool": "", + "enable_dns64": false, + "enable_lni_at_device_index": 0, + "enable_resource_name_dns_a_record_on_launch": false, + "enable_resource_name_dns_aaaa_record_on_launch": false, + "id": "subnet-02fb98c38c541a8d8", + "ipv6_cidr_block": "", + "ipv6_cidr_block_association_id": "", + "ipv6_native": false, + "map_customer_owned_ip_on_launch": false, + "map_public_ip_on_launch": false, + "outpost_arn": "", + "owner_id": "027423573553", + "private_dns_hostname_type_on_launch": "ip-name", + "tags": { + "Name": "observability-stack-public-us-west-2c", + "kubernetes.io/role/elb": "1" + }, + "tags_all": { + "ManagedBy": "terraform", + "Name": "observability-stack-public-us-west-2c", + "Project": "observability-stack", + "kubernetes.io/role/elb": "1" + }, + "timeouts": null, + "vpc_id": "vpc-03aefcf5aa0581d7a" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "data.aws_availability_zones.available", + "module.vpc.aws_vpc.this", + "module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_subnet", + "name": "redshift", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_vpc", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-west-2:027423573553:vpc/vpc-03aefcf5aa0581d7a", + "assign_generated_ipv6_cidr_block": false, + "cidr_block": "10.0.0.0/16", + "default_network_acl_id": "acl-0a5e114c086bba3b7", + "default_route_table_id": "rtb-0eb0c8a9b7385e199", + "default_security_group_id": "sg-01804f6e1f144e984", + "dhcp_options_id": "dopt-092ebbc7273e1297b", + "enable_dns_hostnames": true, + "enable_dns_support": true, + "enable_network_address_usage_metrics": false, + "id": "vpc-03aefcf5aa0581d7a", + "instance_tenancy": "default", + "ipv4_ipam_pool_id": null, + "ipv4_netmask_length": null, + "ipv6_association_id": "", + "ipv6_cidr_block": "", + "ipv6_cidr_block_network_border_group": "", + "ipv6_ipam_pool_id": "", + "ipv6_netmask_length": 0, + "main_route_table_id": "rtb-0eb0c8a9b7385e199", + "owner_id": "027423573553", + "tags": { + "Name": "observability-stack" + }, + "tags_all": { + "ManagedBy": "terraform", + "Name": "observability-stack", + "Project": "observability-stack" + } + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "create_before_destroy": true + } + ] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_vpc_block_public_access_exclusion", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_vpc_block_public_access_options", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_vpc_dhcp_options", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_vpc_dhcp_options_association", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_vpc_ipv4_cidr_block_association", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_vpn_gateway", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_vpn_gateway_attachment", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_vpn_gateway_route_propagation", + "name": "intra", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_vpn_gateway_route_propagation", + "name": "private", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + }, + { + "module": "module.vpc", + "mode": "managed", + "type": "aws_vpn_gateway_route_propagation", + "name": "public", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [] + } + ], + "check_results": [ + { + "object_kind": "resource", + "config_addr": "module.eks.module.eks_managed_node_group.module.user_data.null_resource.validate_cluster_service_cidr", + "status": "unknown", + "objects": [ + { + "object_addr": "module.eks.module.eks_managed_node_group[\"default\"].module.user_data.null_resource.validate_cluster_service_cidr", + "status": "unknown" + } + ] + }, + { + "object_kind": "resource", + "config_addr": "module.eks.module.self_managed_node_group.module.user_data.null_resource.validate_cluster_service_cidr", + "status": "pass", + "objects": null + } + ] +} diff --git a/terraform/aws/values-eks.yaml b/terraform/aws/values-eks.yaml index 32aa76e1..754f2fc2 100644 --- a/terraform/aws/values-eks.yaml +++ b/terraform/aws/values-eks.yaml @@ -20,6 +20,8 @@ opensearch: memory: "4Gi" cpu: "4000m" extraEnvs: + - name: OPENSEARCH_INITIAL_ADMIN_PASSWORD + value: "My_password_123!@#" - name: OPENSEARCH_JAVA_OPTS value: "-Xms2g -Xmx2g" @@ -48,7 +50,7 @@ opensearch-dashboards: # Examples — pulls from GHCR (public images built by .github/workflows/publish-images.yml) examples: - enabled: true + enabled: false # Prometheus — persistent storage + resource limits prometheus: @@ -66,7 +68,7 @@ prometheus: # OTel Demo — realistic telemetry from ~20 microservices opentelemetry-demo: - enabled: true + enabled: false # Gateway not needed — ALB handles ingress directly. gateway: