feat(iac): Enable flag for teeing otel-collector metrics to otel-router

Author: wj-e2b

iac/modules/job-otel-collector/configs/otel-collector.yaml

@@ -457,6 +457,12 @@ exporters:
         name: "metrics_gauge"
       sum:
         name: "metrics_sum"
+%{ if enable_otel_router_metrics }
+  otlp/otel_router:
+    endpoint: 127.0.0.1:${otel_router_grpc_port}
+    tls:
+      insecure: true
+%{ endif }
 service:
   telemetry:
     logs:
@@ -524,6 +530,16 @@ service:
         - batch/clickhouse
       exporters:
         - clickhouse
+%{ if enable_otel_router_metrics }
+    metrics/external/otel_router:
+      receivers:
+        - otlp
+      processors:
+        - filter/external_metrics
+        - batch
+      exporters:
+        - otlp/otel_router
+%{ endif }
     traces:
       receivers:
         - otlp

iac/modules/job-otel-collector/main.tf

@@ -12,6 +12,9 @@ locals {
       clickhouse_port     = var.clickhouse_port
       clickhouse_host     = var.clickhouse_host
       clickhouse_database = var.clickhouse_database
+
+      enable_otel_router_metrics = var.enable_otel_router_metrics
+      otel_router_grpc_port      = var.otel_router_grpc_port
     },
   )
 
@@ -101,6 +104,18 @@ variable "clickhouse_database" {
   default = ""
 }
 
+variable "enable_otel_router_metrics" {
+  type        = bool
+  default     = false
+  description = "Enable teeing external customer metrics from otel-collector to otel-router."
+}
+
+variable "otel_router_grpc_port" {
+  type        = number
+  default     = 4320
+  description = "Local otel-router OTLP gRPC port used by otel-collector when otel-router metric teeing is enabled."
+}
+
 variable "otel_collector_config_override" {
   type        = string
   default     = ""

iac/provider-aws/Makefile

@@ -45,6 +45,8 @@ tf_vars := AWS_PROFILE=$(AWS_PROFILE) AWS_REGION=$(AWS_REGION) \
 	$(call tfvar, AWS_REGION) \
 	$(call tfvar, ENABLE_OTEL_ROUTER_LOGS) \
 	$(call tfvar, OTEL_ROUTER_HTTP_PORT) \
+	$(call tfvar, ENABLE_OTEL_ROUTER_METRICS) \
+	$(call tfvar, OTEL_ROUTER_GRPC_PORT) \
 	$(call tfvar, DB_MAX_OPEN_CONNECTIONS) \
 	$(call tfvar, DB_MIN_IDLE_CONNECTIONS) \
 	$(call tfvar, AUTH_DB_MAX_OPEN_CONNECTIONS) \

iac/provider-aws/main.tf

@@ -233,6 +233,9 @@ module "nomad" {
   enable_otel_router_logs = var.enable_otel_router_logs
   otel_router_http_port   = var.otel_router_http_port
 
+  enable_otel_router_metrics = var.enable_otel_router_metrics
+  otel_router_grpc_port      = var.otel_router_grpc_port
+
   db_max_open_connections      = var.db_max_open_connections
   db_min_idle_connections      = var.db_min_idle_connections
   auth_db_max_open_connections = var.auth_db_max_open_connections

iac/provider-aws/nomad/main.tf

@@ -43,6 +43,9 @@ module "otel_collector" {
   grafana_username             = var.grafana_username
   consul_token                 = var.consul_acl_token
 
+  enable_otel_router_metrics = var.enable_otel_router_metrics
+  otel_router_grpc_port      = var.otel_router_grpc_port
+
   clickhouse_username = var.clickhouse_username
   clickhouse_password = var.clickhouse_password
   clickhouse_port     = var.clickhouse_port

iac/provider-aws/nomad/variables.tf

@@ -309,6 +309,18 @@ variable "otel_router_http_port" {
   description = "Local otel-router Vector-compatible logs port used by Vector when otel-router log teeing is enabled."
 }
 
+variable "enable_otel_router_metrics" {
+  type        = bool
+  default     = false
+  description = "Enable teeing external customer metrics from otel-collector to otel-router."
+}
+
+variable "otel_router_grpc_port" {
+  type        = number
+  default     = 4320
+  description = "Local otel-router OTLP gRPC port used by otel-collector when otel-router metric teeing is enabled."
+}
+
 # Feature flags
 variable "launch_darkly_api_key" {
   type      = string

iac/provider-aws/variables.tf

@@ -204,3 +204,15 @@ variable "otel_router_http_port" {
   default     = 4321
   description = "Local otel-router Vector-compatible logs port used by Vector when otel-router log teeing is enabled."
 }
+
+variable "enable_otel_router_metrics" {
+  type        = bool
+  default     = false
+  description = "Enable teeing external customer metrics from otel-collector to otel-router."
+}
+
+variable "otel_router_grpc_port" {
+  type        = number
+  default     = 4320
+  description = "Local otel-router OTLP gRPC port used by otel-collector when otel-router metric teeing is enabled."
+}

iac/provider-gcp/Makefile

@@ -61,6 +61,8 @@ tf_vars := \
 	$(call tfvar, OTEL_COLLECTOR_RESOURCES_MEMORY_MB) \
 	$(call tfvar, ENABLE_OTEL_ROUTER_LOGS) \
 	$(call tfvar, OTEL_ROUTER_HTTP_PORT) \
+	$(call tfvar, ENABLE_OTEL_ROUTER_METRICS) \
+	$(call tfvar, OTEL_ROUTER_GRPC_PORT) \
 	$(call tfvar, TEMPLATE_BUCKET_NAME) \
 	$(call tfvar, TEMPLATE_BUCKET_LOCATION) \
 	$(call tfvar, ENVD_TIMEOUT) \

iac/provider-gcp/main.tf

@@ -287,6 +287,8 @@ module "nomad" {
   otel_collector_resources_cpu_count = var.otel_collector_resources_cpu_count
   enable_otel_router_logs            = var.enable_otel_router_logs
   otel_router_http_port              = var.otel_router_http_port
+  enable_otel_router_metrics         = var.enable_otel_router_metrics
+  otel_router_grpc_port              = var.otel_router_grpc_port
 
   # Dashboard API
   dashboard_api_count                          = var.dashboard_api_count

iac/provider-gcp/nomad/main.tf

@@ -322,6 +322,9 @@ module "otel_collector" {
   grafana_username             = data.google_secret_manager_secret_version.grafana_username.secret_data
   consul_token                 = var.consul_acl_token_secret
 
+  enable_otel_router_metrics = var.enable_otel_router_metrics
+  otel_router_grpc_port      = var.otel_router_grpc_port
+
   clickhouse_username = var.clickhouse_username
   clickhouse_password = random_password.clickhouse_password.result
   clickhouse_port     = var.clickhouse_server_port.port