feat: enable APM RED metrics ingestion into Prometheus via Data Prepper

.env

@@ -32,7 +32,8 @@ OTEL_COLLECTOR_PORT_HTTP=4318
 OTEL_COLLECTOR_METRICS_PORT=8888
 
 # Data Prepper Configuration
-DATA_PREPPER_VERSION=2.14.0
+DATA_PREPPER_VERSION=2.15.0-SNAPSHOT
+DATA_PREPPER_DOCKER_REPO=sgguruda62324
 DATA_PREPPER_OTLP_PORT=21890
 DATA_PREPPER_HTTP_PORT=21892
 

docker-compose.yml

@@ -61,7 +61,7 @@ services:
 
   # Data Prepper - Transforms and enriches logs/traces before OpenSearch ingestion
   data-prepper:
-    image: opensearchproject/data-prepper:${DATA_PREPPER_VERSION}
+    image: ${DATA_PREPPER_DOCKER_REPO}/opensearch-data-prepper:${DATA_PREPPER_VERSION}
     container_name: data-prepper
     pull_policy: always
     platform: linux/amd64

docker-compose/data-prepper/pipelines.yaml

@@ -93,11 +93,19 @@ service-map-pipeline:
         window_duration: 10s
   route:
     - otel_apm_service_map_route: 'getEventType() == "SERVICE_MAP"'
+    - service_processed_metrics: 'getEventType() == "METRIC"'
   sink:
     - opensearch:
         hosts: ["https://opensearch:9200"]
         username: admin
         password: "My_password_123!@#"
         index_type: otel-v2-apm-service-map
         routes: [otel_apm_service_map_route]
-        insecure: true
+        insecure: true
+    # Route RED metrics to local Prometheus via remote write
+    - prometheus:
+        url: "http://prometheus:9090/api/v1/write"
+        threshold:
+          max_events: 500
+          flush_interval: 5s
+        routes: [service_processed_metrics]