fix: use FQDN for Prometheus host to avoid corporate DNS conflicts (#191)

The bare hostname `prometheus` gets resolved via DNS search domains
(e.g. `prometheus.amazon.com`) instead of Docker's internal DNS,
causing Data Prepper's Armeria client to cache the wrong IP and fail
to write metrics to Prometheus.

- Change default PROMETHEUS_HOST to `prometheus.observability-stack-network`
- Templatize Prometheus host/port in Data Prepper pipeline config
- Add sed replacements and env vars for PROMETHEUS_HOST/PORT in docker-compose
- Update fallback default in opensearch-dashboards init script

Signed-off-by: Vamsi Manohar <reddyvam@amazon.com>

Author: vamsimanohar

.env

@@ -49,7 +49,7 @@ ISM_RETENTION_DAYS=7
 
 # Prometheus Configuration
 PROMETHEUS_VERSION=v3.8.1
-PROMETHEUS_HOST=prometheus
+PROMETHEUS_HOST=prometheus.observability-stack-network
 PROMETHEUS_PORT=9090
 PROMETHEUS_RETENTION=15d
 

docker-compose.yml

@@ -75,6 +75,8 @@ services:
       sed -i 's|OPENSEARCH_PROTOCOL|${OPENSEARCH_PROTOCOL}|g' /tmp/pipelines.yaml &&
       sed -i 's|OPENSEARCH_HOST|${OPENSEARCH_HOST}|g' /tmp/pipelines.yaml &&
       sed -i 's|OPENSEARCH_PORT|${OPENSEARCH_PORT}|g' /tmp/pipelines.yaml &&
+      sed -i 's|PROMETHEUS_HOST|${PROMETHEUS_HOST}|g' /tmp/pipelines.yaml &&
+      sed -i 's|PROMETHEUS_PORT|${PROMETHEUS_PORT}|g' /tmp/pipelines.yaml &&
       mv /tmp/pipelines.yaml /usr/share/data-prepper/pipelines/pipelines.yaml &&
       exec /usr/share/data-prepper/bin/data-prepper"
     volumes:
@@ -90,6 +92,8 @@ services:
       - OPENSEARCH_PORT=${OPENSEARCH_PORT}
       - OPENSEARCH_USER=${OPENSEARCH_USER}
       - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
+      - PROMETHEUS_HOST=${PROMETHEUS_HOST}
+      - PROMETHEUS_PORT=${PROMETHEUS_PORT}
     depends_on:
       opensearch:
         condition: service_healthy

docker-compose/data-prepper/pipelines.template.yaml

@@ -109,7 +109,7 @@ service-map-pipeline:
         insecure: true
     # Route RED metrics to local Prometheus via remote write
     - prometheus:
-        url: "http://prometheus:9090/api/v1/write"
+        url: "http://PROMETHEUS_HOST:PROMETHEUS_PORT/api/v1/write"
         threshold:
           max_events: 500
           flush_interval: 5s

docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py

@@ -11,7 +11,7 @@
 BASE_URL = f"{_dashboards_protocol}://{_dashboards_host}:{_dashboards_port}"
 USERNAME = os.getenv("OPENSEARCH_USER", "admin")
 PASSWORD = os.getenv("OPENSEARCH_PASSWORD", "My_password_123!@#")
-PROMETHEUS_HOST = os.getenv("PROMETHEUS_HOST", "prometheus")
+PROMETHEUS_HOST = os.getenv("PROMETHEUS_HOST", "prometheus.observability-stack-network")
 PROMETHEUS_PORT = os.getenv("PROMETHEUS_PORT", "9090")
 _opensearch_protocol = os.getenv("OPENSEARCH_PROTOCOL", "https")
 OPENSEARCH_ENDPOINT = f"{_opensearch_protocol}://{os.getenv('OPENSEARCH_HOST', 'opensearch')}:{os.getenv('OPENSEARCH_PORT', '9200')}"