feat: support cloud OpenSearch via environment variables (opensearch-project#89)

* feat: support cloud OpenSearch via environment variables

- Make OpenSearch Dashboards URL configurable (host, port, protocol)
- Move opensearch and opensearch-dashboards to docker-compose.local-opensearch.yml,
  included by default; comment out INCLUDE_COMPOSE_LOCAL_OPENSEARCH in .env to
  switch to a cloud cluster
- Make data-prepper pipelines use OPENSEARCH_HOST/PORT placeholders instead of
  hardcoded opensearch:9200
- Move opensearch-dashboards-init to main compose file so it works with both
  local and cloud deployments

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Signed-off-by: fudongying <fudongying@bytedance.com>

* feat: kylehounslow's comment

Signed-off-by: fudongying <fudongying@bytedance.com>

* fix: harden install.sh and document TLS skip-verify behavior

- Extract read_env_var() helper in install.sh to safely read .env values
  without crashing on missing keys (grep non-zero exit + set -e/pipefail)
- Add -k flag to OSD health check curl so HTTPS cloud Dashboards do not hang
- Document in README that all HTTPS connections skip certificate verification
  (Data Prepper, OSD config, init script, install.sh health checks) and
  list where to enable verification for production environments

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Signed-off-by: fudongying <fudongying@bytedance.com>

* chore: delete auth info in document

Signed-off-by: fudongying <fudongying@bytedance.com>

---------

Signed-off-by: fudongying <fudongying@bytedance.com>
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: fudongyingluck

.env

@@ -7,6 +7,8 @@
 # Include additional compose files (comment out to disable)
 INCLUDE_COMPOSE_EXAMPLES=docker-compose.examples.yml
 # INCLUDE_COMPOSE_OTEL_DEMO=docker-compose.otel-demo.yml
+INCLUDE_COMPOSE_LOCAL_OPENSEARCH=docker-compose.local-opensearch.yml
+INCLUDE_COMPOSE_LOCAL_OPENSEARCH_DASHBOARDS=docker-compose.local-opensearch-dashboards.yml
 
 # OPENSEARCH_DOCKER_REPO=opensearchproject
 OPENSEARCH_DOCKER_REPO=opensearchstaging
@@ -18,11 +20,14 @@ OPENSEARCH_USER=admin
 OPENSEARCH_PASSWORD='My_password_123!@#'
 OPENSEARCH_HOST=opensearch
 OPENSEARCH_PORT=9200
+OPENSEARCH_PROTOCOL=https
 OPENSEARCH_JAVA_OPTS=-Xms1g -Xmx1g
 
 # OpenSearch Dashboards Configuration
 OPENSEARCH_DASHBOARDS_VERSION=3.6.0
+OPENSEARCH_DASHBOARDS_HOST=opensearch-dashboards
 OPENSEARCH_DASHBOARDS_PORT=5601
+OPENSEARCH_DASHBOARDS_PROTOCOL=http
 
 # OpenTelemetry Collector Configuration
 OTEL_COLLECTOR_VERSION=0.146.1

docker-compose.local-opensearch-dashboards.yml

@@ -0,0 +1,47 @@
+# Local OpenSearch Dashboards - include this file when running OpenSearch Dashboards locally
+# Included by default via INCLUDE_COMPOSE_LOCAL_OPENSEARCH_DASHBOARDS in .env
+# To use a cloud OpenSearch Dashboards instance, comment out INCLUDE_COMPOSE_LOCAL_OPENSEARCH_DASHBOARDS in .env
+
+x-default-logging: &logging
+  driver: "json-file"
+  options:
+    max-size: "5m"
+    max-file: "2"
+    tag: "{{.Name}}"
+
+services:
+  # OpenSearch Dashboards - Web UI for visualizing logs and traces
+  opensearch-dashboards:
+    image: ${OPENSEARCH_DOCKER_REPO}/opensearch-dashboards:${OPENSEARCH_DASHBOARDS_VERSION}
+    container_name: opensearch-dashboards
+    pull_policy: always
+    command: >
+      /bin/bash -c "
+      cp /tmp/opensearch_dashboards.template.yml /tmp/opensearch_dashboards.yml &&
+      sed -i 's|OPENSEARCH_HOSTS|${OPENSEARCH_PROTOCOL}://${OPENSEARCH_HOST}:${OPENSEARCH_PORT}|g' /tmp/opensearch_dashboards.yml &&
+      sed -i 's|OPENSEARCH_USER|${OPENSEARCH_USER}|g' /tmp/opensearch_dashboards.yml &&
+      sed -i 's|OPENSEARCH_PASSWORD|${OPENSEARCH_PASSWORD}|g' /tmp/opensearch_dashboards.yml &&
+      cp /tmp/opensearch_dashboards.yml /usr/share/opensearch-dashboards/config/opensearch_dashboards.yml &&
+      cd /usr/share/opensearch-dashboards &&
+      exec ./opensearch-dashboards-docker-entrypoint.sh opensearch-dashboards"
+    environment:
+      - OPENSEARCH_DASHBOARD_PORT=${OPENSEARCH_DASHBOARDS_PORT}
+    volumes:
+      - ./docker-compose/opensearch-dashboards/opensearch_dashboards.template.yml:/tmp/opensearch_dashboards.template.yml
+    ports:
+      # Web UI endpoint
+      - "${OPENSEARCH_DASHBOARDS_PORT}:5601"
+    networks:
+      - observability-stack-network
+    restart: unless-stopped
+    deploy:
+      resources:
+        limits:
+          memory: ${DASHBOARDS_MEMORY_LIMIT}
+    healthcheck:
+      test: curl -f -u '${OPENSEARCH_USER}':'${OPENSEARCH_PASSWORD}' http://localhost:5601/api/status || exit 1
+      start_period: 60s
+      interval: 10s
+      timeout: 5s
+      retries: 12
+    logging: *logging

docker-compose.local-opensearch.yml

@@ -0,0 +1,72 @@
+# Local OpenSearch - include this file when running OpenSearch locally instead of cloud
+# Included by default via INCLUDE_COMPOSE_LOCAL_OPENSEARCH in .env
+# To use a cloud OpenSearch instance, comment out INCLUDE_COMPOSE_LOCAL_OPENSEARCH in .env
+
+x-default-logging: &logging
+  driver: "json-file"
+  options:
+    max-size: "5m"
+    max-file: "2"
+    tag: "{{.Name}}"
+
+volumes:
+  opensearch-data:
+    driver: local
+
+services:
+  # OpenSearch - Stores and indexes logs and traces for search and analysis
+  opensearch:
+    image: ${OPENSEARCH_DOCKER_REPO}/opensearch:${OPENSEARCH_VERSION}
+    container_name: opensearch
+    pull_policy: always
+    environment:
+      # Single-node cluster for development
+      - cluster.name=observability-stack-cluster
+      - node.name=observability-stack-node
+      - discovery.type=single-node
+      - bootstrap.memory_lock=true
+      # Set heap size (adjust based on available memory)
+      - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS}
+      # Initial admin password (required for OpenSearch 2.12+)
+      - "OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD}"
+      - plugins.query.datasources.encryption.masterkey=BTqK4Ytdz67La1kShIKV3Pu9
+    volumes:
+      # Persist data across container restarts
+      - opensearch-data:/usr/share/opensearch/data
+    ports:
+      # REST API endpoint
+      - "${OPENSEARCH_PORT}:9200"
+      # Performance analyzer
+      - "9600:9600"
+    networks:
+      - observability-stack-network
+    restart: unless-stopped
+    deploy:
+      resources:
+        limits:
+          memory: ${OPENSEARCH_MEMORY_LIMIT}
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    healthcheck:
+      test: curl -s -k -u '${OPENSEARCH_USER}':'${OPENSEARCH_PASSWORD}' ${OPENSEARCH_PROTOCOL}://localhost:9200/_cluster/health | grep -E '"status":"(green|yellow)"'
+      start_period: 120s
+      interval: 5s
+      timeout: 10s
+      retries: 30
+    logging: *logging
+
+  # Add depends_on opensearch to services that need it for local startup ordering
+  otel-collector:
+    depends_on:
+      opensearch:
+        condition: service_healthy
+
+  data-prepper:
+    depends_on:
+      opensearch:
+        condition: service_healthy

docker-compose.yml

@@ -5,6 +5,8 @@
 include:
   - path: ${INCLUDE_COMPOSE_EXAMPLES:-docker-compose/util/docker-compose.empty.yml}
   - path: ${INCLUDE_COMPOSE_OTEL_DEMO:-docker-compose/util/docker-compose.empty.yml}
+  - path: ${INCLUDE_COMPOSE_LOCAL_OPENSEARCH:-docker-compose/util/docker-compose.empty.yml}
+  - path: ${INCLUDE_COMPOSE_LOCAL_OPENSEARCH_DASHBOARDS:-docker-compose/util/docker-compose.empty.yml}
 
 x-default-logging: &logging
   driver: "json-file"
@@ -19,8 +21,6 @@ networks:
     driver: bridge
 
 volumes:
-  opensearch-data:
-    driver: local
   prometheus-data:
     driver: local
 
@@ -54,9 +54,6 @@ services:
       - OPENSEARCH_HOST=${OPENSEARCH_HOST}
       - OPENSEARCH_PORT=${OPENSEARCH_PORT}
       - GOMEMLIMIT=160MiB
-    depends_on:
-      opensearch:
-        condition: service_healthy
     logging: *logging
 
   # Data Prepper - Transforms and enriches logs/traces before OpenSearch ingestion
@@ -71,6 +68,9 @@ services:
       chmod +w /tmp/pipelines.yaml &&
       sed -i 's|OPENSEARCH_USER|${OPENSEARCH_USER}|g' /tmp/pipelines.yaml &&
       sed -i 's|OPENSEARCH_PASSWORD|${OPENSEARCH_PASSWORD}|g' /tmp/pipelines.yaml &&
+      sed -i 's|OPENSEARCH_PROTOCOL|${OPENSEARCH_PROTOCOL}|g' /tmp/pipelines.yaml &&
+      sed -i 's|OPENSEARCH_HOST|${OPENSEARCH_HOST}|g' /tmp/pipelines.yaml &&
+      sed -i 's|OPENSEARCH_PORT|${OPENSEARCH_PORT}|g' /tmp/pipelines.yaml &&
       mv /tmp/pipelines.yaml /usr/share/data-prepper/pipelines/pipelines.yaml &&
       exec /usr/share/data-prepper/bin/data-prepper"
     volumes:
@@ -86,9 +86,6 @@ services:
       - OPENSEARCH_PORT=${OPENSEARCH_PORT}
       - OPENSEARCH_USER=${OPENSEARCH_USER}
       - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
-    depends_on:
-      opensearch:
-        condition: service_healthy
     networks:
       - observability-stack-network
     restart: unless-stopped
@@ -98,52 +95,6 @@ services:
           memory: ${DATA_PREPPER_MEMORY_LIMIT}
     logging: *logging
 
-  # OpenSearch - Stores and indexes logs and traces for search and analysis
-  opensearch:
-    image: ${OPENSEARCH_DOCKER_REPO}/opensearch:${OPENSEARCH_VERSION}
-    container_name: opensearch
-    pull_policy: always
-    environment:
-      # Single-node cluster for development
-      - cluster.name=observability-stack-cluster
-      - node.name=observability-stack-node
-      - discovery.type=single-node
-      - bootstrap.memory_lock=true
-      # Set heap size (adjust based on available memory)
-      - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS}
-      # Initial admin password (required for OpenSearch 2.12+)
-      - "OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD}"
-      - plugins.query.datasources.encryption.masterkey=BTqK4Ytdz67La1kShIKV3Pu9
-    volumes:
-      # Persist data across container restarts
-      - opensearch-data:/usr/share/opensearch/data
-    ports:
-      # REST API endpoint
-      - "${OPENSEARCH_PORT}:9200"
-      # Performance analyzer
-      - "9600:9600"
-    networks:
-      - observability-stack-network
-    restart: unless-stopped
-    deploy:
-      resources:
-        limits:
-          memory: ${OPENSEARCH_MEMORY_LIMIT}
-    ulimits:
-      memlock:
-        soft: -1
-        hard: -1
-      nofile:
-        soft: 65536
-        hard: 65536
-    healthcheck:
-      test: curl -s -k -u '${OPENSEARCH_USER}':'${OPENSEARCH_PASSWORD}' https://localhost:9200/_cluster/health | grep -E '"status":"(green|yellow)"'
-      start_period: 120s
-      interval: 5s
-      timeout: 10s
-      retries: 30
-    logging: *logging
-
   # Prometheus - Time-series database for metrics storage
   prometheus:
     image: prom/prometheus:${PROMETHEUS_VERSION}
@@ -178,49 +129,20 @@ services:
           memory: ${PROMETHEUS_MEMORY_LIMIT}
     logging: *logging
 
-  # OpenSearch Dashboards - Web UI for visualizing logs and traces
-  opensearch-dashboards:
-    image: ${OPENSEARCH_DOCKER_REPO}/opensearch-dashboards:${OPENSEARCH_DASHBOARDS_VERSION}
-    container_name: opensearch-dashboards
-    pull_policy: always
-    environment:
-      - OPENSEARCH_HOSTS=["https://${OPENSEARCH_HOST}:${OPENSEARCH_PORT}"]
-      - OPENSEARCH_DASHBOARD_PORT=${OPENSEARCH_DASHBOARDS_PORT}
-      - OPENSEARCH_USER=${OPENSEARCH_USER}
-      - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
-    volumes:
-      - ./docker-compose/opensearch-dashboards/opensearch_dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml
-    ports:
-      # Web UI endpoint
-      - "${OPENSEARCH_DASHBOARDS_PORT}:5601"
-    depends_on:
-      opensearch:
-        condition: service_healthy
-    networks:
-      - observability-stack-network
-    restart: unless-stopped
-    deploy:
-      resources:
-        limits:
-          memory: ${DASHBOARDS_MEMORY_LIMIT}
-    healthcheck:
-      test: curl -f -u '${OPENSEARCH_USER}':'${OPENSEARCH_PASSWORD}' http://localhost:5601/api/status || exit 1
-      start_period: 60s
-      interval: 10s
-      timeout: 5s
-      retries: 12
-    logging: *logging
-
-  # OpenSearch Dashboards Initialization - Creates workspace and index patterns
+  # OpenSearch Dashboards Initialization - Creates workspace, index patterns, and saved queries
   opensearch-dashboards-init:
     image: python:3.11-alpine
     container_name: opensearch-dashboards-init
     command: sh -c "pip install requests pyyaml && python /init.py"
-    depends_on:
-      opensearch-dashboards:
-        condition: service_healthy
     environment:
+      - OPENSEARCH_USER=${OPENSEARCH_USER}
       - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
+      - OPENSEARCH_HOST=${OPENSEARCH_HOST}
+      - OPENSEARCH_PORT=${OPENSEARCH_PORT}
+      - OPENSEARCH_PROTOCOL=${OPENSEARCH_PROTOCOL}
+      - OPENSEARCH_DASHBOARDS_HOST=${OPENSEARCH_DASHBOARDS_HOST}
+      - OPENSEARCH_DASHBOARDS_PORT=${OPENSEARCH_DASHBOARDS_PORT}
+      - OPENSEARCH_DASHBOARDS_PROTOCOL=${OPENSEARCH_DASHBOARDS_PROTOCOL}
       - PROMETHEUS_HOST=${PROMETHEUS_HOST}
       - PROMETHEUS_PORT=${PROMETHEUS_PORT}
     volumes:
@@ -232,3 +154,4 @@ services:
       - observability-stack-network
     restart: "no"
     logging: *logging
+