ci: run E2E on push to main, fix compose include conflict (opensearch-project#98)

Author: kylehounslow

.github/workflows/e2e.yml

@@ -1,6 +1,11 @@
 name: E2E Tests
 
 on:
+  push:
+    branches: [main]
+    paths-ignore:
+      - 'docs/**'
+      - '*.md'
   pull_request:
     branches: [main]
     paths-ignore:

docker-compose.local-opensearch.yml

@@ -59,14 +59,3 @@ services:
       timeout: 10s
       retries: 30
     logging: *logging
-
-  # Add depends_on opensearch to services that need it for local startup ordering
-  otel-collector:
-    depends_on:
-      opensearch:
-        condition: service_healthy
-
-  data-prepper:
-    depends_on:
-      opensearch:
-        condition: service_healthy

docker-compose.yml

@@ -54,6 +54,10 @@ services:
       - OPENSEARCH_HOST=${OPENSEARCH_HOST}
       - OPENSEARCH_PORT=${OPENSEARCH_PORT}
       - GOMEMLIMIT=160MiB
+    depends_on:
+      opensearch:
+        condition: service_healthy
+        required: false
     logging: *logging
 
   # Data Prepper - Transforms and enriches logs/traces before OpenSearch ingestion
@@ -86,6 +90,10 @@ services:
       - OPENSEARCH_PORT=${OPENSEARCH_PORT}
       - OPENSEARCH_USER=${OPENSEARCH_USER}
       - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
+    depends_on:
+      opensearch:
+        condition: service_healthy
+        required: false
     networks:
       - observability-stack-network
     restart: unless-stopped

test/e2e-install.sh

@@ -18,8 +18,11 @@ trap cleanup EXIT
 
 echo "==> Running install.sh..."
 # Patch install.sh to read from stdin instead of /dev/tty (no TTY in CI)
+# and replace git clone with a local copy so we test the PR's code
 PATCHED_SCRIPT=$(mktemp)
-sed 's|< /dev/tty||g' "$PROJECT_DIR/install.sh" > "$PATCHED_SCRIPT"
+sed -e 's|< /dev/tty||g' \
+    -e 's|git clone --depth 1 "$REPO_URL" "$INSTALL_DIR" >/dev/null 2>\&1|cp -r "'"$PROJECT_DIR"'/." "$INSTALL_DIR"|' \
+    "$PROJECT_DIR/install.sh" > "$PATCHED_SCRIPT"
 chmod +x "$PATCHED_SCRIPT"
 
 # Feed answers: install dir, no examples, no otel demo, no custom creds

test/e2e.sh

@@ -3,21 +3,16 @@ set -euo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
-COMPOSE_FILE="$PROJECT_DIR/docker-compose.yml"
 WAIT_TIMEOUT="${WAIT_TIMEOUT:-180}"
 
-# Override to only run core stack (no examples/demo)
-export INCLUDE_COMPOSE_EXAMPLES=docker-compose/util/docker-compose.empty.yml
-export INCLUDE_COMPOSE_OTEL_DEMO=docker-compose/util/docker-compose.empty.yml
-
 cleanup() {
   echo "==> Tearing down..."
-  docker compose -f "$COMPOSE_FILE" --project-directory "$PROJECT_DIR" down -v --remove-orphans 2>/dev/null || true
+  docker compose --project-directory "$PROJECT_DIR" down -v --remove-orphans 2>/dev/null || true
 }
 trap cleanup EXIT
 
 echo "==> Starting observability stack..."
-docker compose -f "$COMPOSE_FILE" --project-directory "$PROJECT_DIR" up -d --wait --wait-timeout "$WAIT_TIMEOUT"
+docker compose --project-directory "$PROJECT_DIR" up -d --wait --wait-timeout "$WAIT_TIMEOUT"
 
 # Parse .env safely (don't source — some values aren't shell-safe)
 eval "$(grep -E '^(OPENSEARCH_USER|OPENSEARCH_PASSWORD|OPENSEARCH_PORT|OPENSEARCH_DASHBOARDS_PORT|OTEL_COLLECTOR_PORT_HTTP|PROMETHEUS_PORT)=' "$PROJECT_DIR/.env")"