Skip to content

Commit c18245f

Browse files
kylehounslowkylehounslow
authored
feat: add install.sh E2E test with shared health checks (opensearch-project#97)
1 parent 2fee983 commit c18245f

4 files changed

Lines changed: 144 additions & 87 deletions

File tree

‎.github/workflows/e2e.yml‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,28 @@ on:
99
workflow_dispatch:
1010

1111
jobs:
12-
e2e:
12+
e2e-compose:
1313
runs-on: ubuntu-latest
1414
timeout-minutes: 15
1515
steps:
1616
- uses: actions/checkout@v4
1717

18-
- name: Run E2E tests
18+
- name: Run E2E tests (docker compose)
1919
run: ./test/e2e.sh
2020

2121
- name: Dump logs on failure
2222
if: failure()
2323
run: docker compose logs --tail=50
24+
25+
e2e-install:
26+
runs-on: ubuntu-latest
27+
timeout-minutes: 15
28+
steps:
29+
- uses: actions/checkout@v4
30+
31+
- name: Run E2E tests (install.sh)
32+
run: ./test/e2e-install.sh
33+
34+
- name: Dump logs on failure
35+
if: failure()
36+
run: docker compose -f /tmp/observability-stack-e2e/docker-compose.yml logs --tail=50 2>/dev/null || true

‎test/checks.sh‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
#!/usr/bin/env bash
2+
# Shared health checks and trace verification for E2E tests.
3+
# Expects these variables to be set before sourcing:
4+
# OPENSEARCH_USER, OPENSEARCH_PASSWORD, OPENSEARCH_PORT,
5+
# OPENSEARCH_DASHBOARDS_PORT, OTEL_COLLECTOR_PORT_HTTP, PROMETHEUS_PORT
6+
set -euo pipefail
7+
8+
OPENSEARCH_URL="https://localhost:${OPENSEARCH_PORT}"
9+
CURL_OPTS=(-s -k -u "${OPENSEARCH_USER}:${OPENSEARCH_PASSWORD}")
10+
11+
run_checks() {
12+
echo "==> Checking OpenSearch cluster health..."
13+
health=$(curl "${CURL_OPTS[@]}" "$OPENSEARCH_URL/_cluster/health" | sed -n 's/.*"status":"\([^"]*\)".*/\1/p')
14+
if [[ "$health" == "red" ]]; then
15+
echo "FAIL: OpenSearch cluster health is red"
16+
exit 1
17+
fi
18+
echo " OpenSearch cluster health: $health"
19+
20+
echo "==> Checking OTel Collector is accepting OTLP..."
21+
otel_status=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:${OTEL_COLLECTOR_PORT_HTTP}/v1/traces" \
22+
-H "Content-Type: application/json" \
23+
-d '{"resourceSpans":[]}')
24+
if [[ "$otel_status" != "200" ]]; then
25+
echo "FAIL: OTel Collector OTLP HTTP endpoint returned $otel_status"
26+
exit 1
27+
fi
28+
echo " OTel Collector OTLP HTTP: OK"
29+
30+
echo "==> Checking Prometheus is up..."
31+
prom_status=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:${PROMETHEUS_PORT}/-/healthy")
32+
if [[ "$prom_status" != "200" ]]; then
33+
echo "FAIL: Prometheus health check returned $prom_status"
34+
exit 1
35+
fi
36+
echo " Prometheus: OK"
37+
38+
echo "==> Checking OpenSearch Dashboards is up..."
39+
dashboards_status=$(curl -s -o /dev/null -w "%{http_code}" -u "${OPENSEARCH_USER}:${OPENSEARCH_PASSWORD}" \
40+
"http://localhost:${OPENSEARCH_DASHBOARDS_PORT}/api/status")
41+
if [[ "$dashboards_status" != "200" ]]; then
42+
echo "FAIL: OpenSearch Dashboards returned $dashboards_status"
43+
exit 1
44+
fi
45+
echo " OpenSearch Dashboards: OK"
46+
47+
echo "==> Sending test trace through OTel Collector..."
48+
trace_response=$(curl -s -w "\n%{http_code}" "http://localhost:${OTEL_COLLECTOR_PORT_HTTP}/v1/traces" \
49+
-H "Content-Type: application/json" \
50+
-d '{
51+
"resourceSpans": [{
52+
"resource": {"attributes": [{"key": "service.name", "value": {"stringValue": "e2e-test"}}]},
53+
"scopeSpans": [{
54+
"spans": [{
55+
"traceId": "5b8efff798038103d269b633813fc60c",
56+
"spanId": "eee19b7ec3c1b174",
57+
"name": "e2e-test-span",
58+
"kind": 1,
59+
"startTimeUnixNano": "1000000000",
60+
"endTimeUnixNano": "2000000000",
61+
"status": {}
62+
}]
63+
}]
64+
}]
65+
}')
66+
trace_status=$(echo "$trace_response" | tail -1)
67+
if [[ "$trace_status" != "200" ]]; then
68+
echo "FAIL: Sending test trace returned $trace_status"
69+
exit 1
70+
fi
71+
echo " Test trace sent: OK"
72+
73+
echo "==> Verifying trace landed in OpenSearch..."
74+
TRACE_ID="5b8efff798038103d269b633813fc60c"
75+
MAX_RETRIES=90
76+
for i in $(seq 1 "$MAX_RETRIES"); do
77+
hits=$(curl "${CURL_OPTS[@]}" "$OPENSEARCH_URL/*span*,*trace*/_search" \
78+
-H "Content-Type: application/json" \
79+
-d "{\"query\":{\"bool\":{\"should\":[{\"term\":{\"traceId\":\"$TRACE_ID\"}},{\"term\":{\"traceID\":\"$TRACE_ID\"}}]}}}" \
80+
| sed -n 's/.*"total":{"value":\([0-9]*\).*/\1/p')
81+
if [[ "$hits" -gt 0 ]]; then
82+
echo " Trace found in OpenSearch after ${i}s"
83+
break
84+
fi
85+
if [[ "$i" -eq "$MAX_RETRIES" ]]; then
86+
echo "FAIL: Trace not found in OpenSearch after ${MAX_RETRIES}s"
87+
exit 1
88+
fi
89+
sleep 1
90+
done
91+
92+
echo ""
93+
echo "==> All E2E checks passed!"
94+
}

‎test/e2e-install.sh‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
5+
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
6+
INSTALL_DIR="/tmp/observability-stack-e2e"
7+
8+
rm -rf "$INSTALL_DIR"
9+
10+
cleanup() {
11+
echo "==> Tearing down..."
12+
if [ -d "$INSTALL_DIR" ]; then
13+
docker compose -f "$INSTALL_DIR/docker-compose.yml" --project-directory "$INSTALL_DIR" down -v --remove-orphans 2>/dev/null || true
14+
rm -rf "$INSTALL_DIR"
15+
fi
16+
}
17+
trap cleanup EXIT
18+
19+
echo "==> Running install.sh..."
20+
# Patch install.sh to read from stdin instead of /dev/tty (no TTY in CI)
21+
PATCHED_SCRIPT=$(mktemp)
22+
sed 's|< /dev/tty||g' "$PROJECT_DIR/install.sh" > "$PATCHED_SCRIPT"
23+
chmod +x "$PATCHED_SCRIPT"
24+
25+
# Feed answers: install dir, no examples, no otel demo, no custom creds
26+
printf '%s\n' "$INSTALL_DIR" "n" "n" "n" | bash "$PATCHED_SCRIPT" --skip-pull
27+
rm -f "$PATCHED_SCRIPT"
28+
29+
# Parse .env from the installed directory
30+
eval "$(grep -E '^(OPENSEARCH_USER|OPENSEARCH_PASSWORD|OPENSEARCH_PORT|OPENSEARCH_DASHBOARDS_PORT|OTEL_COLLECTOR_PORT_HTTP|PROMETHEUS_PORT)=' "$INSTALL_DIR/.env")"
31+
32+
source "$SCRIPT_DIR/checks.sh"
33+
run_checks

‎test/e2e.sh‎

Lines changed: 2 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -22,88 +22,5 @@ docker compose -f "$COMPOSE_FILE" --project-directory "$PROJECT_DIR" up -d --wai
2222
# Parse .env safely (don't source — some values aren't shell-safe)
2323
eval "$(grep -E '^(OPENSEARCH_USER|OPENSEARCH_PASSWORD|OPENSEARCH_PORT|OPENSEARCH_DASHBOARDS_PORT|OTEL_COLLECTOR_PORT_HTTP|PROMETHEUS_PORT)=' "$PROJECT_DIR/.env")"
2424

25-
OPENSEARCH_URL="https://localhost:${OPENSEARCH_PORT}"
26-
CURL_OPTS=(-s -k -u "${OPENSEARCH_USER}:${OPENSEARCH_PASSWORD}")
27-
28-
echo "==> Checking OpenSearch cluster health..."
29-
health=$(curl "${CURL_OPTS[@]}" "$OPENSEARCH_URL/_cluster/health" | sed -n 's/.*"status":"\([^"]*\)".*/\1/p')
30-
if [[ "$health" == "red" ]]; then
31-
echo "FAIL: OpenSearch cluster health is red"
32-
exit 1
33-
fi
34-
echo " OpenSearch cluster health: $health"
35-
36-
echo "==> Checking OTel Collector is accepting OTLP..."
37-
otel_status=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:${OTEL_COLLECTOR_PORT_HTTP}/v1/traces" \
38-
-H "Content-Type: application/json" \
39-
-d '{"resourceSpans":[]}')
40-
if [[ "$otel_status" != "200" ]]; then
41-
echo "FAIL: OTel Collector OTLP HTTP endpoint returned $otel_status"
42-
exit 1
43-
fi
44-
echo " OTel Collector OTLP HTTP: OK"
45-
46-
echo "==> Checking Prometheus is up..."
47-
prom_status=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:${PROMETHEUS_PORT}/-/healthy")
48-
if [[ "$prom_status" != "200" ]]; then
49-
echo "FAIL: Prometheus health check returned $prom_status"
50-
exit 1
51-
fi
52-
echo " Prometheus: OK"
53-
54-
echo "==> Checking OpenSearch Dashboards is up..."
55-
dashboards_status=$(curl -s -o /dev/null -w "%{http_code}" -u "${OPENSEARCH_USER}:${OPENSEARCH_PASSWORD}" \
56-
"http://localhost:${OPENSEARCH_DASHBOARDS_PORT}/api/status")
57-
if [[ "$dashboards_status" != "200" ]]; then
58-
echo "FAIL: OpenSearch Dashboards returned $dashboards_status"
59-
exit 1
60-
fi
61-
echo " OpenSearch Dashboards: OK"
62-
63-
echo "==> Sending test trace through OTel Collector..."
64-
trace_response=$(curl -s -w "\n%{http_code}" "http://localhost:${OTEL_COLLECTOR_PORT_HTTP}/v1/traces" \
65-
-H "Content-Type: application/json" \
66-
-d '{
67-
"resourceSpans": [{
68-
"resource": {"attributes": [{"key": "service.name", "value": {"stringValue": "e2e-test"}}]},
69-
"scopeSpans": [{
70-
"spans": [{
71-
"traceId": "5b8efff798038103d269b633813fc60c",
72-
"spanId": "eee19b7ec3c1b174",
73-
"name": "e2e-test-span",
74-
"kind": 1,
75-
"startTimeUnixNano": "1000000000",
76-
"endTimeUnixNano": "2000000000",
77-
"status": {}
78-
}]
79-
}]
80-
}]
81-
}')
82-
trace_status=$(echo "$trace_response" | tail -1)
83-
if [[ "$trace_status" != "200" ]]; then
84-
echo "FAIL: Sending test trace returned $trace_status"
85-
exit 1
86-
fi
87-
echo " Test trace sent: OK"
88-
89-
echo "==> Verifying trace landed in OpenSearch..."
90-
TRACE_ID="5b8efff798038103d269b633813fc60c"
91-
MAX_RETRIES=90
92-
for i in $(seq 1 "$MAX_RETRIES"); do
93-
hits=$(curl "${CURL_OPTS[@]}" "$OPENSEARCH_URL/*span*,*trace*/_search" \
94-
-H "Content-Type: application/json" \
95-
-d "{\"query\":{\"bool\":{\"should\":[{\"term\":{\"traceId\":\"$TRACE_ID\"}},{\"term\":{\"traceID\":\"$TRACE_ID\"}}]}}}" \
96-
| sed -n 's/.*"total":{"value":\([0-9]*\).*/\1/p')
97-
if [[ "$hits" -gt 0 ]]; then
98-
echo " Trace found in OpenSearch after ${i}s"
99-
break
100-
fi
101-
if [[ "$i" -eq "$MAX_RETRIES" ]]; then
102-
echo "FAIL: Trace not found in OpenSearch after ${MAX_RETRIES}s"
103-
exit 1
104-
fi
105-
sleep 1
106-
done
107-
108-
echo ""
109-
echo "==> All E2E checks passed!"
25+
source "$SCRIPT_DIR/checks.sh"
26+
run_checks

0 commit comments

Comments
 (0)