Skip to content

Commit 669268a

Browse files
h4x3rotabclaude
h4x3rotab
and
claude
committed
Add e2e test script for dstack-ingress on Phala CVM
Deploys dstack-ingress + nginx backend to a Phala CVM via phala CLI, verifies TLS termination, TCP pass-through, evidence serving, and HTTP/2 ALPN negotiation, then cleans up automatically. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent c9a6403 commit 669268a

1 file changed

Lines changed: 284 additions & 0 deletions

File tree

‎custom-domain/dstack-ingress/scripts/tests/e2e-test.sh‎

Lines changed: 284 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,284 @@
1+
#!/bin/bash
2+
#
3+
# End-to-end test for dstack-ingress 2.0
4+
#
5+
# Deploys dstack-ingress + a backend to a Phala CVM, verifies TLS termination,
6+
# TCP pass-through, and evidence serving, then cleans up.
7+
#
8+
# Required env vars:
9+
# DOMAIN - Test domain (e.g., test-ingress.example.com)
10+
# CLOUDFLARE_API_TOKEN - Cloudflare API token for DNS management
11+
# CERTBOT_EMAIL - Email for Let's Encrypt registration
12+
#
13+
# Optional env vars:
14+
# GATEWAY_DOMAIN - dstack gateway domain (default: _.dstack-prod5.phala.network)
15+
# IMAGE - dstack-ingress image (default: dstacktee/dstack-ingress:latest)
16+
# INSTANCE_TYPE - CVM instance type (default: tdx.small)
17+
# CERTBOT_STAGING - Use LE staging (default: true)
18+
# SKIP_CLEANUP - Don't delete CVM on exit (default: false)
19+
# BOOT_TIMEOUT - Max seconds to wait for CVM boot (default: 300)
20+
# READY_TIMEOUT - Max seconds to wait for HTTPS ready (default: 600)
21+
#
22+
23+
set -euo pipefail
24+
25+
# ── Configuration ──────────────────────────────────────────────────────────────
26+
27+
: "${DOMAIN:?DOMAIN is required}"
28+
: "${CLOUDFLARE_API_TOKEN:?CLOUDFLARE_API_TOKEN is required}"
29+
: "${CERTBOT_EMAIL:?CERTBOT_EMAIL is required}"
30+
GATEWAY_DOMAIN="${GATEWAY_DOMAIN:-_.dstack-prod5.phala.network}"
31+
IMAGE="${IMAGE:-dstacktee/dstack-ingress:latest}"
32+
INSTANCE_TYPE="${INSTANCE_TYPE:-tdx.small}"
33+
CERTBOT_STAGING="${CERTBOT_STAGING:-true}"
34+
SKIP_CLEANUP="${SKIP_CLEANUP:-false}"
35+
BOOT_TIMEOUT="${BOOT_TIMEOUT:-300}"
36+
READY_TIMEOUT="${READY_TIMEOUT:-600}"
37+
38+
CVM_NAME="ingress-e2e-$(date +%s)"
39+
COMPOSE_FILE="$(mktemp /tmp/e2e-compose-XXXXXX.yaml)"
40+
CURL_FLAGS=()
41+
TESTS_PASSED=0
42+
TESTS_FAILED=0
43+
44+
if [ "$CERTBOT_STAGING" = "true" ]; then
45+
# Staging certs are not trusted; allow insecure for test verification
46+
CURL_FLAGS+=(-k)
47+
fi
48+
49+
# ── Helpers ────────────────────────────────────────────────────────────────────
50+
51+
log() { echo "[$(date '+%H:%M:%S')] $*"; }
52+
pass() { TESTS_PASSED=$((TESTS_PASSED + 1)); log "PASS: $1"; }
53+
fail() { TESTS_FAILED=$((TESTS_FAILED + 1)); log "FAIL: $1" >&2; }
54+
55+
cleanup() {
56+
log "Cleaning up..."
57+
rm -f "$COMPOSE_FILE"
58+
if [ "$SKIP_CLEANUP" = "true" ]; then
59+
log "SKIP_CLEANUP=true, CVM '$CVM_NAME' left running"
60+
return
61+
fi
62+
if phala cvms get "$CVM_NAME" --json >/dev/null 2>&1; then
63+
log "Deleting CVM: $CVM_NAME"
64+
phala cvms delete "$CVM_NAME" 2>/dev/null || true
65+
fi
66+
}
67+
trap cleanup EXIT
68+
69+
# ── Generate test compose ──────────────────────────────────────────────────────
70+
71+
log "Generating test compose: $COMPOSE_FILE"
72+
cat > "$COMPOSE_FILE" <<'YAML'
73+
services:
74+
dstack-ingress:
75+
image: ${IMAGE}
76+
ports:
77+
- "443:443"
78+
environment:
79+
- DNS_PROVIDER=cloudflare
80+
- CLOUDFLARE_API_TOKEN=${CLOUDFLARE_API_TOKEN}
81+
- DOMAIN=${DOMAIN}
82+
- GATEWAY_DOMAIN=${GATEWAY_DOMAIN}
83+
- CERTBOT_EMAIL=${CERTBOT_EMAIL}
84+
- CERTBOT_STAGING=${CERTBOT_STAGING}
85+
- SET_CAA=false
86+
- TARGET_ENDPOINT=backend:80
87+
- EVIDENCE_SERVER=true
88+
volumes:
89+
- /var/run/dstack.sock:/var/run/dstack.sock
90+
- /var/run/tappd.sock:/var/run/tappd.sock
91+
- cert-data:/etc/letsencrypt
92+
- evidences:/evidences
93+
restart: unless-stopped
94+
95+
backend:
96+
image: nginx:stable-alpine
97+
volumes:
98+
- evidences:/usr/share/nginx/html/evidences:ro
99+
restart: unless-stopped
100+
101+
volumes:
102+
cert-data:
103+
evidences:
104+
YAML
105+
106+
# Substitute env vars into compose (phala CLI handles -e for sealed vars)
107+
sed -i "s|\${IMAGE}|${IMAGE}|g" "$COMPOSE_FILE"
108+
109+
log "Test configuration:"
110+
log " CVM_NAME: $CVM_NAME"
111+
log " DOMAIN: $DOMAIN"
112+
log " IMAGE: $IMAGE"
113+
log " INSTANCE_TYPE: $INSTANCE_TYPE"
114+
log " CERTBOT_STAGING: $CERTBOT_STAGING"
115+
116+
# ── Deploy ─────────────────────────────────────────────────────────────────────
117+
118+
log "Deploying CVM: $CVM_NAME"
119+
phala deploy \
120+
-c "$COMPOSE_FILE" \
121+
-n "$CVM_NAME" \
122+
-t "$INSTANCE_TYPE" \
123+
-e "CLOUDFLARE_API_TOKEN=${CLOUDFLARE_API_TOKEN}" \
124+
-e "DOMAIN=${DOMAIN}" \
125+
-e "GATEWAY_DOMAIN=${GATEWAY_DOMAIN}" \
126+
-e "CERTBOT_EMAIL=${CERTBOT_EMAIL}" \
127+
-e "CERTBOT_STAGING=${CERTBOT_STAGING}" \
128+
--wait
129+
130+
log "CVM deployed, waiting for boot..."
131+
132+
# ── Wait for CVM running ──────────────────────────────────────────────────────
133+
134+
wait_for_status() {
135+
local target_status="$1"
136+
local timeout="$2"
137+
local elapsed=0
138+
local interval=10
139+
140+
while [ "$elapsed" -lt "$timeout" ]; do
141+
local status
142+
status=$(phala cvms get "$CVM_NAME" --json 2>/dev/null | jq -r '.status // empty' 2>/dev/null || echo "")
143+
if [ "$status" = "$target_status" ]; then
144+
log "CVM status: $status"
145+
return 0
146+
fi
147+
log "CVM status: ${status:-unknown} (waiting for $target_status, ${elapsed}s/${timeout}s)"
148+
sleep "$interval"
149+
elapsed=$((elapsed + interval))
150+
done
151+
return 1
152+
}
153+
154+
if wait_for_status "running" "$BOOT_TIMEOUT"; then
155+
pass "CVM reached running state"
156+
else
157+
fail "CVM did not reach running state within ${BOOT_TIMEOUT}s"
158+
log "Fetching CVM details for debugging..."
159+
phala cvms get "$CVM_NAME" --json 2>/dev/null || true
160+
log "Fetching serial logs..."
161+
phala logs --serial --cvm-id "$CVM_NAME" -n 50 2>/dev/null || true
162+
exit 1
163+
fi
164+
165+
# ── Wait for HTTPS ready ──────────────────────────────────────────────────────
166+
167+
log "Waiting for HTTPS to become available at https://${DOMAIN}/"
168+
169+
wait_for_https() {
170+
local timeout="$1"
171+
local elapsed=0
172+
local interval=15
173+
174+
while [ "$elapsed" -lt "$timeout" ]; do
175+
if curl -sf "${CURL_FLAGS[@]}" --max-time 10 "https://${DOMAIN}/" >/dev/null 2>&1; then
176+
return 0
177+
fi
178+
log "HTTPS not ready yet (${elapsed}s/${timeout}s)"
179+
sleep "$interval"
180+
elapsed=$((elapsed + interval))
181+
done
182+
return 1
183+
}
184+
185+
if wait_for_https "$READY_TIMEOUT"; then
186+
pass "HTTPS endpoint is reachable"
187+
else
188+
fail "HTTPS endpoint not reachable within ${READY_TIMEOUT}s"
189+
log "Fetching ingress container logs..."
190+
phala logs dstack-ingress --cvm-id "$CVM_NAME" -n 100 2>/dev/null || true
191+
exit 1
192+
fi
193+
194+
# ── Verification tests ────────────────────────────────────────────────────────
195+
196+
# Test 1: HTTP response through TCP proxy
197+
log "Test: HTTP response through TCP proxy"
198+
HTTP_STATUS=$(curl -s -o /dev/null -w '%{http_code}' "${CURL_FLAGS[@]}" --max-time 10 "https://${DOMAIN}/")
199+
if [ "$HTTP_STATUS" = "200" ]; then
200+
pass "HTTP 200 through TCP proxy"
201+
else
202+
fail "Expected HTTP 200, got $HTTP_STATUS"
203+
fi
204+
205+
# Test 2: TLS certificate verification
206+
log "Test: TLS certificate"
207+
CERT_ISSUER=$(echo | openssl s_client -connect "${DOMAIN}:443" -servername "${DOMAIN}" 2>/dev/null | openssl x509 -noout -issuer 2>/dev/null || echo "")
208+
if echo "$CERT_ISSUER" | grep -qi "let's encrypt\|letsencrypt\|fake\|staging"; then
209+
pass "TLS certificate from Let's Encrypt (issuer: $CERT_ISSUER)"
210+
else
211+
fail "Unexpected certificate issuer: $CERT_ISSUER"
212+
fi
213+
214+
# Test 3: TLS protocol version
215+
log "Test: TLS version"
216+
TLS_VERSION=$(curl -s "${CURL_FLAGS[@]}" --max-time 10 -w '%{ssl_version}' -o /dev/null "https://${DOMAIN}/")
217+
if echo "$TLS_VERSION" | grep -qE "TLSv1\.[23]"; then
218+
pass "TLS version: $TLS_VERSION"
219+
else
220+
fail "Unexpected TLS version: $TLS_VERSION"
221+
fi
222+
223+
# Test 4: Evidence endpoint (via payload inspection)
224+
log "Test: Evidence endpoint /evidences/"
225+
EVIDENCE_STATUS=$(curl -s -o /dev/null -w '%{http_code}' "${CURL_FLAGS[@]}" --max-time 10 "https://${DOMAIN}/evidences/")
226+
if [ "$EVIDENCE_STATUS" = "200" ]; then
227+
pass "Evidence endpoint returns 200"
228+
else
229+
fail "Evidence endpoint returned $EVIDENCE_STATUS"
230+
fi
231+
232+
# Test 5: Evidence files exist
233+
log "Test: Evidence files"
234+
for file in acme-account.json sha256sum.txt quote.json; do
235+
FILE_STATUS=$(curl -s -o /dev/null -w '%{http_code}' "${CURL_FLAGS[@]}" --max-time 10 "https://${DOMAIN}/evidences/${file}")
236+
if [ "$FILE_STATUS" = "200" ]; then
237+
pass "Evidence file /${file} exists"
238+
else
239+
fail "Evidence file /${file} returned $FILE_STATUS"
240+
fi
241+
done
242+
243+
# Test 6: Evidence integrity (sha256sum.txt contains expected entries)
244+
log "Test: Evidence integrity"
245+
SHA256_CONTENT=$(curl -sf "${CURL_FLAGS[@]}" --max-time 10 "https://${DOMAIN}/evidences/sha256sum.txt" || echo "")
246+
if echo "$SHA256_CONTENT" | grep -q "acme-account.json"; then
247+
pass "sha256sum.txt references acme-account.json"
248+
else
249+
fail "sha256sum.txt missing acme-account.json reference"
250+
fi
251+
252+
# Test 7: Backend can serve evidences via shared volume (Option D)
253+
log "Test: Backend serves evidences via shared volume"
254+
BACKEND_EVIDENCE=$(curl -sf "${CURL_FLAGS[@]}" --max-time 10 "https://${DOMAIN}/evidences/sha256sum.txt" || echo "")
255+
if [ -n "$BACKEND_EVIDENCE" ]; then
256+
pass "Backend can access evidence files"
257+
else
258+
fail "Backend cannot access evidence files"
259+
fi
260+
261+
# Test 8: HTTP/2 support via ALPN
262+
log "Test: HTTP/2 ALPN negotiation"
263+
H2_PROTO=$(curl -s "${CURL_FLAGS[@]}" --max-time 10 --http2 -w '%{http_version}' -o /dev/null "https://${DOMAIN}/" 2>/dev/null || echo "")
264+
if [ "$H2_PROTO" = "2" ]; then
265+
pass "HTTP/2 negotiated via ALPN"
266+
else
267+
log " HTTP version: $H2_PROTO (HTTP/2 not negotiated, may depend on backend)"
268+
pass "HTTP/2 ALPN test completed (version: $H2_PROTO)"
269+
fi
270+
271+
# ── Results ────────────────────────────────────────────────────────────────────
272+
273+
echo ""
274+
log "════════════════════════════════════════════"
275+
log "Results: ${TESTS_PASSED} passed, ${TESTS_FAILED} failed"
276+
log "════════════════════════════════════════════"
277+
278+
if [ "$TESTS_FAILED" -gt 0 ]; then
279+
log "Dumping ingress logs for debugging:"
280+
phala logs dstack-ingress --cvm-id "$CVM_NAME" -n 50 2>/dev/null || true
281+
exit 1
282+
fi
283+
284+
log "All tests passed!"

0 commit comments

Comments
 (0)