Add host-side compose TDX E2E suite

Author: kvinwang

dstack/gateway/dstack-app/bootstrap-cluster.sh

@@ -33,7 +33,7 @@ echo "Waiting for gateway admin API at $ADMIN_ADDR..."
 max_retries=60
 retry=0
 while [ $retry -lt $max_retries ]; do
-  if curl -sf "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/Status" >/dev/null 2>&1; then
+  if curl -sf "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/Admin.Status" >/dev/null 2>&1; then
     break
   fi
   retry=$((retry + 1))
@@ -48,29 +48,45 @@ fi
 
 echo "Admin API ready, bootstrapping configuration..."
 
-# Set ACME URL
-if [ "$ACME_STAGING" = "yes" ]; then
+# Set ACME URL. ACME_URL is useful for local E2E suites (for example Pebble).
+if [ -n "${ACME_URL:-}" ]; then
+  :
+elif [ "$ACME_STAGING" = "yes" ]; then
   ACME_URL="https://acme-staging-v02.api.letsencrypt.org/directory"
 else
   ACME_URL="https://acme-v02.api.letsencrypt.org/directory"
 fi
 
 echo "Setting certbot config (ACME URL: $ACME_URL)..."
-curl -sf -X POST "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/SetCertbotConfig" \
+curl -sf -X POST "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/Admin.SetCertbotConfig" \
   -H "Content-Type: application/json" \
   -d '{"acme_url":"'"$ACME_URL"'","renew_interval_secs":3600,"renew_before_expiration_secs":864000,"renew_timeout_secs":300}' >/dev/null \
   && echo "  Certbot config set" || echo "  WARN: failed to set certbot config"
 
 # Create DNS credential if CF_API_TOKEN is provided and no credentials exist yet
 if [ -n "$CF_API_TOKEN" ]; then
-  existing=$(curl -sf "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/ListDnsCredentials" 2>/dev/null)
+  existing=$(curl -sf "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/Admin.ListDnsCredentials" 2>/dev/null)
   cred_count=$(echo "$existing" | jq -r '.credentials | length' 2>/dev/null || echo "0")
 
   if [ "$cred_count" = "0" ]; then
     echo "Creating default DNS credential..."
-    curl -sf -X POST "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/CreateDnsCredential" \
+    dns_payload=$(jq -cn \
+      --arg token "$CF_API_TOKEN" \
+      --arg api_url "${CF_API_URL:-}" \
+      --argjson ttl "${DNS_TXT_TTL:-60}" \
+      --argjson max_wait "${MAX_DNS_WAIT:-300}" \
+      '{
+        name: "cloudflare",
+        provider_type: "cloudflare",
+        cf_api_token: $token,
+        set_as_default: true,
+        dns_txt_ttl: $ttl,
+        max_dns_wait: $max_wait
+      }
+      + (if $api_url == "" then {} else {cf_api_url: $api_url} end)')
+    curl -sf -X POST "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/Admin.CreateDnsCredential" \
       -H "Content-Type: application/json" \
-      -d '{"name":"cloudflare","provider_type":"cloudflare","cf_api_token":"'"$CF_API_TOKEN"'","set_as_default":true}' >/dev/null \
+      -d "$dns_payload" >/dev/null \
       && echo "  DNS credential created" || echo "  WARN: failed to create DNS credential"
   else
     echo "  DNS credentials already exist ($cred_count), skipping"
@@ -81,12 +97,12 @@ fi
 
 # Add ZT-Domain if SRV_DOMAIN is provided and domain doesn't exist yet
 if [ -n "$SRV_DOMAIN" ]; then
-  existing=$(curl -sf "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/ListZtDomains" 2>/dev/null)
+  existing=$(curl -sf "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/Admin.ListZtDomains" 2>/dev/null)
   has_domain=$(echo "$existing" | jq -r '.domains[]? | select(.domain=="'"$SRV_DOMAIN"'") | .domain' 2>/dev/null)
 
   if [ -z "$has_domain" ]; then
     echo "Adding ZT-Domain: $SRV_DOMAIN..."
-    curl -sf -X POST "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/AddZtDomain" \
+    curl -sf -X POST "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/Admin.AddZtDomain" \
       -H "Content-Type: application/json" \
       -d '{"domain":"'"$SRV_DOMAIN"'","port":443,"priority":100}' >/dev/null \
       && echo "  ZT-Domain added" || echo "  WARN: failed to add ZT-Domain"

dstack/gateway/dstack-app/deploy-to-vmm.sh

@@ -64,6 +64,12 @@ NODE_ID=1
 
 # Whether to use ACME staging (yes/no)
 ACME_STAGING=no
+# Optional ACME directory override for local E2E (for example Pebble).
+# ACME_URL=http://10.0.2.2:14000/dir
+# Optional Cloudflare API override for local E2E mock.
+# CF_API_URL=http://10.0.2.2:18080/client/v4
+# DNS_TXT_TTL=60
+# MAX_DNS_WAIT=300
 
 # Networking mode: bridge or user (default: user)
 # NET_MODE=bridge

dstack/key-provider-build/Dockerfile.key-provider

@@ -59,9 +59,20 @@ RUN git clone https://github.com/MoeMahhouk/gramine-sealing-key-provider && \
 WORKDIR /gramine-sealing-key-provider
 COPY Cargo.lock .
 
-# Build key provider, generate keys, and build manifest
+# Build key provider, generate keys, and build manifest.
+#
+# Do not use `gramine-sgx-gen-private-key` here. On some hosts the Python
+# cryptography/OpenSSL stack in the Gramine image fails while generating the
+# RSA key with a generic "Unknown OpenSSL error". Generate the exact same kind
+# of Gramine signing key directly with OpenSSL instead:
+#   - 3072-bit RSA
+#   - public exponent 3
+#   - traditional PEM accepted by gramine-sgx-sign
 RUN make target/release/gramine-sealing-key-provider && \
-    gramine-sgx-gen-private-key && \
+    mkdir -p /root/.config/gramine && \
+    chmod 0700 /root/.config /root/.config/gramine && \
+    openssl genrsa -traditional -3 -out /root/.config/gramine/enclave-key.pem 3072 && \
+    chmod 0600 /root/.config/gramine/enclave-key.pem && \
     make RUST_LOG=info
 
 RUN gramine-sgx-sigstruct-view --output-format json gramine-sealing-key-provider.sig

test-suites/full-stack-compose/.env.example

@@ -0,0 +1,50 @@
+# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network>
+# SPDX-License-Identifier: Apache-2.0
+
+# Directory containing unpacked dstack guest images: <store>/<image-name>/{bzImage,initramfs...,rootfs...,digest.txt}
+# In meta-dstack this is usually ../../../build/images from this directory, or an absolute path.
+DSTACK_E2E_IMAGE_STORE=../../../build/images
+DSTACK_E2E_IMAGE_NAME=dstack-0.6.0
+DSTACK_E2E_PLATFORM=tdx
+
+# App image used inside the only CVM launched by this suite.
+DSTACK_E2E_APP_IMAGE=nginx:alpine
+
+# Mock ACME/DNS + externally visible test domain for Gateway SNI.
+DSTACK_E2E_BASE_DOMAIN=e2e.test
+DSTACK_E2E_PEBBLE_IMAGE=kvin/pebble:latest
+
+# Host ports. Defaults are chosen to avoid the common dstackup ports.
+DSTACK_E2E_VMM_PORT=29080
+DSTACK_E2E_AUTH_PORT=28011
+DSTACK_E2E_KMS_HOST_PORT=28082
+DSTACK_E2E_GATEWAY_RPC_HOST_PORT=28000
+DSTACK_E2E_GATEWAY_ADMIN_HOST_PORT=28001
+DSTACK_E2E_GATEWAY_PROXY_HOST_PORT=28443
+DSTACK_E2E_GATEWAY_WG_HOST_PORT=28120
+DSTACK_E2E_KEY_PROVIDER_PORT=13443
+DSTACK_E2E_HOST_API_PORT=20011
+DSTACK_E2E_MOCK_CF_HTTP_PORT=38080
+DSTACK_E2E_PEBBLE_HTTP_PORT=34000
+DSTACK_E2E_PEBBLE_MGMT_PORT=35000
+
+# Dev-compose gateway policy. "any" matches the tdxlab host-side gateway mode:
+# app CVMs skip gateway app-id pinning but still use KMS authorization for apps.
+DSTACK_E2E_GATEWAY_APP_ID=any
+DSTACK_E2E_GATEWAY_WG_INTERFACE=wg-ds-e2e
+DSTACK_E2E_GATEWAY_WG_IP=10.8.0.1/16
+DSTACK_E2E_GATEWAY_WG_RESERVED_NET=10.8.0.1/32
+DSTACK_E2E_GATEWAY_WG_CLIENT_RANGE=10.8.0.0/18
+
+# VMM/QEMU knobs.
+DSTACK_E2E_CID_START=15000
+DSTACK_E2E_QGS_PORT=4050
+DSTACK_E2E_QEMU_PATH=/usr/bin/qemu-system-x86_64
+
+# SGX QCNL config used by AESMD. On hosts with a local PCCS, point this at the
+# host config, e.g. /etc/sgx_default_qcnl.conf.
+DSTACK_E2E_QCNL_CONF=../../dstack/key-provider-build/sgx_default_qcnl.conf
+
+# Runner behaviour.
+DSTACK_E2E_CLEAN_START=true
+DSTACK_E2E_CLEANUP_AFTER=false

test-suites/full-stack-compose/.gitignore

@@ -0,0 +1,5 @@
+.env
+/state/*
+!/state/.gitkeep
+__pycache__/
+*.pyc

test-suites/full-stack-compose/README.md

@@ -0,0 +1,109 @@
+# dstack full-stack compose E2E suite
+
+This directory contains a Docker Compose based test suite that mirrors the
+`tdxlab`/`dstack-k8s deploy/compose` shape: host-side services run as normal
+processes/containers, and QEMU is used only for the real app CVM under test.
+
+```text
+docker compose
+  ├─ mock-cf-dns-api        # tiny Cloudflare API + TXT DNS mock
+  ├─ pebble                 # ACME test server (HTTP-capable custom Pebble image)
+  ├─ aesmd + local-keyprovider
+  ├─ dstack-auth            # KMS auth webhook backed by auth-allowlist.json
+  ├─ dstack-kms             # host-side dev KMS, not a CVM
+  ├─ dstack-gateway         # host-side dev Gateway, not a CVM
+  ├─ dstack-vmm             # launches real TDX/SNP app CVMs via QEMU
+  └─ runner
+       ├─ configures Gateway certbot to use mock CF + Pebble
+       ├─ deploys nginx test app CVM with kms_enabled=true + gateway_enabled=true
+       └─ verifies HTTPS Gateway -> app routing
+```
+
+It is intended for a **real TDX/SGX host** (or SEV-SNP host after setting the
+platform/image knobs). It is not a pure unit-test environment; QEMU still starts
+a confidential VM for the app.
+
+## Prerequisites
+
+1. From this directory, build the host binaries from the dstack checkout:
+
+   ```bash
+   cargo build --release \
+     --manifest-path ../../dstack/Cargo.toml \
+     -p dstack-cli -p dstack-auth -p dstack-vmm -p dstack-kms \
+     -p dstack-gateway -p supervisor
+   ```
+
+2. Make sure the host has the required devices/services:
+
+   - `/dev/kvm`
+   - Intel TDX support and QGS for TDX (`DSTACK_E2E_QGS_PORT`, default `4050`)
+   - `/dev/sgx_enclave` and `/dev/sgx_provision` for the local key provider
+   - an SGX QCNL config that works on the host. By default the suite uses
+     `../../dstack/key-provider-build/sgx_default_qcnl.conf`; on hosts with a local
+     PCCS, set `DSTACK_E2E_QCNL_CONF=/etc/sgx_default_qcnl.conf`.
+   - `/dev/vhost-vsock` for guest↔host vsock services
+   - a TDX-capable `qemu-system-x86_64` available inside the runtime container.
+     The default runtime image follows `../dstack-k8s/deploy/compose` and
+     installs QEMU from `ppa:kobuk-team/tdx-release`.
+   - WireGuard kernel support on the host. The Gateway service runs privileged
+     with host networking and creates `DSTACK_E2E_GATEWAY_WG_INTERFACE`.
+
+3. Provide an unpacked guest image store. From `meta-dstack` this is usually
+   `../../../build/images`; otherwise copy `.env.example` and set:
+
+   ```bash
+   cp .env.example .env
+   $EDITOR .env
+   ```
+
+No Gateway image publishing is needed in this architecture: `dstack-gateway` and
+`dstack-kms` are run from the local `target/release/` binaries on the host side.
+
+## Run
+
+From this directory, run:
+
+```bash
+DOCKER_BUILDKIT=0 docker compose --env-file .env -f compose.yml up --build --abort-on-container-exit runner
+```
+
+For iterative debugging, keep the stack running:
+
+```bash
+DOCKER_BUILDKIT=0 docker compose --env-file .env -f compose.yml up --build
+```
+
+The runner leaves the app VM running by default for inspection. Set
+`DSTACK_E2E_CLEANUP_AFTER=true` to remove suite VMs at the end. Stale VMs whose
+names start with `DSTACK_E2E_NAME_PREFIX` are removed at the start when
+`DSTACK_E2E_CLEAN_START=true`.
+
+## What is verified
+
+The runner checks:
+
+1. Host-side KMS is reachable over TLS.
+2. Host-side Gateway Admin API accepts configuration.
+3. Gateway obtains a wildcard certificate for `*.DSTACK_E2E_BASE_DOMAIN` from
+   Pebble using the mock Cloudflare DNS API.
+4. VMM API becomes reachable.
+5. A real nginx app CVM boots with `kms_enabled=true` and `gateway_enabled=true`.
+6. `curl -k` through the Gateway SNI route returns the nginx welcome page.
+
+Artifacts are written under `state/work/`.
+
+## Notes / flexibility gaps
+
+- This is a dev compose KMS/Gateway setup. KMS and Gateway key material is
+  generated under `state/`; the allowlist returns `gatewayAppId = "any"`, which
+  matches the host-side tdxlab-style deployment and avoids requiring Gateway to
+  be a CVM.
+- `dstackup install` is systemd-oriented; this suite renders `vmm.toml`,
+  `kms.toml`, `gateway.toml`, and `auth-allowlist.json` itself.
+- Running VMM in a container depends on the container image having a QEMU build
+  that supports the target TEE platform. The runtime Dockerfile uses Intel's
+  kobuk-team TDX PPA for QEMU, matching the dstack-k8s compose deployment.
+- Host API is vsock-only by design. The suite therefore uses privileged host
+  networking and a separate `DSTACK_E2E_HOST_API_PORT` to avoid conflicts with
+  other VMM instances.