Merge remote-tracking branch 'origin/master' into e2e-host-kms-gateway-compose

# Conflicts:
#	dstack/local-key-provider/build/Dockerfile.key-provider

Author: kvinwang

.github/scripts/cargo-publish-idempotent.sh

@@ -11,7 +11,7 @@ set -euo pipefail
 
 crate=${1:?missing crate name}
 
-if output=$(cargo publish --manifest-path dstack/Cargo.toml -p "$crate" 2>&1); then
+if output=$(cargo publish --manifest-path sdk/rust/Cargo.toml -p "$crate" 2>&1); then
     echo "$output"
     exit 0
 fi

.github/workflows/local-key-provider-release.yml

@@ -0,0 +1,109 @@
+# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+name: Local Key Provider Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: Release version without the local-key-provider-v prefix
+        required: true
+        type: string
+  push:
+    tags:
+      - 'local-key-provider-v*'
+
+permissions:
+  attestations: write
+  id-token: write
+  contents: write
+  packages: write
+
+jobs:
+  build-and-release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+
+      - name: Parse version from tag
+        env:
+          DISPATCH_VERSION: ${{ inputs.version }}
+        run: |
+          if [[ "$GITHUB_REF" == refs/tags/local-key-provider-v* ]]; then
+            VERSION=${GITHUB_REF#refs/tags/local-key-provider-v}
+          else
+            VERSION=$DISPATCH_VERSION
+          fi
+          echo "VERSION=$VERSION" >> "$GITHUB_ENV"
+          echo "Parsed version: $VERSION"
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v4
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Build and push Docker image
+        id: build-and-push
+        uses: docker/build-push-action@v7
+        with:
+          context: dstack
+          file: dstack/local-key-provider/build/Dockerfile.key-provider
+          push: true
+          tags: ${{ vars.DOCKERHUB_ORG }}/local-key-provider:${{ env.VERSION }}
+          platforms: linux/amd64
+          provenance: false
+
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v4
+        with:
+          subject-name: "docker.io/${{ vars.DOCKERHUB_ORG }}/local-key-provider"
+          subject-digest: ${{ steps.build-and-push.outputs.digest }}
+          push-to-registry: true
+
+      - name: Export enclave measurements
+        env:
+          IMAGE_DIGEST: ${{ steps.build-and-push.outputs.digest }}
+          IMAGE_NAME: ${{ vars.DOCKERHUB_ORG }}/local-key-provider
+        run: |
+          docker pull "${IMAGE_NAME}@${IMAGE_DIGEST}"
+          docker run --rm \
+            --entrypoint gramine-sgx-sigstruct-view \
+            "${IMAGE_NAME}@${IMAGE_DIGEST}" \
+            --output-format json \
+            /dstack/local-key-provider/build/local-key-provider.sig \
+            | tee local-key-provider.sig.json
+
+          MRENCLAVE=$(jq -er .mr_enclave local-key-provider.sig.json)
+          MRSIGNER=$(jq -er .mr_signer local-key-provider.sig.json)
+          echo "MRENCLAVE=$MRENCLAVE" >> "$GITHUB_ENV"
+          echo "MRSIGNER=$MRSIGNER" >> "$GITHUB_ENV"
+
+      - name: GitHub Release
+        uses: softprops/action-gh-release@v3
+        with:
+          name: "Local Key Provider Release v${{ env.VERSION }}"
+          tag_name: local-key-provider-v${{ env.VERSION }}
+          files: local-key-provider.sig.json
+          body: |
+            ## Docker Image Information
+
+            **Image**: `docker.io/${{ vars.DOCKERHUB_ORG }}/local-key-provider:${{ env.VERSION }}`
+
+            **Digest (SHA256)**: `${{ steps.build-and-push.outputs.digest }}`
+
+            **MRENCLAVE**: `${{ env.MRENCLAVE }}`
+
+            **MRSIGNER**: `${{ env.MRSIGNER }}`
+
+            **Verification**: [Verify on Sigstore](https://search.sigstore.dev/?hash=${{ steps.build-and-push.outputs.digest }})
+
+            The enclave signing key is generated for each build. Pin the
+            release's image digest and MRENCLAVE rather than expecting MRSIGNER
+            to remain stable across releases.

.github/workflows/rust-sdk-release.yml

@@ -53,8 +53,8 @@ jobs:
                   pkg_version("sdk/rust/types/Cargo.toml")),
               ("sdk/rust/Cargo.toml [package.version]",
                   pkg_version("sdk/rust/Cargo.toml")),
-              ("dstack/Cargo.toml [workspace.dependencies.dstack-sdk-types.version]",
-                  ws_dep_version("dstack/Cargo.toml", "dstack-sdk-types")),
+              ("sdk/rust/Cargo.toml [workspace.dependencies.dstack-sdk-types.version]",
+                  ws_dep_version("sdk/rust/Cargo.toml", "dstack-sdk-types")),
           ]
 
           fail = False
@@ -64,7 +64,7 @@ jobs:
               print(f"  {'OK ' if ok else 'BAD'}  {label}: {got}")
 
           if fail:
-              print(f"\ntag is dstack-sdk-v{want}; bump all three to {want} before tagging.",
+              print(f"\ntag is dstack-sdk-v{want}; bump SDK package versions to {want} before tagging.",
                     file=sys.stderr)
               sys.exit(1)
           PY

.github/workflows/sdk.yaml

@@ -34,9 +34,9 @@ jobs:
 
       - name: Verify WASM compilation
         # Ensures SDK types can be used in smart contracts
-        run: cargo check --manifest-path dstack/Cargo.toml --target=wasm32-unknown-unknown -p dstack-sdk-types
+        run: cargo check --manifest-path sdk/rust/Cargo.toml --target=wasm32-unknown-unknown -p dstack-sdk-types
 
       - name: Verify no_std compatibility
         run: |
-          cargo test --manifest-path dstack/Cargo.toml -p dstack-sdk-types --test no_std_test --no-default-features
-          cargo check --manifest-path dstack/Cargo.toml -p no_std_check --target thumbv6m-none-eabi
+          cargo test --manifest-path sdk/rust/Cargo.toml -p dstack-sdk-types --test no_std_test --no-default-features
+          cargo check --manifest-path sdk/rust/Cargo.toml -p no_std_check --target thumbv6m-none-eabi

.gitignore

@@ -1,4 +1,5 @@
 /dstack/target
+/dstack/local-key-provider/build/target
 /certs
 /build-config.sh
 /build/*

CLAUDE.md

@@ -146,11 +146,12 @@ forge clean
 
 ### Logging and Error Messages
 
-- **Never capitalize** the first letter of log messages and error messages
+- Start log messages and error messages with lowercase text
+- Preserve the conventional capitalization of identifiers and acronyms when they
+  begin a message (for example, `RIM`, `OCSP`, or `HTTP`)
 - Example: `log::info!("starting server on port {}", port);`
 - Example: `anyhow::bail!("failed to connect to server");`
-
-This rule is enforced in `.cursorrules`.
+- Example: `log::warn!("RIM upstream is unavailable");`
 
 ## Key Security Concepts
 

CONTRIBUTING.md

@@ -32,6 +32,47 @@ Use the narrowest owning directory. Fixture explanations and component
 READMEs should stay with their fixtures/components; general guides should be
 linked from the root README and live under `docs/`.
 
+## Test a guest without TEE hardware
+
+Development images include the extensible `dstack-tee-simulator`. Its default
+platform backend is TDX. On a VM launched with `no_tee`, that backend exposes
+the Linux interfaces used by the normal guest software:
+
+- configfs-tsm quote generation under `/sys/kernel/config/tsm/report`;
+- RTMR extension files used by `tdx-attest`;
+- a CCEL boot-event fixture.
+
+This keeps `dstack-prepare`, measurement, encrypted-storage setup, the guest
+agent, and attestation APIs on their production code paths. The generated quote
+has an intentionally invalid signature, so a production verifier or KMS must
+reject it.
+
+The service uses the default TDX backend. For direct simulator development, the
+same selection can be made explicitly with `dstack-tee-simulator --platform
+tdx`. New TEE platforms are implemented as separate backends behind the shared
+simulator lifecycle.
+
+Build or install a `dstack-dev-*` image, then deploy without KMS or gateway:
+
+```bash
+dstack deploy ./docker-compose.yml \
+  --name no-tee-dev \
+  --image dstack-dev-VERSION \
+  --no-kms \
+  --no-tee
+```
+
+To exercise persistent TPM-backed app keys as well, install `swtpm` on the VMM
+host and select `key_provider=tpm` in the VMM console (or pass `--key-provider
+tpm` in tooling that exposes the compose option). The VMM keeps the software
+TPM state in the VM work directory so that seal/unseal survives guest restarts.
+The software TPM is host-controlled and does not provide hardware isolation.
+
+The simulator package is installed only in development images. This mode
+provides no hardware isolation and must never be used with production
+workloads or secrets. Real quote generation, hardware isolation, and KMS
+authorization still require TDX or another supported TEE.
+
 ## Commit Convention
 
 This project uses [Conventional Commits](https://www.conventionalcommits.org/). Please format your commit messages as:

REUSE.toml

@@ -59,7 +59,7 @@ SPDX-License-Identifier = "Apache-2.0"
 path = [
     "dstack/gateway/templates/wg.conf",
     "dstack/guest-agent/templates/metrics.tpl",
-    "dstack/key-provider-build/sgx_default_qcnl.conf",
+    "dstack/local-key-provider/build/sgx_default_qcnl.conf",
 ]
 SPDX-FileCopyrightText = "Copyright (c) 2024-2025 The Project Contributors"
 SPDX-License-Identifier = "Apache-2.0"

docs/amd-sev-snp.md

@@ -82,6 +82,15 @@ Verification is fail-closed and includes:
 5. the unified OS image identity, `sha256(sha256sum.txt)`, which must match
    `digest.txt` and the SNP measurement document.
 
+For a GPU VM, the optional `MrConfigV3.gpu_policy_hash` field contains
+`SHA-256(JCS(requirements.gpu_policy))`. The signed SNP report binds the exact
+MrConfigV3 document through `HOST_DATA`, so a verifier can validate the report
+and document binding and then compare this field with the expected GPU policy
+digest. If the field is absent, this optional check is not asserted. This binds
+the GPU policy, but not the later `gpu-attestation` runtime event or the
+`GpuInfo` output: the current SEV-SNP path has no quote-bound runtime
+measurement register.
+
 The verifier supports the AMD Milan, Genoa, and Turin KDS product families.
 Bergamo and Siena are handled through AMD's canonical Genoa KDS product path.
 

docs/attestation-tdx.md

@@ -25,12 +25,16 @@ The MR register values indicate the following:
     - RTMR0: OVMF records CVM's virtual hardware setup, including CPU count, memory size, and device configuration. While dstack uses fixed devices, CPU and memory specifications can vary. RTMR0 can be computed from these specifications.
     - RTMR1: OVMF records the Linux kernel measurement.
     - RTMR2: Linux kernel records kernel cmdline (including rootfs hash) and initrd measurements.
-    - RTMR3: initrd records dstack App details, including compose hash, instance id, app id, and key provider.
+    - RTMR3: initrd records dstack App details, including compose hash, GPU policy and attestation events, instance id, app id, and key provider.
 
 MRTD, RTMR0, RTMR1, and RTMR2 can be pre-calculated from the built image (given CPU+RAM specifications). Compare these with the verified quote's MRs to confirm correct base image code execution.
 
 RTMR3 differs as it contains runtime information like compose hash and instance id. Verify this by replaying the event log - if the calculated RTMR3 matches the quote's RTMR3, the event log information is valid. Then verify the compose hash, key provider, and other event log details match expectations.
 
+For a GPU launch, `compose-hash` is followed by `gpu-policy-hash` and, after successful NVIDIA attestation and policy evaluation, `gpu-attestation`. The `gpu-policy-hash` payload is `SHA-256(JCS(requirements.gpu_policy))`, using `{}` when the field is omitted. The `gpu-attestation` payload is JSON containing the verified device count, CC/DevTools state, and `evidence_sha256`.
+
+The guest-agent `GpuInfo` API returns the complete `nvattest` JSON captured during boot. It is not trustworthy by itself. After verifying the TDX quote and replaying the event log to RTMR3, hash the exact UTF-8 bytes of `GpuInfo.attestation` and require the result to equal the `gpu-attestation` event's `evidence_sha256`. See [GPU Security for AI Workloads](./security/security-model.md#gpu-security-for-ai-workloads) for the event schema, ordering, Rego example, and platform differences.
+
 ### 2.2. Determining expected MRs
 MRTD, RTMR0, RTMR1, and RTMR2 correspond to the image. dstack OS builds all related software from source.
 Build the exact image revision you intend to verify. See