Merge remote-tracking branch 'origin/main' into glitch003/lambda-parity-payments-storage

# Conflicts:
#	lit-api-server/src/accounts/signable_contract.rs
#	lit-api-server/src/main.rs
#	lit-payments/src/config.rs
#	lit-payments/src/main.rs

Author: glitch003

.github/workflows/contract-bindings-check.yml

@@ -19,9 +19,6 @@ jobs:
       github.event.pull_request.head.repo.full_name == github.repository
     name: contract-bindings
     runs-on: self-hosted
-    concurrency:
-      group: foundry-toolchain
-      cancel-in-progress: false
     steps:
       - uses: actions/checkout@v4
 
@@ -37,8 +34,17 @@ jobs:
       - name: Check Foundry toolchain
         id: foundry-toolchain
         run: |
-          if command -v forge >/dev/null 2>&1 && forge --version | grep -Eq '(^|[^0-9])1\.5\.1([^0-9]|$)'; then
-            forge --version
+          # The global install on the shared runner lives in ~/.foundry/bin, which
+          # is only on PATH via the user's shell profile. GH Actions steps run in a
+          # non-login shell that doesn't source it, so probe the known location too.
+          forge_bin="$(command -v forge || true)"
+          if [ -z "$forge_bin" ] && [ -x "$HOME/.foundry/bin/forge" ]; then
+            forge_bin="$HOME/.foundry/bin/forge"
+          fi
+          if [ -n "$forge_bin" ] && "$forge_bin" --version | grep -Eq '(^|[^0-9])1\.5\.1([^0-9]|$)'; then
+            "$forge_bin" --version
+            # Put it on PATH for later steps so they don't trigger a reinstall.
+            echo "$(dirname "$forge_bin")" >> "$GITHUB_PATH"
             echo "installed=true" >> "$GITHUB_OUTPUT"
           else
             echo "installed=false" >> "$GITHUB_OUTPUT"

.github/workflows/contract-tests.yml

@@ -19,9 +19,6 @@ jobs:
       github.event.pull_request.head.repo.full_name == github.repository
     name: forge-test
     runs-on: self-hosted
-    concurrency:
-      group: foundry-toolchain
-      cancel-in-progress: false
     steps:
       - uses: actions/checkout@v4
         with:
@@ -41,8 +38,17 @@ jobs:
       - name: Check Foundry toolchain
         id: foundry-toolchain
         run: |
-          if command -v forge >/dev/null 2>&1 && forge --version | grep -Eq '(^|[^0-9])1\.5\.1([^0-9]|$)'; then
-            forge --version
+          # The global install on the shared runner lives in ~/.foundry/bin, which
+          # is only on PATH via the user's shell profile. GH Actions steps run in a
+          # non-login shell that doesn't source it, so probe the known location too.
+          forge_bin="$(command -v forge || true)"
+          if [ -z "$forge_bin" ] && [ -x "$HOME/.foundry/bin/forge" ]; then
+            forge_bin="$HOME/.foundry/bin/forge"
+          fi
+          if [ -n "$forge_bin" ] && "$forge_bin" --version | grep -Eq '(^|[^0-9])1\.5\.1([^0-9]|$)'; then
+            "$forge_bin" --version
+            # Put it on PATH for later steps so they don't trigger a reinstall.
+            echo "$(dirname "$forge_bin")" >> "$GITHUB_PATH"
             echo "installed=true" >> "$GITHUB_OUTPUT"
           else
             echo "installed=false" >> "$GITHUB_OUTPUT"

.github/workflows/deploy-staging.yml

@@ -26,6 +26,9 @@ on:
       api_root_url:
         description: "API base URL (e.g. https://example.com/core/v1)"
         value: ${{ jobs.wait.outputs.api_root_url }}
+      base_url:
+        description: "Site base URL without the /core/v1 suffix (e.g. https://example.com)"
+        value: ${{ jobs.wait.outputs.base_url }}
     inputs:
       expected_sha:
         description: "Expected git commit SHA for the new deployment (omit to skip version check)"
@@ -60,6 +63,7 @@ jobs:
     runs-on: ${{ fromJson(inputs.runner) }}
     outputs:
       api_root_url: ${{ steps.resolve-url.outputs.api_root_url }}
+      base_url: ${{ steps.resolve-url.outputs.base_url }}
     steps:
       - name: Checkout (for git describe)
         if: inputs.expected_sha != ''
@@ -106,6 +110,7 @@ jobs:
             exit 1
           fi
           echo "api_root_url=$BASE_URL" >> "$GITHUB_OUTPUT"
+          echo "base_url=${BASE_URL%/core/v1}" >> "$GITHUB_OUTPUT"
           echo "API base URL: $BASE_URL"
 
       - name: Wait for expected version

.github/workflows/wait-for-api-restart.yml

@@ -19,6 +19,7 @@ FROM debian:bookworm-slim AS runtime
 RUN apt-get update && apt-get install -y \
     ca-certificates \
     libssl3 \
+    curl \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app

Dockerfile.lit-api-server

@@ -0,0 +1,70 @@
+# Security Policy
+
+Lit Chipotle runs inside Intel TDX Trusted Execution Environments with key release
+gated by smart contracts on Base. We take the security of the stack — code,
+contracts, deployment pipeline, and governance — seriously, and we welcome reports
+from the security community.
+
+## Reporting a vulnerability
+
+**Please do not open a public GitHub issue for security vulnerabilities.**
+
+Report privately, via either:
+
+- **Email:** [security@litprotocol.com](mailto:security@litprotocol.com) (preferred).
+  Encrypt sensitive details with our PGP key if you have one; ask in your first
+  message and we will provide it.
+- **GitHub:** open a [private security advisory](https://github.com/LIT-Protocol/chipotle/security/advisories/new)
+  on this repository.
+
+In your report, please include:
+
+- A description of the issue and its potential impact.
+- Steps to reproduce, or a proof of concept.
+- The component affected (API server, Lit Actions runtime, contracts, deployment/CI,
+  governance, attestation/verification flow).
+- Any relevant version, commit hash, compose hash, or deployment URL.
+
+## What to expect
+
+- **Acknowledgement** within 3 business days.
+- **Triage and severity assessment** within 7 business days, with an initial
+  remediation plan.
+- **Coordinated disclosure.** We will work with you on a disclosure timeline and
+  credit you (if you wish) once a fix is released. Please give us a reasonable window
+  to remediate before any public disclosure.
+
+## Scope
+
+In scope:
+
+- The `lit-api-server`, `lit-actions` runtime, and `lit-static` dashboard.
+- Smart contracts in this repository (account/permission model, and the
+  attestation-governance contracts on Base).
+- The deployment pipeline and TEE attestation / verification flow.
+- The on-chain governance and key-release model.
+
+Out of scope (report upstream, but tell us if it affects us):
+
+- Vulnerabilities in Intel TDX, the dstack OS, or Phala Cloud infrastructure —
+  report to [Intel](https://www.intel.com/content/www/us/en/security-center/default.html),
+  [dstack](https://github.com/Dstack-TEE/dstack), or
+  [Phala](https://docs.phala.com/) respectively.
+- Third-party dependencies — report upstream; we track advisories via `deny.toml`.
+
+## Verifying the production deployment
+
+Much of our security model is publicly verifiable rather than asserted. Before
+reporting an issue that depends on "what code is running," confirm the live state
+yourself:
+
+- [What Is Attestation?](docs/architecture/verification/attestation.mdx) — the model in plain English.
+- [Upgrade Governance](docs/architecture/verification/upgrade-governance.mdx) — how releases are approved (2-of-4 Safe on Base).
+- [On-Chain KMS](docs/architecture/verification/onchain-kms.mdx) — how key release is gated.
+- [Verify in 30 Seconds](docs/architecture/verification/quick-verify.mdx) — Trust Center report + three terminal commands.
+
+## Operational security references (internal)
+
+- [Upgrade governance & release approval](architectureDocs/deployment/vm-code-upgrade.md)
+- [Incident response & key-compromise runbook](architectureDocs/deployment/incident-response.md)
+- [Secrets management](architectureDocs/deployment/secrets-management.md)

SECURITY.md

@@ -0,0 +1,122 @@
+# Incident Response & Key-Compromise Runbook
+
+How to respond when something in the trust chain goes wrong. The planned-upgrade and
+rollback flow lives in [vm-code-upgrade.md](vm-code-upgrade.md); this document covers
+the *adversarial* and *failure* scenarios that one does not.
+
+> **Status:** several response steps below depend on operational decisions that are
+> not yet finalized (marked **TODO/DECIDE**). Treat this as the working runbook and
+> resolve the open items before relying on it in a real incident.
+
+## Roles & contacts
+
+| Role | Who | Responsibility |
+|---|---|---|
+| Incident commander | **TODO/DECIDE** | Owns the response, declares severity, coordinates comms |
+| Safe signers | 4 keyholders (2-of-4 quorum) | Execute on-chain governance actions (whitelist removal, owner change) |
+| Phala account owner | **TODO/DECIDE** | Holds `PHALA_CLOUD_API_KEY`; can redeploy/stop CVMs |
+| Comms / disclosure | **TODO/DECIDE** | External notifications, status page, `security@` triage |
+
+Confirm the live Safe signer set before any action:
+
+```bash
+cast call 0xF688411c0FFc300cAb33EB1dA651DBb3E6891098 \
+  "getOwners()(address[])" --rpc-url https://mainnet.base.org
+```
+
+## Key facts that shape the response
+
+- **Governance is 2-of-4 with no timelock.** A quorum can act immediately — good for
+  emergency response, but also means two compromised signer keys are sufficient to
+  change the whitelist. There is no delay to "catch" a malicious change in flight;
+  detection must be real-time (see [Monitoring](#detective-controls)).
+- **Key derivation is deterministic per compose hash.** The same attested code always
+  derives the same application keys from the KMS root. This is why rollback works
+  without re-issuance — and why a compromised *code version* that obtained keys must
+  be treated as having compromised the data those keys protect.
+- **Removing a compose hash blocks key issuance, not a running process.** See below.
+
+## Scenario 1 — Malicious or vulnerable compose hash got whitelisted
+
+Goal: stop the bad version from obtaining keys and serving.
+
+1. **Revoke the whitelist.** Have the Safe execute `removeComposeHash(badHash)` on
+   `DstackApp` (`0x3F91…05FfC`). 2-of-4 approval required.
+2. **Stop the running CVM.** Revocation prevents *future* key issuance and prevents
+   the version from booting again — but a CVM that already holds its keys keeps
+   running until restarted. The Phala account owner must explicitly **stop/redeploy**
+   the affected CVM via the Phala CLI/dashboard to take it out of service.
+   **TODO/DECIDE:** confirm exact Phala command and whether gateway can drain traffic
+   first.
+3. **Re-deploy a known-good version** whose compose hash is still whitelisted.
+4. **Assess data exposure.** Any data the bad version could decrypt with its derived
+   keys must be considered exposed.
+
+## Scenario 2 — Safe signer key compromised
+
+1. **If still below quorum (1 key):** rotate immediately but no funds/governance at
+   immediate risk. The remaining 3 honest signers still control the 2-of-4.
+2. **Replace the signer:** Safe executes `swapOwner` (or `removeOwner` +
+   `addOwnerWithThreshold`) to drop the compromised key. 2-of-4 of the *honest*
+   signers required.
+3. **If two keys compromised (quorum reached by attacker):** treat as full governance
+   compromise — the attacker can whitelist arbitrary code. **TODO/DECIDE:** there is
+   currently no higher-authority backstop above the Safe. Mitigations to evaluate:
+   raising the threshold, adding a timelock to create a reaction window, or a
+   guardian/recovery module.
+
+## Scenario 3 — `PHALA_CLOUD_API_KEY` leaked
+
+The Phala API key can redeploy and reconfigure CVMs and read/set CVM secrets. It
+**cannot** mint keys for un-whitelisted code (the on-chain gate still holds), but it
+can disrupt availability and reconfigure encrypted env injection.
+
+1. **Revoke the key** in the Phala dashboard (Avatar → API Tokens) and issue a new one.
+2. **Rotate all CVM-injected secrets** (see Scenario 5) — assume they were readable.
+3. **Audit recent deployments** for unexpected CVM changes.
+
+## Scenario 4 — Base / KMS availability failure
+
+- The KMS issues keys at boot. If Base RPC or the KMS is unreachable, **already-running
+  CVMs are unaffected** (they hold their keys); only restarts/new boots are blocked.
+- Do **not** force a redeploy during a KMS outage — a restarted CVM may fail to
+  obtain keys and go down.
+- **TODO/DECIDE:** document RPC redundancy / fallback endpoints used by the KMS path.
+
+## Scenario 5 — Secret compromise (Stripe, AWS, GCP, RPC)
+
+These are injected as encrypted Phala env vars and are **outside the attestation
+measurement** (not part of the compose hash). See
+[secrets-management.md](secrets-management.md).
+
+1. Rotate the upstream credential (Stripe dashboard, AWS IAM, GCP service account).
+2. Update the encrypted Phala secret and redeploy.
+3. Scope the blast radius per credential (e.g. Route53 IAM should be DNS-01 only).
+
+## Detective controls
+
+Real-time detection matters more here than usual because there is no governance
+timelock to provide a reaction window.
+
+- **Alert on governance writes.** Monitor `DstackApp` and the Safe for
+  `addComposeHash` / `removeComposeHash`, `addOwner` / `swapOwner` / `removeOwner`,
+  and `changeThreshold` events; page on any unplanned occurrence. **TODO/DECIDE:**
+  wire this into the existing Grafana stack (see chipotle monitoring) or a
+  contract-watcher.
+- **Alert on attestation drift.** Periodically confirm the live `compose_hash` from
+  `/info` is still whitelisted and matches the expected release.
+- **Alert on CVM lifecycle changes** from the Phala account.
+
+## Post-incident
+
+- Capture a timeline and the on-chain transaction hashes of every governance action.
+- File a coordinated-disclosure note per [SECURITY.md](../../SECURITY.md) if external
+  parties are affected.
+- Open follow-ups for any **TODO/DECIDE** item this incident exercised.
+
+## References
+
+- [vm-code-upgrade.md](vm-code-upgrade.md) — planned upgrade & rollback
+- [derot-key-issuance.md](derot-key-issuance.md) — how keys are issued
+- [secrets-management.md](secrets-management.md) — out-of-band secret injection
+- [SECURITY.md](../../SECURITY.md) — disclosure policy

architectureDocs/deployment/incident-response.md

@@ -0,0 +1,68 @@
+# Secrets Management
+
+How runtime secrets reach the Lit Chipotle CVM, what the attestation does and does
+**not** cover about them, and the policies that govern them.
+
+> **Status:** the mechanism below is accurate to `docker-compose.phala.yml`. The
+> policy items (who can set/read, rotation cadence, dev/prod separation) are marked
+> **TODO/DECIDE** where not yet finalized.
+
+## The important caveat: secrets are outside the attestation
+
+Attestation proves *what code* the enclave runs (the compose hash in RTMR3) and *what
+OS* it booted. It does **not** measure the values of injected environment variables.
+Secrets are supplied to the CVM as **encrypted Phala CVM environment variables**,
+resolved by Phala at container startup and decrypted inside the TEE.
+
+Consequences a reviewer should understand:
+
+- The compose hash pins the *names and wiring* of env vars (they appear in the
+  digest-pinned `docker-compose`), but **not their values**. Changing a secret's
+  value does not change the compose hash.
+- Whoever controls the Phala account (the `PHALA_CLOUD_API_KEY`) controls what gets
+  injected. This is why that key is a high-value target — see
+  [incident-response.md](incident-response.md) Scenario 3.
+- The secrets are protected *in transit and at rest* by Phala's encrypted-env
+  mechanism and only decrypted inside the attested enclave; they are not exposed to
+  the host. But their *provenance* is not part of the on-chain trust chain.
+
+## Secrets injected today
+
+From `docker-compose.phala.yml` (all set as encrypted Phala CVM env vars):
+
+| Secret | Used by | Purpose | Blast radius if leaked |
+|---|---|---|---|
+| `BASE_CHAIN_RPC` | lit-api-server | Base RPC endpoint | Low (read-mostly endpoint) |
+| `STRIPE_SECRET_KEY` | lit-api-server | Stripe billing (live in prod) | **High** — payments |
+| `STRIPE_PUBLISHABLE_KEY` | lit-api-server | Stripe client key | Low (publishable) |
+| `GCP_SERVICE_ACCOUNT_JSON` | otel-collector | Ship telemetry to GCP | Medium — scoped to telemetry project |
+| `GCP_PROJECT_ID` | otel-collector | GCP target project | Low |
+| `CERTBOT_AWS_ACCESS_KEY_ID` / `…SECRET_ACCESS_KEY` / `…ROLE_ARN` | dstack-ingress | Route53 DNS-01 for TLS | **High** — DNS control; scope to DNS-01 only |
+
+## Policies
+
+- **Least privilege per credential.** Each credential should be scoped to exactly its
+  function (e.g. the Route53 IAM principal limited to DNS-01 TXT records on the cert
+  domain; the GCP service account limited to telemetry ingestion). **TODO/DECIDE:**
+  confirm and document the IAM/role scopes actually in use.
+- **Dev vs prod separation.** Dev uses Stripe sandbox keys and the dev environment;
+  prod uses live keys. **TODO/DECIDE:** confirm separate Phala accounts/projects and
+  that no prod secret is reachable from a dev deploy.
+- **Who can set/read.** Setting CVM secrets requires the Phala account credentials.
+  **TODO/DECIDE:** name the owning role and how access is granted/revoked.
+- **Rotation.** **TODO/DECIDE:** define rotation cadence per credential and the
+  rotate-then-redeploy procedure (see incident runbook Scenario 5).
+
+## Telemetry egress note
+
+The `otel-collector` ships logs/metrics to GCP from inside the CVM — a data-plane
+egress out of the TEE. Ensure exported telemetry contains **no secrets or request
+plaintext** (only operational metrics and scrubbed logs). **TODO/DECIDE:** document
+the scrubbing/allow-list applied before export.
+
+## References
+
+- `docker-compose.phala.yml` — the source of truth for injected env vars
+- [deployment.md](deployment.md) — required CI secrets/variables
+- [incident-response.md](incident-response.md) — secret-compromise response
+- [Phala: CVM environment variables / secrets](https://docs.phala.com/)

architectureDocs/deployment/secrets-management.md

@@ -54,9 +54,14 @@ flowchart TB
 2. **Compute compose-hash** — dstack hashes the full `docker-compose.yml` (with pinned image
    digests) to produce the compose-hash that will appear in RTMR3 at boot.
 
-3. **Governance approval** — The DstackApp owner (wallet, multisig, or timelock) submits an
-   on-chain transaction to whitelist the new compose-hash. A timelock or multisig introduces a
-   mandatory delay, giving stakeholders time to review before the new code can receive keys.
+3. **Governance approval** — The DstackApp owner submits an on-chain transaction to whitelist
+   the new compose-hash. For the **production hosted deployment** this owner is a **2-of-4 Safe
+   multisig** (`0xF688…1098`) with **no timelock** — an approved change takes effect as soon as a
+   quorum executes it. Self-hosters can configure their own owner (wallet, higher-threshold
+   multisig, timelock, or DAO) to add a mandatory review delay. What signers verify before
+   approving (provenance via Sigstore, reproducible compose-hash, diff review) and the live
+   governance facts are documented in the published
+   [Upgrade Governance](../../docs/architecture/verification/upgrade-governance.mdx) page.
 
 4. **Deploy** — After the whitelist tx is confirmed, CI redeploys (`phala deploy` or DeRoT
    equivalent) with the new image digest substituted into docker-compose.