Backend fallback (cloud run) (#1306)

- Added support for `@opentelemetry/sdk-node` in the backend.
- Updated various dependencies including AWS SDK and OpenTelemetry
packages.
- Implemented graceful shutdown handling for non-Vercel runtimes in
`prisma-client.tsx`.
- Enhanced AWS credentials retrieval to support GCP Workload Identity
Federation.
- Introduced a Dockerfile for Cloud Run deployment, optimizing the
backend build process.
- Updated `.gitignore` to include Terraform runtime files and secrets.

This commit improves the backend's observability and deployment
flexibility, particularly for Cloud Run environments.

<!--

Make sure you've read the CONTRIBUTING.md guidelines:
https://github.com/stack-auth/stack-auth/blob/dev/CONTRIBUTING.md

-->


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* OpenTelemetry observability with dynamic provider selection per
deployment.
  * Cloud Run trusted-proxy support for accurate client IP handling.
  * Graceful shutdown that waits for in-flight background work.
* New background-task handling to improve async webhook/email delivery
reliability.
* AWS credential providers added (Vercel OIDC & GCP Workload Identity
Federation).
  * Dockerized backend image for Cloud Run / self-host deployments.

* **Chores**
  * Updated dependencies for OpenTelemetry and AWS SDK support.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: Konstantin Wohlwend <n2d4xc@gmail.com>

Author: mantrakp04

.github/workflows/e2e-fallback-tests.yaml

@@ -0,0 +1,164 @@
+# TODO: keep in sync with e2e-tests.yaml — this is a near-copy with the backend
+# started on the fallback port (8110) only, so the SDK exercises fallback logic.
+name: Runs E2E Fallback Tests
+
+on:
+  push:
+    branches:
+      - main
+      - dev
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/dev' }}
+
+jobs:
+  build:
+    name: E2E Fallback Tests (Node ${{ matrix.node-version }})
+    runs-on: ubicloud-standard-8
+    env:
+      NODE_ENV: test
+      STACK_ENABLE_HARDCODED_PASSKEY_CHALLENGE_FOR_TESTING: yes
+      STACK_DATABASE_CONNECTION_STRING: "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:8128/stackframe"
+      STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000"
+      STACK_EXTERNAL_DB_SYNC_DIRECT: "false"
+      # SDK reads this as the primary URL, discovers hardcoded fallback to port 8110
+      NEXT_PUBLIC_STACK_API_URL: "http://localhost:8102"
+      # Tells js-helpers to omit explicit baseUrl so the SDK exercises fallback logic
+      STACK_TEST_SDK_FALLBACK: "true"
+
+    strategy:
+      matrix:
+        node-version: [22.x]
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Setup Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v6
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+
+      - name: Start Docker Compose in background
+        uses: JarvusInnovations/background-action@v1.0.7
+        with:
+          run: docker compose -f docker/dependencies/docker.compose.yaml up --pull always -d &
+          wait-on: /dev/null
+          tail: true
+          wait-for: 3s
+          log-output-if: true
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Create .env.test.local files
+        run: |
+          cp apps/backend/.env.development apps/backend/.env.test.local
+          cp apps/dashboard/.env.development apps/dashboard/.env.test.local
+          cp apps/e2e/.env.development apps/e2e/.env.test.local
+          cp docs/.env.development docs/.env.test.local
+          cp examples/cjs-test/.env.development examples/cjs-test/.env.test.local
+          cp examples/demo/.env.development examples/demo/.env.test.local
+          cp examples/docs-examples/.env.development examples/docs-examples/.env.test.local
+          cp examples/e-commerce/.env.development examples/e-commerce/.env.test.local
+          cp examples/middleware/.env.development examples/middleware/.env.test.local
+          cp examples/supabase/.env.development examples/supabase/.env.test.local
+          cp examples/convex/.env.development examples/convex/.env.test.local
+
+      - name: Build
+        run: pnpm build
+
+      - name: Wait on Postgres
+        run: pnpm run wait-until-postgres-is-ready:pg_isready
+
+      - name: Wait on Inbucket
+        run: pnpx wait-on tcp:localhost:8129
+
+      - name: Wait on Svix
+        run: pnpx wait-on tcp:localhost:8113
+
+      - name: Wait on QStash
+        run: pnpx wait-on tcp:localhost:8125
+
+      - name: Wait on ClickHouse
+        run: pnpx wait-on http://localhost:8136/ping
+
+      - name: Initialize database
+        run: pnpm run db:init
+
+      # Start backend ONLY on fallback port 8110 — primary port 8102 is intentionally left down
+      # so the SDK exercises its fallback logic for every request.
+      - name: Start stack-backend on fallback port (8110)
+        uses: JarvusInnovations/background-action@v1.0.7
+        with:
+          run: pnpm -C apps/backend run with-env:test next start --port 8110 &
+          wait-on: |
+            http://localhost:8110
+          tail: true
+          wait-for: 30s
+          log-output-if: true
+
+      - name: Start stack-dashboard in background
+        uses: JarvusInnovations/background-action@v1.0.7
+        with:
+          run: pnpm run start:dashboard --log-order=stream &
+          wait-on: |
+            http://localhost:8101
+          tail: true
+          wait-for: 30s
+          log-output-if: true
+
+      - name: Start mock-oauth-server in background
+        uses: JarvusInnovations/background-action@v1.0.7
+        with:
+          run: pnpm run start:mock-oauth-server --log-order=stream &
+          wait-on: |
+            http://localhost:8110
+          tail: true
+          wait-for: 30s
+          log-output-if: true
+
+      - name: Start run-email-queue in background
+        uses: JarvusInnovations/background-action@v1.0.7
+        with:
+          run: pnpm -C apps/backend run run-email-queue --log-order=stream &
+          wait-on: |
+            http://localhost:8110
+          tail: true
+          wait-for: 30s
+          log-output-if: true
+
+      - name: Start run-cron-jobs in background
+        uses: JarvusInnovations/background-action@v1.0.7
+        with:
+          run: pnpm -C apps/backend run run-cron-jobs:test --log-order=stream &
+          wait-on: |
+            http://localhost:8110
+          tail: true
+          wait-for: 30s
+          log-output-if: true
+
+      - name: Wait 10 seconds
+        run: sleep 10
+
+      - name: Verify primary port 8102 is NOT running
+        run: |
+          if curl -s -o /dev/null -w "%{http_code}" http://localhost:8102/health 2>/dev/null | grep -q "200"; then
+            echo "ERROR: Primary backend on port 8102 should NOT be running for fallback tests"
+            exit 1
+          fi
+          echo "Confirmed: primary port 8102 is down, fallback tests will exercise SDK fallback logic"
+
+      # Only run JS SDK tests — these exercise the SDK's fallback logic.
+      # Backend API tests use direct HTTP calls that don't go through fallback.
+      # Exclude cross-domain-auth which hardcodes the primary URL.
+      - name: Run SDK fallback tests
+        run: pnpm -w run pre && cd apps/e2e && npx vitest run tests/js/ --exclude '**/{cross-domain-auth,oauth,email-template-existing-project}*'
+
+      - name: Print Docker Compose logs
+        if: always()
+        run: docker compose -f docker/dependencies/docker.compose.yaml logs

apps/backend/package.json

@@ -11,7 +11,7 @@
     "with-env:dev": "dotenv -c development --",
     "with-env:prod": "dotenv -c production --",
     "with-env:test": "dotenv -c test --",
-    "dev": "concurrently -n \"dev,codegen,prisma-studio,email-queue,cron-jobs\" -k \"next dev --port ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}02 ${STACK_BACKEND_DEV_EXTRA_ARGS:-}\" \"pnpm run codegen:watch\" \"pnpm run prisma-studio\" \"pnpm run run-email-queue\" \"pnpm run run-cron-jobs\"",
+    "dev": "BACKEND_PORT=${STACK_DEV_FALLBACK_BACKEND:+${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}10} && BACKEND_PORT=${BACKEND_PORT:-${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}02} && concurrently -n \"dev,codegen,prisma-studio,email-queue,cron-jobs\" -k \"next dev --port $BACKEND_PORT ${STACK_BACKEND_DEV_EXTRA_ARGS:-}\" \"pnpm run codegen:watch\" \"pnpm run prisma-studio\" \"pnpm run run-email-queue\" \"pnpm run run-cron-jobs\"",
     "dev:inspect": "STACK_BACKEND_DEV_EXTRA_ARGS=\"--inspect\" pnpm run dev",
     "dev:profile": "STACK_BACKEND_DEV_EXTRA_ARGS=\"--experimental-cpu-prof\" pnpm run dev",
     "build": "pnpm run codegen && next build",

apps/backend/src/app/api/latest/auth/password/sign-up/route.tsx

@@ -4,7 +4,7 @@ import { createAuthTokens } from "@/lib/tokens";
 import { getRequestContextAndBotChallengeAssessment, botChallengeFlowRequestSchemaFields } from "@/lib/turnstile";
 import { createOrUpgradeAnonymousUserWithRules } from "@/lib/users";
 import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler";
-import { runAsynchronouslyAndWaitUntil } from "@/utils/vercel";
+import { runAsynchronouslyAndWaitUntil } from "@/utils/background-tasks";
 import { KnownErrors } from "@stackframe/stack-shared";
 import { getPasswordError } from "@stackframe/stack-shared/dist/helpers/password";
 import { adaptSchema, clientOrHigherAuthTypeSchema, emailVerificationCallbackUrlSchema, passwordSchema, signInEmailSchema, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields";

apps/backend/src/app/api/latest/internal/backend-urls/route.test.tsx

@@ -0,0 +1,68 @@
+import { describe, expect, it } from 'vitest';
+import { parseAndValidateConfig } from './route';
+
+describe('parseAndValidateConfig', () => {
+  it('should parse a single entry with probability 1', () => {
+    const result = parseAndValidateConfig({
+      "1": ["https://api.stack-auth.com"],
+    });
+    expect(result).toEqual([
+      { probability: 1, urls: ["https://api.stack-auth.com"] },
+    ]);
+  });
+
+  it('should parse multiple entries', () => {
+    const result = parseAndValidateConfig({
+      "0.7": ["https://api.stack-auth.com", "https://api2.stack-auth.com"],
+      "0.3": ["https://api2.stack-auth.com", "https://api.stack-auth.com"],
+    });
+    expect(result).toEqual([
+      { probability: 0.7, urls: ["https://api.stack-auth.com", "https://api2.stack-auth.com"] },
+      { probability: 0.3, urls: ["https://api2.stack-auth.com", "https://api.stack-auth.com"] },
+    ]);
+  });
+
+  it('should allow probabilities summing to less than 1', () => {
+    const result = parseAndValidateConfig({
+      "0.5": ["https://api.stack-auth.com"],
+      "0.3": ["https://api2.stack-auth.com"],
+    });
+    expect(result).toHaveLength(2);
+  });
+
+  it('should reject non-object input', () => {
+    expect(() => parseAndValidateConfig("string")).toThrow("must be a JSON object");
+    expect(() => parseAndValidateConfig(null)).toThrow("must be a JSON object");
+    expect(() => parseAndValidateConfig([])).toThrow("must be a JSON object");
+    expect(() => parseAndValidateConfig(42)).toThrow("must be a JSON object");
+  });
+
+  it('should reject empty object', () => {
+    expect(() => parseAndValidateConfig({})).toThrow("at least one entry");
+  });
+
+  it('should reject invalid probability keys', () => {
+    expect(() => parseAndValidateConfig({ "abc": ["https://a.com"] })).toThrow("must be a number between 0 and 1");
+    expect(() => parseAndValidateConfig({ "-0.1": ["https://a.com"] })).toThrow("must be a number between 0 and 1");
+    expect(() => parseAndValidateConfig({ "1.5": ["https://a.com"] })).toThrow("must be a number between 0 and 1");
+  });
+
+  it('should reject probabilities summing to more than 1', () => {
+    expect(() => parseAndValidateConfig({
+      "0.6": ["https://api.stack-auth.com"],
+      "0.5": ["https://api2.stack-auth.com"],
+    })).toThrow("exceeds 1");
+  });
+
+  it('should reject invalid URL values', () => {
+    expect(() => parseAndValidateConfig({ "1": ["not-a-url"] })).toThrow();
+  });
+
+  it('should reject empty URL arrays', () => {
+    expect(() => parseAndValidateConfig({ "1": [] })).toThrow();
+  });
+
+  it('should reject non-array values', () => {
+    expect(() => parseAndValidateConfig({ "1": "https://api.stack-auth.com" })).toThrow();
+  });
+});

apps/backend/src/app/api/latest/internal/backend-urls/route.tsx

@@ -0,0 +1,104 @@
+import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler";
+import { urlSchema, yupArray, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields";
+import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors";
+import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env";
+import { getDefaultApiUrls } from "@stackframe/stack-shared/dist/utils/urls";
+
+/**
+ * Env var format: JSON object mapping probability (as string number) to URL arrays.
+ * Probabilities must sum to <= 1. Remaining probability uses the last entry as fallback.
+ *
+ * Example:
+ * {
+ *   "0.7": ["https://api.stack-auth.com", "https://api2.stack-auth.com"],
+ *   "0.3": ["https://api2.stack-auth.com", "https://api.stack-auth.com"]
+ * }
+ */
+
+const urlsArraySchema = yupArray(urlSchema.defined()).min(1).defined();
+
+export function parseAndValidateConfig(raw: unknown): Array<{ probability: number, urls: string[] }> {
+  if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
+    throw new StackAssertionError("STACK_BACKEND_URLS_CONFIG must be a JSON object mapping probability strings to URL arrays");
+  }
+
+  const entries = Object.entries(raw as Record<string, unknown>).map(([key, value]) => {
+    const probability = Number(key);
+    if (isNaN(probability) || probability < 0 || probability > 1) {
+      throw new StackAssertionError(`Invalid probability key "${key}": must be a number between 0 and 1`);
+    }
+    const urls = urlsArraySchema.validateSync(value);
+    return { probability, urls };
+  });
+
+  if (entries.length === 0) {
+    throw new StackAssertionError("STACK_BACKEND_URLS_CONFIG must have at least one entry");
+  }
+
+  const sum = entries.reduce((acc, e) => acc + e.probability, 0);
+  if (sum > 1 + 1e-9) {
+    throw new StackAssertionError(`Probabilities sum to ${sum}, which exceeds 1`);
+  }
+
+  return entries;
+}
+
+let cachedEntries: ReturnType<typeof parseAndValidateConfig> | undefined;
+function getCachedConfig() {
+  if (!cachedEntries) {
+    const rawEnv = getEnvVariable("STACK_BACKEND_URLS_CONFIG", "");
+    if (rawEnv) {
+      let parsed;
+      try {
+        parsed = JSON.parse(rawEnv);
+      } catch (e) {
+        throw new StackAssertionError(`STACK_BACKEND_URLS_CONFIG is not valid JSON: ${e}`);
+      }
+      cachedEntries = parseAndValidateConfig(parsed);
+    } else {
+      cachedEntries = [{ probability: 1, urls: getDefaultApiUrls(getEnvVariable("NEXT_PUBLIC_STACK_API_URL")) }];
+    }
+  }
+  return cachedEntries;
+}
+
+export const GET = createSmartRouteHandler({
+  metadata: {
+    hidden: true,
+    summary: "Get backend URLs",
+    description: "Returns a prioritized list of backend API URLs for client-side failover",
+    tags: ["Internal"],
+  },
+  request: yupObject({
+    method: yupString().oneOf(["GET"]).defined(),
+  }),
+  response: yupObject({
+    statusCode: yupNumber().oneOf([200]).defined(),
+    bodyType: yupString().oneOf(["json"]).defined(),
+    body: yupObject({
+      urls: yupArray(yupString().defined()).defined(),
+    }).defined(),
+  }),
+  handler: async () => {
+    const entries = getCachedConfig();
+
+    const roll = Math.random();
+    let cumulative = 0;
+    for (const entry of entries) {
+      cumulative += entry.probability;
+      if (roll < cumulative) {
+        return {
+          statusCode: 200,
+          bodyType: "json",
+          body: { urls: entry.urls },
+        } as const;
+      }
+    }
+
+    return {
+      statusCode: 200,
+      bodyType: "json",
+      body: { urls: entries[entries.length - 1].urls },
+    } as const;
+  },
+});

apps/backend/src/app/api/latest/project-permissions/crud.tsx

@@ -3,7 +3,7 @@ import { ensureProjectPermissionExists, ensureUserExists } from "@/lib/request-c
 import { sendProjectPermissionCreatedWebhook, sendProjectPermissionDeletedWebhook } from "@/lib/webhooks";
 import { getPrismaClientForTenancy, retryTransaction } from "@/prisma-client";
 import { createCrudHandlers } from "@/route-handlers/crud-handler";
-import { runAsynchronouslyAndWaitUntil } from "@/utils/vercel";
+import { runAsynchronouslyAndWaitUntil } from "@/utils/background-tasks";
 import { KnownErrors } from "@stackframe/stack-shared";
 import { projectPermissionsCrud } from '@stackframe/stack-shared/dist/interface/crud/project-permissions';
 import { permissionDefinitionIdSchema, userIdOrMeSchema, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields";

apps/backend/src/app/api/latest/team-memberships/crud.tsx

@@ -6,7 +6,7 @@ import { PrismaTransaction } from "@/lib/types";
 import { sendTeamMembershipCreatedWebhook, sendTeamMembershipDeletedWebhook, sendTeamPermissionCreatedWebhook } from "@/lib/webhooks";
 import { getPrismaClientForTenancy, retryTransaction } from "@/prisma-client";
 import { createCrudHandlers } from "@/route-handlers/crud-handler";
-import { runAsynchronouslyAndWaitUntil } from "@/utils/vercel";
+import { runAsynchronouslyAndWaitUntil } from "@/utils/background-tasks";
 import { KnownErrors } from "@stackframe/stack-shared";
 import { teamMembershipsCrud } from "@stackframe/stack-shared/dist/interface/crud/team-memberships";
 import { userIdOrMeSchema, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields";

apps/backend/src/app/api/latest/team-permissions/crud.tsx

@@ -3,7 +3,7 @@ import { ensureTeamMembershipExists, ensureUserTeamPermissionExists } from "@/li
 import { sendTeamPermissionCreatedWebhook, sendTeamPermissionDeletedWebhook } from "@/lib/webhooks";
 import { getPrismaClientForTenancy, retryTransaction } from "@/prisma-client";
 import { createCrudHandlers } from "@/route-handlers/crud-handler";
-import { runAsynchronouslyAndWaitUntil } from "@/utils/vercel";
+import { runAsynchronouslyAndWaitUntil } from "@/utils/background-tasks";
 import { KnownErrors } from "@stackframe/stack-shared";
 import { teamPermissionsCrud } from '@stackframe/stack-shared/dist/interface/crud/team-permissions';
 import { permissionDefinitionIdSchema, userIdOrMeSchema, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields";

apps/backend/src/app/api/latest/teams/crud.tsx

@@ -4,7 +4,7 @@ import { sendTeamCreatedWebhook, sendTeamDeletedWebhook, sendTeamUpdatedWebhook
 import { getPrismaClientForTenancy, retryTransaction } from "@/prisma-client";
 import { createCrudHandlers } from "@/route-handlers/crud-handler";
 import { uploadAndGetUrl } from "@/s3";
-import { runAsynchronouslyAndWaitUntil } from "@/utils/vercel";
+import { runAsynchronouslyAndWaitUntil } from "@/utils/background-tasks";
 import { Prisma } from "@/generated/prisma/client";
 import { KnownErrors } from "@stackframe/stack-shared";
 import { teamsCrud } from "@stackframe/stack-shared/dist/interface/crud/teams";