Auto migration (#526)

<!--

Make sure you've read the CONTRIBUTING.md guidelines:
https://github.com/stack-auth/stack-auth/blob/dev/CONTRIBUTING.md

-->

<!-- ELLIPSIS_HIDDEN -->


----

> [!IMPORTANT]
> Introduces an automated database migration system, replacing manual
Prisma commands with new scripts and updating workflows, configurations,
and tests accordingly.
> 
>   - **Auto-Migration System**:
> - Introduces `db-migrations.ts` script for handling database
migrations automatically.
> - Adds utility functions in `utils.tsx` for managing migration files.
> - Implements `applyMigrations` and `runMigrationNeeded` in `index.tsx`
for executing migrations.
>   - **Workflow and Scripts**:
> - Updates GitHub workflows (`check-prisma-migrations.yaml`,
`e2e-api-tests.yaml`) to use new migration commands.
> - Replaces `prisma migrate` commands with `db:init`, `db:migrate`,
etc., in `package.json` and `README.md`.
>   - **Testing**:
> - Adds `auto-migration.tests.ts` for testing migration logic and
concurrency handling.
>   - **Configuration**:
> - Updates `.env.development` and `vitest.config.ts` for new
environment variables and paths.
> - Modifies `turbo.json` and `package.json` to include new migration
tasks and scripts.
> 
> <sup>This description was created by </sup>[<img alt="Ellipsis"
src="https://img.shields.io/badge/Ellipsis-blue?color=175173">](https://www.ellipsis.dev?ref=stack-auth%2Fstack-auth&utm_source=github&utm_medium=referral)<sup>
for 2c24183. You can
[customize](https://app.ellipsis.dev/stack-auth/settings/summaries) this
summary. It will automatically update as commits are pushed.</sup>


<!-- ELLIPSIS_HIDDEN -->

---------

Co-authored-by: Konsti Wohlwend <n2d4xc@gmail.com>
Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>

Author: 3 people

.github/workflows/check-prisma-migrations.yaml

@@ -39,4 +39,4 @@ jobs:
         run: docker run -d --name postgres-prisma-diff-shadow -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=PLACEHOLDER-PASSWORD--dfaBC1hm1v -e POSTGRES_DB=postgres -p 5432:5432 postgres:latest
 
       - name: Check for differences in Prisma schema and migrations
-        run: pnpm run prisma migrate diff --from-migrations ./prisma/migrations --to-schema-datamodel ./prisma/schema.prisma --shadow-database-url postgres://postgres:PLACEHOLDER-PASSWORD--dfaBC1hm1v@localhost:5432/postgres --exit-code
+        run: cd apps/backend && pnpm run prisma migrate diff --from-migrations ./prisma/migrations --to-schema-datamodel ./prisma/schema.prisma --shadow-database-url postgres://postgres:PLACEHOLDER-PASSWORD--dfaBC1hm1v@localhost:5432/postgres --exit-code

.github/workflows/e2e-api-tests.yaml

@@ -17,6 +17,7 @@ jobs:
     env:
       NODE_ENV: test
       STACK_ENABLE_HARDCODED_PASSKEY_CHALLENGE_FOR_TESTING: yes
+      STACK_DIRECT_DATABASE_CONNECTION_STRING: "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:5432/stackframe"
 
     strategy:
       matrix:
@@ -96,7 +97,7 @@ jobs:
         run: npx wait-on tcp:localhost:8113
 
       - name: Initialize database
-        run: pnpm run prisma -- migrate reset --force
+        run: pnpm run db:init
 
       - name: Start stack-backend in background
         uses: JarvusInnovations/background-action@v1.0.7

.github/workflows/e2e-source-of-truth-api-tests.yaml

@@ -19,6 +19,7 @@ jobs:
       STACK_ENABLE_HARDCODED_PASSKEY_CHALLENGE_FOR_TESTING: yes
       STACK_OVERRIDE_SOURCE_OF_TRUTH: '{"type": "postgres", "connectionString": "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:5432/source-of-truth-db?schema=sot-schema"}'
       STACK_TEST_SOURCE_OF_TRUTH: true
+      STACK_DIRECT_DATABASE_CONNECTION_STRING: "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:5432/stackframe"
 
     strategy:
       matrix:
@@ -97,11 +98,14 @@ jobs:
       - name: Wait on Svix
         run: npx wait-on tcp:localhost:8113
 
-      - name: Initialize source of truth database
-        run: "STACK_DIRECT_DATABASE_CONNECTION_STRING='postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:5432/source-of-truth-db?schema=sot-schema' pnpm run prisma -- migrate reset --force --skip-seed"
+      - name: Create source-of-truth database and schema
+        run: |
+          psql postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:5432/postgres -c "CREATE DATABASE \"source-of-truth-db\";"
+          psql postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:5432/source-of-truth-db -c "CREATE SCHEMA \"sot-schema\";"
 
       - name: Initialize database
-        run: pnpm run prisma -- migrate reset --force
+        run: pnpm run db:init
+
       - name: Start stack-backend in background
         uses: JarvusInnovations/background-action@v1.0.7
         with:

README.md

@@ -163,10 +163,10 @@ pnpm run prisma studio
 
 ### Database migrations
 
-If you make changes to the Prisma schema, you need to run the following command to create a migration:
+If you make changes to the Prisma schema, you need to run the following command to create a migration file:
 
 ```sh
-pnpm run prisma migrate dev
+pnpm run db:migration-gen
 ```
 
 ### Chat with the codebase

apps/backend/package.json

@@ -17,12 +17,19 @@
     "codegen-prisma:watch": "pnpm run prisma generate --watch",
     "codegen-route-info": "pnpm run with-env tsx scripts/generate-route-info.ts",
     "codegen-route-info:watch": "pnpm run with-env tsx watch --clear-screen=false scripts/generate-route-info.ts",
-    "codegen": "pnpm run codegen-prisma && pnpm run codegen-route-info",
+    "codegen": "pnpm run with-env pnpm run generate-migration-imports && pnpm run with-env bash -c 'if [ \"$STACK_ACCELERATE_ENABLED\" = \"true\" ]; then pnpm run prisma generate --no-engine && pnpm run generate-openapi; else pnpm run codegen-prisma && pnpm run generate-openapi; fi' && pnpm run codegen-route-info",
     "codegen:watch": "concurrently -n \"prisma,docs,route-info\" -k \"pnpm run codegen-prisma:watch\" \"pnpm run watch-docs\" \"pnpm run codegen-route-info:watch\"",
     "psql-inner": "psql $STACK_DATABASE_CONNECTION_STRING",
     "psql": "pnpm run with-env pnpm run psql-inner",
-    "prisma": "pnpm run with-env prisma",
     "prisma-studio": "pnpm run with-env prisma studio --port 8106 --browser none",
+    "prisma": "pnpm run with-env prisma",
+    "db:migration-gen": "pnpm run with-env tsx scripts/db-migrations.ts generate-migration-file",
+    "db:reset": "pnpm run with-env tsx scripts/db-migrations.ts reset",
+    "db:seed": "pnpm run with-env tsx scripts/db-migrations.ts seed",
+    "db:init": "pnpm run with-env tsx scripts/db-migrations.ts init",
+    "db:migrate": "pnpm run with-env tsx scripts/db-migrations.ts migrate",
+    "generate-migration-imports": "pnpm run with-env tsx scripts/generate-migration-imports.ts",
+    "generate-migration-imports:watch": "chokidar 'prisma/migrations/**/*.sql' -c 'pnpm run generate-migration-imports'",
     "lint": "next lint",
     "watch-docs": "pnpm run with-env bash -c 'tsx watch --clear-screen=false scripts/generate-openapi-fumadocs.ts && pnpm run --filter=@stackframe/stack-docs generate-openapi-docs'",
     "generate-openapi": "pnpm run with-env tsx scripts/generate-openapi.ts",
@@ -62,6 +69,8 @@
     "@vercel/otel": "^1.10.4",
     "ai": "^4.3.17",
     "bcrypt": "^5.1.1",
+    "chokidar-cli": "^3.0.0",
+    "dotenv": "^16.4.5",
     "dotenv-cli": "^7.3.0",
     "freestyle-sandboxes": "^0.0.92",
     "jose": "^5.2.2",
@@ -70,13 +79,15 @@
     "nodemailer": "^6.9.10",
     "oidc-provider": "^8.5.1",
     "openid-client": "5.6.4",
+    "postgres": "^3.4.5",
     "pg": "^8.16.3",
     "posthog-node": "^4.1.0",
     "react": "19.0.0",
     "react-dom": "19.0.0",
     "semver": "^7.6.3",
     "sharp": "^0.32.6",
     "svix": "^1.25.0",
+    "vite": "^6.1.0",
     "yaml": "^2.4.5",
     "yup": "^1.4.0",
     "zod": "^3.23.8"

apps/backend/prisma/migrations/20240910211533_remove_shared_facebook/migration.sql

@@ -37,10 +37,9 @@ DELETE FROM "ProxiedOAuthProviderConfig"
 WHERE "type" = 'FACEBOOK';
 
 -- AlterEnum
-BEGIN;
+-- SPLIT_STATEMENT_SENTINEL
 CREATE TYPE "ProxiedOAuthProviderType_new" AS ENUM ('GITHUB', 'GOOGLE', 'MICROSOFT', 'SPOTIFY');
 ALTER TABLE "ProxiedOAuthProviderConfig" ALTER COLUMN "type" TYPE "ProxiedOAuthProviderType_new" USING ("type"::text::"ProxiedOAuthProviderType_new");
 ALTER TYPE "ProxiedOAuthProviderType" RENAME TO "ProxiedOAuthProviderType_old";
 ALTER TYPE "ProxiedOAuthProviderType_new" RENAME TO "ProxiedOAuthProviderType";
 DROP TYPE "ProxiedOAuthProviderType_old";
-COMMIT;

apps/backend/prisma/migrations/20250304200822_add_project_user_count/migration.sql

@@ -8,6 +8,8 @@ UPDATE "Project" SET "userCount" = (
 );
 
 -- Create function to update userCount
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
 CREATE OR REPLACE FUNCTION update_project_user_count()
 RETURNS TRIGGER AS $$
 BEGIN
@@ -30,6 +32,7 @@ BEGIN
     RETURN NULL;
 END;
 $$ LANGUAGE plpgsql;
+-- SPLIT_STATEMENT_SENTINEL
 
 -- Create triggers
 DROP TRIGGER IF EXISTS project_user_insert_trigger ON "ProjectUser";

apps/backend/prisma/migrations/20250325235813_project_user_permissions/migration.sql

@@ -5,13 +5,12 @@
 
 */
 -- AlterEnum
-BEGIN;
 CREATE TYPE "PermissionScope_new" AS ENUM ('PROJECT', 'TEAM');
 ALTER TABLE "Permission" ALTER COLUMN "scope" TYPE "PermissionScope_new" USING ("scope"::text::"PermissionScope_new");
 ALTER TYPE "PermissionScope" RENAME TO "PermissionScope_old";
 ALTER TYPE "PermissionScope_new" RENAME TO "PermissionScope";
 DROP TYPE "PermissionScope_old";
-COMMIT;
+-- SPLIT_STATEMENT_SENTINEL
 
 -- AlterTable
 ALTER TABLE "Permission" ADD COLUMN     "isDefaultProjectPermission" BOOLEAN NOT NULL DEFAULT false;

apps/backend/prisma/migrations/20250425171311_remove_old_config/migration.sql

@@ -167,17 +167,14 @@ ALTER TABLE "AuthMethod" DROP COLUMN "authMethodConfigId",
 DROP COLUMN "projectConfigId";
 
 -- AlterTable
-BEGIN;
 ALTER TABLE "ConnectedAccount" ADD COLUMN "configOAuthProviderId" TEXT;
 UPDATE "ConnectedAccount" SET "configOAuthProviderId" = "oauthProviderConfigId";
 ALTER TABLE "ConnectedAccount" ALTER COLUMN "configOAuthProviderId" SET NOT NULL;
 ALTER TABLE "ConnectedAccount" DROP COLUMN "oauthProviderConfigId";
 ALTER TABLE "ConnectedAccount" DROP COLUMN "connectedAccountConfigId";
 ALTER TABLE "ConnectedAccount" DROP COLUMN "projectConfigId";
-COMMIT;
 
 -- AlterTable
-BEGIN;
 ALTER TABLE "EmailTemplate" DROP CONSTRAINT "EmailTemplate_pkey";
 ALTER TABLE "EmailTemplate" ADD COLUMN "projectId" TEXT;
 
@@ -189,12 +186,15 @@ JOIN "Project" P ON P."configId" = PC."id"
 WHERE ET."projectConfigId" = PC."id";
 
 -- Check if we have any null projectId values
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
 DO $$
 BEGIN
     IF EXISTS (SELECT 1 FROM "EmailTemplate" WHERE "projectId" IS NULL) THEN
         RAISE EXCEPTION 'Some EmailTemplate records have null projectId values after migration';
     END IF;
 END $$;
+-- SPLIT_STATEMENT_SENTINEL
 
 -- Now make the column NOT NULL
 ALTER TABLE "EmailTemplate" ALTER COLUMN "projectId" SET NOT NULL;
@@ -204,38 +204,30 @@ ALTER TABLE "EmailTemplate" ADD CONSTRAINT "EmailTemplate_pkey" PRIMARY KEY ("pr
 
 -- Drop the old column
 ALTER TABLE "EmailTemplate" DROP COLUMN "projectConfigId";
-COMMIT;
 
 -- AlterTable
-BEGIN;
 ALTER TABLE "OAuthAccessToken" ADD COLUMN "configOAuthProviderId" TEXT;
 UPDATE "OAuthAccessToken" SET "configOAuthProviderId" = "oAuthProviderConfigId";
 ALTER TABLE "OAuthAccessToken" ALTER COLUMN "configOAuthProviderId" SET NOT NULL;
 ALTER TABLE "OAuthAccessToken" DROP COLUMN "oAuthProviderConfigId";
-COMMIT;
 
 -- AlterTable
-BEGIN;
 ALTER TABLE "OAuthAuthMethod" ADD COLUMN "configOAuthProviderId" TEXT;
 UPDATE "OAuthAuthMethod" SET "configOAuthProviderId" = "oauthProviderConfigId";
 ALTER TABLE "OAuthAuthMethod" ALTER COLUMN "configOAuthProviderId" SET NOT NULL;
 ALTER TABLE "OAuthAuthMethod" DROP COLUMN "oauthProviderConfigId";
 ALTER TABLE "OAuthAuthMethod" DROP COLUMN "projectConfigId";
-COMMIT;
 
 -- AlterTable
-BEGIN;
 ALTER TABLE "OAuthToken" ADD COLUMN "configOAuthProviderId" TEXT;
 UPDATE "OAuthToken" SET "configOAuthProviderId" = "oAuthProviderConfigId";
 ALTER TABLE "OAuthToken" ALTER COLUMN "configOAuthProviderId" SET NOT NULL;
 ALTER TABLE "OAuthToken" DROP COLUMN "oAuthProviderConfigId";
-COMMIT;
 
 -- AlterTable
 ALTER TABLE "Project" DROP COLUMN "configId";
 
 -- AlterTable
-BEGIN;
 ALTER TABLE "ProjectUserDirectPermission" ADD COLUMN "permissionId" TEXT;
 
 -- Update permissionId with values from Permission table
@@ -249,24 +241,22 @@ ALTER TABLE "ProjectUserDirectPermission" ALTER COLUMN "permissionId" SET NOT NU
 
 -- Drop the old column
 ALTER TABLE "ProjectUserDirectPermission" DROP COLUMN "permissionDbId";
-COMMIT;
 
 -- AlterTable
-BEGIN;
 ALTER TABLE "ProjectUserOAuthAccount" ADD COLUMN "configOAuthProviderId" TEXT;
 UPDATE "ProjectUserOAuthAccount" SET "configOAuthProviderId" = "oauthProviderConfigId";
 ALTER TABLE "ProjectUserOAuthAccount" ALTER COLUMN "configOAuthProviderId" SET NOT NULL;
 ALTER TABLE "ProjectUserOAuthAccount" DROP CONSTRAINT "ProjectUserOAuthAccount_pkey";
 ALTER TABLE "ProjectUserOAuthAccount" DROP COLUMN "oauthProviderConfigId";
 ALTER TABLE "ProjectUserOAuthAccount" DROP COLUMN "projectConfigId";
 ALTER TABLE "ProjectUserOAuthAccount" ADD CONSTRAINT "ProjectUserOAuthAccount_pkey" PRIMARY KEY ("tenancyId", "configOAuthProviderId", "providerAccountId");
-COMMIT;
 
 -- AlterTable
-BEGIN;
 ALTER TABLE "TeamMemberDirectPermission" ADD COLUMN "permissionId" TEXT;
 
 -- Check for rows where both or neither field is populated
+-- SPLIT_STATEMENT_SENTINEL
+-- SINGLE_STATEMENT_SENTINEL
 DO $$
 BEGIN
     IF EXISTS (
@@ -277,6 +267,7 @@ BEGIN
         RAISE EXCEPTION 'Invalid state: Each TeamMemberDirectPermission must have exactly one of permissionDbId or systemPermission set';
     END IF;
 END $$;
+-- SPLIT_STATEMENT_SENTINEL
 
 -- Update permissionId using systemPermission when available
 UPDATE "TeamMemberDirectPermission"
@@ -295,7 +286,6 @@ ALTER TABLE "TeamMemberDirectPermission" ALTER COLUMN "permissionId" SET NOT NUL
 -- Then drop the old columns
 ALTER TABLE "TeamMemberDirectPermission" DROP COLUMN "permissionDbId";
 ALTER TABLE "TeamMemberDirectPermission" DROP COLUMN "systemPermission";
-COMMIT;
 
 -- DropTable
 DROP TABLE "AuthMethodConfig";

apps/backend/prisma/seed.ts

@@ -58,7 +58,7 @@ async function seed() {
   }
 
   const internalTenancy = await getSoleTenancyFromProjectBranch("internal", DEFAULT_BRANCH_ID);
-  const internalPrisma = getPrismaClientForTenancy(internalTenancy);
+  const internalPrisma = await getPrismaClientForTenancy(internalTenancy);
 
   internalProject = await createOrUpdateProject({
     projectId: 'internal',
@@ -142,7 +142,7 @@ async function seed() {
       }
 
       if (adminGithubId) {
-        const githubAccount = await getPrismaClientForTenancy(internalTenancy).projectUserOAuthAccount.findFirst({
+        const githubAccount = await internalPrisma.projectUserOAuthAccount.findFirst({
           where: {
             tenancyId: internalTenancy.id,
             configOAuthProviderId: 'github',
@@ -153,7 +153,7 @@ async function seed() {
         if (githubAccount) {
           console.log(`GitHub account already exists, skipping creation`);
         } else {
-          await getPrismaClientForTenancy(internalTenancy).projectUserOAuthAccount.create({
+          await internalPrisma.projectUserOAuthAccount.create({
             data: {
               tenancyId: internalTenancy.id,
               projectUserId: newUser.projectUserId,