[#438] Add DB CI Migration to remove Dependency on Dev Lead (#439)

Author: DVidal1205

.github/PULL_REQUEST_TEMPLATE

@@ -33,8 +33,9 @@ This could include:
 ## Checklist
 
 <!--
-IMPORTANT: Non-automated step. Make sure to confirm this, or Prod will go down.
+IMPORTANT: Schema changes must include committed migration files.
+If production auto-migrations are still disabled, coordinate the rollout plan before merge.
 -->
 
-- [ ] Database: No schema changes, OR I have contacted the Development Lead to run `db:push` before merging
+- [ ] Database: No schema changes, OR I ran `pnpm db:generate` and committed the generated files in `packages/db/drizzle/`
 - [ ] Environment Variables: No environment variables changed, OR I have contacted the Development Lead to modify them on Coolify BEFORE merging.

.github/workflows/ci.yml

@@ -15,6 +15,115 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
 jobs:
+  migration_check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup
+        uses: ./tooling/github/setup
+
+      - name: Generate migrations
+        env:
+          DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres
+        run: pnpm db:generate
+
+      - name: Verify migrations are committed
+        run: |
+          if [ -n "$(git status --porcelain -- packages/db/drizzle)" ]; then
+            echo "Detected uncommitted migration changes in packages/db/drizzle."
+            echo "Run 'pnpm db:generate' and commit the generated files."
+            git status --short -- packages/db/drizzle
+            exit 1
+          fi
+
+  migration_apply_fresh:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: forge
+        options: >-
+          --health-cmd="pg_isready -U postgres -d forge"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=5
+        ports:
+          - 5432:5432
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup
+        uses: ./tooling/github/setup
+
+      - name: Apply migrations on a fresh database
+        env:
+          DATABASE_URL: postgresql://postgres:postgres@localhost:5432/forge
+        run: |
+          pnpm db:migrate
+          pnpm db:migrate
+
+  migration_upgrade_smoke:
+    if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && vars.ENABLE_DB_UPGRADE_SMOKE == 'true'
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: local
+        options: >-
+          --health-cmd="pg_isready -U postgres -d forge"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=5
+        ports:
+          - 5432:5432
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup
+        uses: ./tooling/github/setup
+
+      - name: Create base branch worktree
+        run: git worktree add /tmp/forge-base "${{ github.event.pull_request.base.sha }}"
+
+      - name: Install base branch dependencies
+        working-directory: /tmp/forge-base
+        run: pnpm install --ignore-scripts
+
+      - name: Materialize base branch schema
+        working-directory: /tmp/forge-base
+        env:
+          DATABASE_URL: postgresql://postgres:postgres@localhost:5432/local
+        run: |
+          if node -e "const pkg = require('./package.json'); process.exit(pkg.scripts['db:migrate'] ? 0 : 1)"; then
+            pnpm db:migrate
+          else
+            pnpm --filter=@forge/db push
+          fi
+
+      - name: Restore filtered prod-like data
+        env:
+          DATABASE_URL: postgresql://postgres:postgres@localhost:5432/local
+          MINIO_ENDPOINT: ${{ secrets.MINIO_ENDPOINT }}
+          MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }}
+          MINIO_SECRET_KEY: ${{ secrets.MINIO_SECRET_KEY }}
+        run: pnpm db:pull --truncate
+
+      - name: Apply branch migrations on prod-like data
+        env:
+          DATABASE_URL: postgresql://postgres:postgres@localhost:5432/local
+        run: |
+          pnpm db:migrate
+          pnpm db:migrate
+
   lint:
     runs-on: ubuntu-latest
     steps:
@@ -50,7 +159,7 @@ jobs:
 
   build:
     runs-on: ubuntu-latest
-    needs: [lint, format, typecheck]
+    needs: [migration_check, migration_apply_fresh, lint, format, typecheck]
     permissions:
       contents: read
     steps:
@@ -64,3 +173,23 @@ jobs:
 
       - name: Build
         run: pnpm build
+
+  migrate_prod:
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main' && vars.ENABLE_PROD_DB_MIGRATIONS == 'true'
+    runs-on: ubuntu-latest
+    needs: [build]
+    concurrency:
+      group: db-migrate-prod
+      cancel-in-progress: false
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup
+        uses: ./tooling/github/setup
+
+      - name: Apply database migrations
+        env:
+          DATABASE_URL: ${{ secrets.PROD_DATABASE_URL }}
+        run: pnpm db:migrate

CONTRIBUTING.md

@@ -23,7 +23,9 @@ If you are not a Knight Hacks dev team member, you can still contribute to Forge
 
 Install Docker Desktop if you don't already have it.
 
-To create a Postgres database locally with docker, you can run `docker compose up`. You will then need to push the schema to the database by running `pnpm db:push`.
+To create a Postgres database locally with Docker, run `docker compose up`. Then apply the committed schema migrations by running `pnpm db:migrate`.
+
+If you change any schema files in `packages/db/src/schemas`, generate and commit a migration with `pnpm db:generate`.
 
 To stop the Postgres container, run `docker compose stop`. To completely reset your database, run `docker compose down --volumes`.
 

docs/ARCHITECTURE.md

@@ -78,9 +78,11 @@ Database layer using Drizzle ORM with PostgreSQL.
 
 - Contains all database schemas
 - Exports the database client
-- Includes migration scripts
+- Includes committed SQL migrations and migration helper scripts
 - Located in `packages/db/src/schemas/`
 
+Local development applies schema changes with `pnpm db:migrate`. Schema edits should be followed by `pnpm db:generate`, and the generated files in `packages/db/drizzle/` are part of the reviewed source of truth.
+
 ### `@forge/auth`
 
 Authentication setup using Better Auth with Discord OAuth.

docs/GETTING-STARTED.md

@@ -75,12 +75,20 @@ docker compose up
 
 > IMPORTANT!
 
-You must push the database schema to your local database before running the project. This is a common source of errors for new contributors. The most common error for this will be a "Failed to get current session" error on any page within Blade.
+You must apply the committed migrations to your local database before running the project. This is a common source of errors for new contributors. The most common symptom is a "Failed to get current session" error on Blade pages.
 
 ```bash
-pnpm db:push
+pnpm db:migrate
 ```
 
+When you change files in `packages/db/src/schemas`, generate a new migration and commit it:
+
+```bash
+pnpm db:generate
+```
+
+Pull requests with schema changes are expected to include the generated files in `packages/db/drizzle/`. CI verifies that migrations are committed, that they apply on a fresh database, and that they can upgrade a prod-like sanitized dataset.
+
 **Optional:** View the database contents with Drizzle Studio:
 
 ```bash
@@ -119,7 +127,7 @@ After running this, you'll have full superadmin permissions and can manage other
 
 ### 6. (Optional) Populate Test Data from Production
 
-**Dev Team Members Only:** If you have access to the shared MinIO instance, you can pull a sanitized copy of production data to test with realistic data locally.
+**Dev Team Members Only:** If you have access to the shared MinIO instance, you can pull a sanitized copy of production data to test with realistic data locally. This restores data into the database currently pointed to by `DATABASE_URL`, so make sure you have already run `pnpm db:migrate` first.
 
 ```bash
 pnpm db:pull
@@ -191,11 +199,13 @@ Look for issues labeled [`Onboarding`](https://github.com/KnightHacks/forge/labe
 3. Test your changes locally
 4. Run checks before submitting:
    ```bash
+   pnpm db:migrate
    pnpm format
    pnpm lint
    pnpm typecheck
    pnpm build
    ```
+   If you changed any schema files, also run `pnpm db:generate` and commit the generated migration files.
 5. Commit your changes (use lowercase, descriptive commit messages)
 6. Push your branch and open a pull request
 

package.json

@@ -11,6 +11,8 @@
     "start": "turbo run start",
     "clean": "git clean -xdf node_modules",
     "clean:workspaces": "turbo run clean",
+    "db:generate": "pnpm --filter=@forge/db generate",
+    "db:migrate": "pnpm --filter=@forge/db migrate",
     "db:push": "turbo -F @forge/db push",
     "db:studio": "turbo -F @forge/db studio",
     "db:pull": "pnpm --filter=@forge/db with-env tsx scripts/get_prod_db.ts",

packages/db/README.md

@@ -0,0 +1,37 @@
+# Database Migrations (Drizzle)
+
+This package uses migration files committed to the repository.
+
+## Standard workflow
+
+1. Start a local Postgres database.
+2. Apply committed migrations:
+
+```bash
+pnpm db:migrate
+```
+
+3. Update schema files in `src/schemas/`.
+4. Generate migration SQL:
+
+```bash
+pnpm db:generate
+```
+
+5. Commit the generated files in `packages/db/drizzle/`.
+6. Open a PR. CI will fail if schema and migration files are out of sync and will smoke test applying migrations.
+7. After merge to `main`, CI can apply pending migrations with `pnpm db:migrate` once production is baselined and the deploy gate is enabled.
+
+## Commands
+
+- `pnpm db:generate`: Generate SQL migration files from schema changes.
+- `pnpm db:migrate`: Apply pending migrations using `DATABASE_URL`.
+- `pnpm db:pull`: Restore the filtered prod-like backup into the database at `DATABASE_URL`.
+- `pnpm db:push`: Directly push schema changes (emergency/non-standard only).
+
+## Notes
+
+- Migrations are the reviewable source of truth for DB changes.
+- Local development should use `pnpm db:migrate`, not `pnpm db:push`.
+- `pnpm db:pull` is intended for maintainers/CI jobs that have MinIO credentials.
+- Production migration apply job expects `PROD_DATABASE_URL` in GitHub secrets and maps it to `DATABASE_URL`.

packages/db/drizzle.config.ts

@@ -4,6 +4,7 @@ import { env } from "./src/env";
 
 export default {
   schema: "./src/schemas",
+  out: "./drizzle",
   dialect: "postgresql",
   dbCredentials: { url: env.DATABASE_URL },
   casing: "snake_case",