refactor: Add dedicated setup-db job to eliminate cache race condition (calcom#26171)

* refactor: Add dedicated setup-db job to eliminate cache race condition

- Create new setup-db.yml workflow that runs cache-db action
- Update pr.yml to add setup-db job that runs before all E2E jobs
- Update integration-test, e2e, e2e-api-v2, e2e-app-store, e2e-embed,
  and e2e-embed-react jobs to depend on setup-db instead of build-api-v1
- Add setup-db to required job needs list and result check

This eliminates the race condition where multiple jobs could try to
write to the same database cache simultaneously.

Co-Authored-By: keith@cal.com <keithwillcode@gmail.com>

* refactor: Remove DB parts from api-v1-production-build.yml

The database setup is now handled by the dedicated setup-db job,
so the postgres service and cache-db action are no longer needed
in the API v1 build workflow.

Co-Authored-By: keith@cal.com <keithwillcode@gmail.com>

* Apply suggestion from @keithwillcode

* Made integration tests dependency consistent

* refactor: Remove unnecessary env vars from setup-db.yml

Only keep the env vars needed for database setup:
- CALENDSO_ENCRYPTION_KEY (for db-seed)
- DATABASE_URL / DATABASE_DIRECT_URL (for database connection)
- TURBO_TOKEN / TURBO_TEAM (for turbo caching)

Co-Authored-By: keith@cal.com <keithwillcode@gmail.com>

* fix: Add back E2E_TEST_CALCOM_QA_* env vars needed by db-seed

These env vars are used by scripts/seed.ts to create the QA user
with Google Calendar credentials during database seeding.

Co-Authored-By: keith@cal.com <keithwillcode@gmail.com>

* fix: Restore postgres service and cache-db to api-v1-production-build.yml

The API v1 build still needs a database to run properly. Restored the
postgres service and cache-db action, and added dependency on setup-db
so it waits for the database cache to be created first.

Co-Authored-By: keith@cal.com <keithwillcode@gmail.com>

* E2E for API v2 no longer waiting on API v2 build

* Reordering the jobs by dependencies

* add GOOGLE_API_CREDENTIALS env var to setup db

* Added back all env vars to setup-db

* fix: Include commit SHA in cache-db key to invalidate cache on new commits

Co-Authored-By: keith@cal.com <keithwillcode@gmail.com>

* refactor: Strip back setup-db.yml env vars to minimal set needed for db-seed

Co-Authored-By: keith@cal.com <keithwillcode@gmail.com>

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

Author: keithwillcode

.github/actions/cache-db/action.yml

@@ -7,6 +7,10 @@ inputs:
   path:
     required: false
     default: "backups/backup.sql"
+  COMMIT_SHA:
+    required: false
+    default: ""
+    description: "Commit SHA to include in cache key (passed from workflow)"
 runs:
   using: "composite"
   steps:
@@ -17,7 +21,7 @@ runs:
         cache-name: cache-db
         key-1: ${{ hashFiles('packages/prisma/schema.prisma', 'packages/prisma/migrations/**/**.sql', 'packages/prisma/*.ts') }}
         key-2: ${{ github.event.pull_request.number || github.ref }}
-        key-3: ${{ github.event.pull_request.head.sha }}
+        key-3: ${{ inputs.COMMIT_SHA || github.event.pull_request.head.sha || github.sha }}
         DATABASE_URL: ${{ inputs.DATABASE_URL }}
         DATABASE_DIRECT_URL: ${{ inputs.DATABASE_URL }}
         E2E_TEST_CALCOM_QA_EMAIL: ${{ inputs.E2E_TEST_CALCOM_QA_EMAIL }}

.github/workflows/e2e.yml

@@ -5,7 +5,6 @@ permissions:
   actions: write
   contents: read
 env:
-  NODE_OPTIONS: --max-old-space-size=4096
   ALLOWED_HOSTNAMES: ${{ vars.CI_ALLOWED_HOSTNAMES }}
   CALENDSO_ENCRYPTION_KEY: ${{ secrets.CI_CALENDSO_ENCRYPTION_KEY }}
   DAILY_API_KEY: ${{ secrets.CI_DAILY_API_KEY }}
@@ -15,11 +14,11 @@ env:
   E2E_TEST_APPLE_CALENDAR_EMAIL: ${{ secrets.E2E_TEST_APPLE_CALENDAR_EMAIL }}
   E2E_TEST_APPLE_CALENDAR_PASSWORD: ${{ secrets.E2E_TEST_APPLE_CALENDAR_PASSWORD }}
   E2E_TEST_MAILHOG_ENABLED: ${{ vars.E2E_TEST_MAILHOG_ENABLED }}
-  GOOGLE_API_CREDENTIALS: ${{ secrets.CI_GOOGLE_API_CREDENTIALS }}
   EMAIL_SERVER_HOST: ${{ secrets.CI_EMAIL_SERVER_HOST }}
   EMAIL_SERVER_PORT: ${{ secrets.CI_EMAIL_SERVER_PORT }}
   EMAIL_SERVER_USER: ${{ secrets.CI_EMAIL_SERVER_USER }}
   EMAIL_SERVER_PASSWORD: ${{ secrets.CI_EMAIL_SERVER_PASSWORD}}
+  GOOGLE_API_CREDENTIALS: ${{ secrets.CI_GOOGLE_API_CREDENTIALS }}
   GOOGLE_LOGIN_ENABLED: ${{ vars.CI_GOOGLE_LOGIN_ENABLED }}
   NEXTAUTH_SECRET: ${{ secrets.CI_NEXTAUTH_SECRET }}
   NEXTAUTH_URL: ${{ secrets.CI_NEXTAUTH_URL }}
@@ -30,6 +29,7 @@ env:
   NEXT_PUBLIC_STRIPE_PUBLIC_KEY: ${{ secrets.CI_NEXT_PUBLIC_STRIPE_PUBLIC_KEY }}
   NEXT_PUBLIC_WEBAPP_URL: ${{ vars.CI_NEXT_PUBLIC_WEBAPP_URL }}
   NEXT_PUBLIC_WEBSITE_URL: ${{ vars.CI_NEXT_PUBLIC_WEBSITE_URL }}
+  NODE_OPTIONS: --max-old-space-size=4096
   PAYMENT_FEE_FIXED: ${{ vars.CI_PAYMENT_FEE_FIXED }}
   PAYMENT_FEE_PERCENTAGE: ${{ vars.CI_PAYMENT_FEE_PERCENTAGE }}
   SAML_ADMINS: ${{ secrets.CI_SAML_ADMINS }}

.github/workflows/pr.yml

@@ -139,9 +139,18 @@ jobs:
     uses: ./.github/workflows/unit-tests.yml
     secrets: inherit
 
+  setup-db:
+    name: Setup Database
+    needs: [prepare, deps]
+    if: ${{ needs.prepare.outputs.run-e2e == 'true' && needs.prepare.outputs.has-files-requiring-all-checks == 'true' }}
+    uses: ./.github/workflows/setup-db.yml
+    with:
+      COMMIT_SHA: ${{ needs.prepare.outputs.commit-sha }}
+    secrets: inherit
+
   build-api-v1:
     name: Production builds
-    needs: [prepare, deps]
+    needs: [prepare, deps, setup-db]
     if: ${{ needs.prepare.outputs.run-e2e == 'true' && needs.prepare.outputs.has-files-requiring-all-checks == 'true' }}
     uses: ./.github/workflows/api-v1-production-build.yml
     secrets: inherit
@@ -183,42 +192,42 @@ jobs:
 
   integration-test:
     name: Tests
-    needs: [prepare, build, build-api-v1]
-    if: ${{ needs.prepare.outputs.has-files-requiring-all-checks == 'true' }}
+    needs: [prepare, build, setup-db]
+    if: ${{ needs.prepare.outputs.run-e2e == 'true' && needs.prepare.outputs.has-files-requiring-all-checks == 'true' }}
     uses: ./.github/workflows/integration-tests.yml
     secrets: inherit
 
   e2e:
     name: Tests
-    needs: [prepare, build, build-api-v1]
+    needs: [prepare, build, setup-db]
     if: ${{ needs.prepare.outputs.run-e2e == 'true' && needs.prepare.outputs.has-files-requiring-all-checks == 'true' }}
     uses: ./.github/workflows/e2e.yml
     secrets: inherit
 
   e2e-api-v2:
     name: Tests
-    needs: [prepare, build-api-v2]
+    needs: [prepare, setup-db]
     if: ${{ needs.prepare.outputs.run-e2e == 'true' && needs.prepare.outputs.has-files-requiring-all-checks == 'true' }}
     uses: ./.github/workflows/e2e-api-v2.yml
     secrets: inherit
 
   e2e-app-store:
     name: Tests
-    needs: [prepare, build]
+    needs: [prepare, build, setup-db]
     if: ${{ needs.prepare.outputs.run-e2e == 'true' && needs.prepare.outputs.has-files-requiring-all-checks == 'true' }}
     uses: ./.github/workflows/e2e-app-store.yml
     secrets: inherit
 
   e2e-embed:
     name: Tests
-    needs: [prepare, build]
+    needs: [prepare, build, setup-db]
     if: ${{ needs.prepare.outputs.run-e2e == 'true' && needs.prepare.outputs.has-files-requiring-all-checks == 'true' }}
     uses: ./.github/workflows/e2e-embed.yml
     secrets: inherit
 
   e2e-embed-react:
     name: Tests
-    needs: [prepare, build]
+    needs: [prepare, build, setup-db]
     if: ${{ needs.prepare.outputs.run-e2e == 'true' && needs.prepare.outputs.has-files-requiring-all-checks == 'true' }}
     uses: ./.github/workflows/e2e-embed-react.yml
     secrets: inherit
@@ -243,6 +252,7 @@ jobs:
         build-atoms,
         build-docs,
         build-companion,
+        setup-db,
         e2e,
         e2e-api-v2,
         e2e-embed,
@@ -266,6 +276,7 @@ jobs:
               needs.build-api-v2.result != 'success' ||
               needs.build-atoms.result != 'success' ||
               needs.build-docs.result != 'success' ||
+              needs.setup-db.result != 'success' ||
               needs.integration-test.result != 'success' ||
               needs.e2e.result != 'success' ||
               needs.e2e-api-v2.result != 'success' ||

.github/workflows/setup-db.yml

@@ -0,0 +1,54 @@
+name: Setup Database
+
+on:
+  workflow_call:
+    inputs:
+      COMMIT_SHA:
+        required: false
+        type: string
+        default: ""
+        description: "Commit SHA to include in cache key"
+
+permissions:
+  contents: read
+
+env:
+  CALENDSO_ENCRYPTION_KEY: ${{ secrets.CI_CALENDSO_ENCRYPTION_KEY }}
+  DATABASE_URL: ${{ secrets.CI_DATABASE_URL }}
+  DATABASE_DIRECT_URL: ${{ secrets.CI_DATABASE_URL }}
+  E2E_TEST_CALCOM_QA_EMAIL: ${{ secrets.E2E_TEST_CALCOM_QA_EMAIL }}
+  E2E_TEST_CALCOM_QA_PASSWORD: ${{ secrets.E2E_TEST_CALCOM_QA_PASSWORD }}
+  E2E_TEST_CALCOM_QA_GCAL_CREDENTIALS: ${{ secrets.E2E_TEST_CALCOM_QA_GCAL_CREDENTIALS }}
+  GOOGLE_API_CREDENTIALS: ${{ secrets.CI_GOOGLE_API_CREDENTIALS }}
+  TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
+  TURBO_TEAM: ${{ secrets.TURBO_TEAM }}
+
+jobs:
+  setup-db:
+    name: Setup Database
+    runs-on: buildjet-4vcpu-ubuntu-2204
+    timeout-minutes: 15
+    services:
+      postgres:
+        image: postgres:13
+        credentials:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: calendso
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/dangerous-git-checkout
+      - uses: ./.github/actions/yarn-install
+      - uses: ./.github/actions/cache-db
+        with:
+          COMMIT_SHA: ${{ inputs.COMMIT_SHA || github.sha }}