Fix cron jobs using dev env instead of test env in CI workflows (#1319)

The custom-base-port and db-migration-backwards-compatibility workflows
were running cron jobs with `with-env:dev` instead of `with-env:test`,
causing ClickHouse sync mismatches in verify-data-integrity.

<!--

Make sure you've read the CONTRIBUTING.md guidelines:
https://github.com/stack-auth/stack-auth/blob/dev/CONTRIBUTING.md

-->


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **Chores**
* Streamlined CI test workflows to standardize background cron job
startup for more consistent test runs.
* **Tests**
* Improved end-to-end test reliability by aligning background process
behavior across suites.
* **Bug Fixes**
* Enhanced data verification reliability by ensuring external database
sync before integrity checks and tightening comparison ordering for
certain records, reducing false mismatch detections.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Author: BilalG1

.github/workflows/db-migration-backwards-compatibility.yaml

@@ -200,7 +200,7 @@ jobs:
         uses: JarvusInnovations/background-action@v1.0.7
         if: ${{ hashFiles('apps/backend/scripts/run-cron-jobs.ts') != '' }}
         with:
-          run: pnpm -C apps/backend run with-env:dev tsx scripts/run-cron-jobs.ts --log-order=stream &
+          run: pnpm -C apps/backend run run-cron-jobs:test --log-order=stream &
           wait-on: |
             http://localhost:8102
           tail: true
@@ -394,7 +394,7 @@ jobs:
         uses: JarvusInnovations/background-action@v1.0.7
         if: ${{ hashFiles('apps/backend/scripts/run-cron-jobs.ts') != '' }}
         with:
-          run: pnpm -C apps/backend run with-env:dev tsx scripts/run-cron-jobs.ts --log-order=stream &
+          run: pnpm -C apps/backend run run-cron-jobs:test --log-order=stream &
           wait-on: |
             http://localhost:8102
           tail: true

.github/workflows/e2e-custom-base-port-api-tests.yaml

@@ -145,7 +145,7 @@ jobs:
       - name: Start run-cron-jobs in background
         uses: JarvusInnovations/background-action@v1.0.7
         with:
-          run: pnpm -C apps/backend run run-cron-jobs --log-order=stream &
+          run: pnpm -C apps/backend run run-cron-jobs:test --log-order=stream &
           wait-on: |
             http://localhost:6702
           tail: true

apps/backend/scripts/verify-data-integrity/clickhouse-sync-verifier.ts

@@ -18,7 +18,7 @@ const SORT_KEYS = {
   team_invitations: ["id"],
   email_outboxes: ["id"],
   project_permissions: ["user_id", "permission_id"],
-  notification_preferences: ["id"],
+  notification_preferences: ["user_id", "notification_category_id"],
   refresh_tokens: ["id"],
   connected_accounts: ["user_id", "provider", "provider_account_id"],
 } satisfies Record<keyof typeof DEFAULT_DB_SYNC_MAPPINGS, string[]>;

apps/backend/scripts/verify-data-integrity/index.ts

@@ -1,3 +1,4 @@
+import { syncExternalDatabases } from "@/lib/external-db-sync";
 import { DEFAULT_BRANCH_ID, getSoleTenancyFromProjectBranch } from "@/lib/tenancies";
 import { getPrismaClientForTenancy, globalPrismaClient } from "@/prisma-client";
 import type { OrganizationRenderedConfig } from "@stackframe/stack-shared/dist/config/schema";
@@ -228,6 +229,10 @@ async function main() {
 
       if (!shouldSkipClickhouse && clickhouseAvailable && tenancy) {
         await recurse("[clickhouse sync]", async (recurse) => {
+          // Flush any pending ClickHouse syncs by running a direct sync before verifying.
+          // This avoids race conditions where QStash hasn't delivered all sync callbacks yet.
+          await syncExternalDatabases(tenancy);
+
           await verifyClickhouseSync({
             tenancy,
             projectId,