Reduce test flakeyness

N2D4 · N2D4 · commit 9bd9e00d454a · 2026-02-06T14:51:58.000-08:00
diff --git a/.github/workflows/e2e-api-tests.yaml b/.github/workflows/e2e-api-tests.yaml
@@ -19,7 +19,6 @@ jobs:
       NODE_ENV: test
       STACK_ENABLE_HARDCODED_PASSKEY_CHALLENGE_FOR_TESTING: yes
       STACK_DATABASE_CONNECTION_STRING: "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:8128/stackframe"
-      STACK_FORCE_EXTERNAL_DB_SYNC: "true"
       STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000"
       STACK_EXTERNAL_DB_SYNC_DIRECT: "false"
 
@@ -160,7 +159,7 @@ jobs:
         run: sleep 10
 
       - name: Run tests
-        run: pnpm test run ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }}
+        run: pnpm test run ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }} ${{ matrix.freestyle-mode == 'prod' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/dev' && 'mail' || '' }}
 
       - name: Run tests again (attempt 1)
         if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'
diff --git a/.github/workflows/e2e-custom-base-port-api-tests.yaml b/.github/workflows/e2e-custom-base-port-api-tests.yaml
@@ -19,7 +19,6 @@ jobs:
       STACK_ENABLE_HARDCODED_PASSKEY_CHALLENGE_FOR_TESTING: yes
       STACK_DATABASE_CONNECTION_STRING: "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:6728/stackframe"
       NEXT_PUBLIC_STACK_PORT_PREFIX: "67"
-      STACK_FORCE_EXTERNAL_DB_SYNC: "true"
       STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000"
       STACK_EXTERNAL_DB_SYNC_DIRECT: "false"
 
diff --git a/.github/workflows/e2e-source-of-truth-api-tests.yaml b/.github/workflows/e2e-source-of-truth-api-tests.yaml
@@ -21,7 +21,6 @@ jobs:
       STACK_OVERRIDE_SOURCE_OF_TRUTH: '{"type": "postgres", "connectionString": "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:8128/source-of-truth-db?schema=sot-schema"}'
       STACK_TEST_SOURCE_OF_TRUTH: true
       STACK_DATABASE_CONNECTION_STRING: "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:8128/stackframe"
-      STACK_FORCE_EXTERNAL_DB_SYNC: "true"
       STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000"
       STACK_EXTERNAL_DB_SYNC_DIRECT: "false"
 
diff --git a/.github/workflows/restart-dev-and-test-with-custom-base-port.yaml b/.github/workflows/restart-dev-and-test-with-custom-base-port.yaml
@@ -19,7 +19,6 @@ jobs:
     runs-on: ubicloud-standard-16
     env:
       NEXT_PUBLIC_STACK_PORT_PREFIX: "69"
-      STACK_FORCE_EXTERNAL_DB_SYNC: "true"
       STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000"
       STACK_EXTERNAL_DB_SYNC_DIRECT: "false"
 
diff --git a/.github/workflows/restart-dev-and-test.yaml b/.github/workflows/restart-dev-and-test.yaml
@@ -18,7 +18,6 @@ jobs:
   restart-dev-and-test:
     runs-on: ubicloud-standard-16
     env:
-      STACK_FORCE_EXTERNAL_DB_SYNC: "true"
       STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000"
       STACK_EXTERNAL_DB_SYNC_DIRECT: "false"
 
diff --git a/.github/workflows/setup-tests-with-custom-base-port.yaml b/.github/workflows/setup-tests-with-custom-base-port.yaml
@@ -19,7 +19,6 @@ jobs:
     runs-on: ubicloud-standard-16
     env:
       NEXT_PUBLIC_STACK_PORT_PREFIX: "69"
-      STACK_FORCE_EXTERNAL_DB_SYNC: "true"
       STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000"
       STACK_EXTERNAL_DB_SYNC_DIRECT: "false"
 
diff --git a/.github/workflows/setup-tests.yaml b/.github/workflows/setup-tests.yaml
@@ -18,7 +18,6 @@ jobs:
   setup-tests:
     runs-on: ubicloud-standard-16
     env:
-      STACK_FORCE_EXTERNAL_DB_SYNC: "true"
       STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000"
       STACK_EXTERNAL_DB_SYNC_DIRECT: "false"
     steps:
diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/poller/route.ts b/apps/backend/src/app/api/latest/internal/external-db-sync/poller/route.ts
@@ -30,13 +30,6 @@ function parseMaxDurationMs(value: string | undefined): number {
   return parsed;
 }
 
-function parseStopWhenIdle(value: string | undefined): boolean {
-  if (!value) return false;
-  if (value === "true") return true;
-  if (value === "false") return false;
-  throw new StatusError(400, "stopWhenIdle must be 'true' or 'false'");
-}
-
 function directSyncEnabled(): boolean {
   return getEnvVariable(DIRECT_SYNC_ENV, "") === "true";
 }
@@ -94,13 +87,11 @@ export const GET = createSmartRouteHandler({
     return await traceSpan("external-db-sync.poller", async (span) => {
       const startTime = performance.now();
       const maxDurationMs = parseMaxDurationMs(query.maxDurationMs);
-      const stopWhenIdle = parseStopWhenIdle(query.stopWhenIdle);
       const pollIntervalMs = 50;
       const staleClaimIntervalMinutes = 5;
       const pollerClaimLimit = getPollerClaimLimit();
 
       span.setAttribute("stack.external-db-sync.max-duration-ms", maxDurationMs);
-      span.setAttribute("stack.external-db-sync.stop-when-idle", stopWhenIdle);
       span.setAttribute("stack.external-db-sync.poll-interval-ms", pollIntervalMs);
       span.setAttribute("stack.external-db-sync.poller-claim-limit", pollerClaimLimit);
       span.setAttribute("stack.external-db-sync.direct-sync", directSyncEnabled());
@@ -258,8 +249,8 @@ export const GET = createSmartRouteHandler({
           const pendingRequests = await claimPendingRequests();
           iterationSpan.setAttribute("stack.external-db-sync.pending-count", pendingRequests.length);
 
-          if (stopWhenIdle && pendingRequests.length === 0) {
-            return { stopReason: "idle", processed: 0 };
+          if (pendingRequests.length > 0) {
+            console.log(`[Poller] Processing ${pendingRequests.length} pending requests`);
           }
 
           const processed = await processRequests(pendingRequests);
diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts b/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts
@@ -27,13 +27,6 @@ function parseMaxDurationMs(value: string | undefined): number {
   return parsed;
 }
 
-function parseStopWhenIdle(value: string | undefined): boolean {
-  if (!value) return false;
-  if (value === "true") return true;
-  if (value === "false") return false;
-  throw new StatusError(400, "stopWhenIdle must be 'true' or 'false'");
-}
-
 function getSequencerBatchSize(): number {
   const rawValue = getEnvVariable(SEQUENCER_BATCH_SIZE_ENV, "");
   if (!rawValue) return DEFAULT_BATCH_SIZE;
@@ -145,6 +138,10 @@ async function backfillSequenceIds(batchSize: number): Promise<boolean> {
 
     span.setAttribute("stack.external-db-sync.did-update", didUpdate);
 
+    if (didUpdate) {
+      console.log(`[Sequencer] Backfilled USR=${projectUserTenants.length} CC=${contactChannelTenants.length} DEL=${deletedRowTenants.length}`);
+    }
+
     return didUpdate;
   });
 }
@@ -169,7 +166,6 @@ export const GET = createSmartRouteHandler({
     }).defined(),
     query: yupObject({
       maxDurationMs: yupString().optional(),
-      stopWhenIdle: yupString().optional(),
     }).defined(),
   }),
   response: yupObject({
@@ -189,12 +185,10 @@ export const GET = createSmartRouteHandler({
     return await traceSpan("external-db-sync.sequencer", async (span) => {
       const startTime = performance.now();
       const maxDurationMs = parseMaxDurationMs(query.maxDurationMs);
-      const stopWhenIdle = parseStopWhenIdle(query.stopWhenIdle);
       const pollIntervalMs = 50;
       const batchSize = getSequencerBatchSize();
 
       span.setAttribute("stack.external-db-sync.max-duration-ms", maxDurationMs);
-      span.setAttribute("stack.external-db-sync.stop-when-idle", stopWhenIdle);
       span.setAttribute("stack.external-db-sync.poll-interval-ms", pollIntervalMs);
       span.setAttribute("stack.external-db-sync.batch-size", batchSize);
 
@@ -220,9 +214,6 @@ export const GET = createSmartRouteHandler({
           try {
             const didUpdate = await backfillSequenceIds(batchSize);
             iterationSpan.setAttribute("stack.external-db-sync.did-update", didUpdate);
-            if (stopWhenIdle && !didUpdate) {
-              return { stopReason: "idle" };
-            }
           } catch (error) {
             iterationSpan.setAttribute("stack.external-db-sync.iteration-error", true);
             captureError(
diff --git a/apps/e2e/tests/backend/backend-helpers.ts b/apps/e2e/tests/backend/backend-helpers.ts
@@ -186,6 +186,36 @@ export async function bumpEmailAddress(options: { unindexed?: boolean } = {}) {
   return mailbox;
 }
 
+// Helper to get emails from the outbox, filtered by subject if provided
+export async function getOutboxEmails(options?: { subject?: string }) {
+  const listResponse = await niceBackendFetch("/api/v1/emails/outbox", {
+    method: "GET",
+    accessType: "server",
+  });
+  if (options?.subject) {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    return listResponse.body.items.filter((e: any) => e.subject === options.subject);
+  }
+  return listResponse.body.items;
+}
+
+// Helper to poll the outbox until an email with the expected subject and status appears
+export async function waitForOutboxEmailWithStatus(subject: string, status: string) {
+  const maxRetries = 24;
+  let emails: any[] = [];
+  for (let i = 0; i < maxRetries; i++) {
+    emails = await getOutboxEmails({ subject });
+    if (emails.length > 0 && emails[0].status === status) {
+      return emails;
+    }
+    await wait(500);
+  }
+  throw new StackAssertionError(
+    `Timeout waiting for outbox email with subject "${subject}" and status "${status}"`,
+    { foundEmails: emails }
+  );
+}
+
 export namespace Auth {
   export async function fastSignUp(body: any = {}) {
     const { userId } = await User.create(body);
@@ -405,7 +435,11 @@ export namespace Auth {
         }
         await wait(100 + i * 20);
         if (i >= 30) {
-          throw new StackAssertionError(`Sign-in code message not found after ${i} attempts`, { response, messages: messages.map(m => ({ ...m, body: m.body && omit(m.body, ["html"]) })) });
+          throw new StackAssertionError(`Sign-in code message not found after ${i} attempts`, {
+            response,
+            messages: messages.map(m => ({ ...m, body: m.body && omit(m.body, ["html"]) })),
+            outboxEmails: await getOutboxEmails(),
+          });
         }
       }
       return {
diff --git a/apps/e2e/tests/backend/endpoints/api/v1/emails/email-helpers.ts b/apps/e2e/tests/backend/endpoints/api/v1/emails/email-helpers.ts
@@ -1,18 +1 @@
-import { niceBackendFetch } from "../../../../backend-helpers";
-
-/**
- * Helper to get emails from the outbox, filtered by subject if provided.
- * Shared across email test files to avoid duplication.
- */
-export async function getOutboxEmails(options?: { subject?: string }) {
-  const listResponse = await niceBackendFetch("/api/v1/emails/outbox", {
-    method: "GET",
-    accessType: "server",
-  });
-  if (options?.subject) {
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    return listResponse.body.items.filter((e: any) => e.subject === options.subject);
-  }
-  return listResponse.body.items;
-}
-
+export { getOutboxEmails, waitForOutboxEmailWithStatus } from "../../../../backend-helpers";
diff --git a/apps/e2e/tests/backend/endpoints/api/v1/emails/email-queue.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/emails/email-queue.test.ts
@@ -1,49 +1,10 @@
-import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors";
 import { wait } from "@stackframe/stack-shared/dist/utils/promises";
 import { deindent, nicify } from "@stackframe/stack-shared/dist/utils/strings";
 import beautify from "js-beautify";
 import { describe } from "vitest";
 import { it, logIfTestFails } from "../../../../../helpers";
 import { withPortPrefix } from "../../../../../helpers/ports";
-import { Auth, Project, User, backendContext, bumpEmailAddress, niceBackendFetch } from "../../../../backend-helpers";
-
-type OutboxEmail = {
-  id: string,
-  subject?: string,
-  status: string,
-  skipped_reason?: string,
-  is_transactional?: boolean,
-  tsx_source?: string,
-};
-
-// Helper to get emails from the outbox, filtered by subject if provided
-async function getOutboxEmails(options?: { subject?: string }): Promise<OutboxEmail[]> {
-  const listResponse = await niceBackendFetch("/api/v1/emails/outbox", {
-    method: "GET",
-    accessType: "server",
-  });
-  if (options?.subject) {
-    return listResponse.body.items.filter((e: any) => e.subject === options.subject);
-  }
-  return listResponse.body.items;
-}
-
-// Helper to poll the outbox until an email with the expected subject and status appears
-async function waitForOutboxEmailWithStatus(subject: string, status: string): Promise<OutboxEmail[]> {
-  const maxRetries = 20;
-  let emails: OutboxEmail[] = [];
-  for (let i = 0; i < maxRetries; i++) {
-    emails = await getOutboxEmails({ subject });
-    if (emails.length > 0 && emails[0].status === status) {
-      return emails;
-    }
-    await wait(500);
-  }
-  throw new StackAssertionError(
-    `Timeout waiting for outbox email with subject "${subject}" and status "${status}"`,
-    { foundEmails: emails }
-  );
-}
+import { Auth, Project, User, backendContext, bumpEmailAddress, getOutboxEmails, niceBackendFetch, waitForOutboxEmailWithStatus } from "../../../../backend-helpers";
 
 const testEmailConfig = {
   type: "standard",
diff --git a/apps/e2e/tests/backend/endpoints/api/v1/emails/outbox-api.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/emails/outbox-api.test.ts
diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-race.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-race.test.ts
diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts
diff --git a/apps/e2e/tests/backend/endpoints/api/v1/users.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/users.test.ts
