test(e2e): improve integration test reliability (#8422)

Author: jacekradko

.changeset/fix-e2e-compromised-password.md

@@ -0,0 +1,2 @@
+---
+---

integration/models/helpers.ts

@@ -67,6 +67,21 @@ const dedent = (strings: string | Array<string>, ...values: Array<string>) => {
 
 export const hash = () => randomBytes(5).toString('hex');
 
+/**
+ * Generates a strong, unique password for fake test users.
+ *
+ * Avoids any pattern derived from the user's email or other guessable inputs,
+ * so it doesn't collide with HIBP / compromised-password lists that would
+ * cause FAPI to reject sign-in with `form_password_compromised` (HTTP 422).
+ *
+ * Includes upper, lower, digit, and symbol to satisfy default Clerk password
+ * complexity rules.
+ */
+export const fakerPassword = () => {
+  const bytes = randomBytes(18).toString('base64url');
+  return `Aa1!${bytes}`;
+};
+
 export const waitUntilMessage = async (stream: Readable, message: string) => {
   return new Promise<void>(resolve => {
     stream.on('data', chunk => {

integration/testUtils/machineAuthHelpers.ts

@@ -220,9 +220,9 @@ export const registerApiKeyAuthTests = (adapter: MachineAuthTestAdapter): void =
     });
 
     test.afterAll(async () => {
-      await fakeAPIKey.revoke();
-      await fakeUser.deleteIfExists();
-      await app.teardown();
+      await fakeAPIKey?.revoke();
+      await fakeUser?.deleteIfExists();
+      await app?.teardown();
     });
 
     test('should return 401 if no API key is provided', async ({ request }) => {
@@ -311,8 +311,8 @@ export const registerM2MAuthTests = (adapter: MachineAuthTestAdapter): void => {
     });
 
     test.afterAll(async () => {
-      await network.cleanup();
-      await app.teardown();
+      await network?.cleanup();
+      await app?.teardown();
     });
 
     test('rejects requests with invalid M2M tokens', async ({ request }) => {
@@ -345,28 +345,6 @@ export const registerM2MAuthTests = (adapter: MachineAuthTestAdapter): void => {
       expect(body.tokenType).toBe(TokenType.M2MToken);
     });
 
-    test('authorizes after dynamically granting scope', async ({ page, context }) => {
-      const u = createTestUtils({ app, page, context });
-
-      await u.services.clerk.machines.createScope(network.unscopedSender.id, network.primaryServer.id);
-      const m2mToken = await u.services.clerk.m2m.createToken({
-        machineSecretKey: network.unscopedSender.secretKey,
-        secondsUntilExpiration: 60 * 30,
-      });
-
-      try {
-        const res = await u.page.request.get(new URL(adapter.m2m.path, app.serverUrl).toString(), {
-          headers: { Authorization: `Bearer ${m2mToken.token}` },
-        });
-        expect(res.status()).toBe(200);
-        const body = await res.json();
-        expect(body.subject).toBe(network.unscopedSender.id);
-        expect(body.tokenType).toBe(TokenType.M2MToken);
-      } finally {
-        await u.services.clerk.m2m.revokeToken({ m2mTokenId: m2mToken.id });
-      }
-    });
-
     test('verifies JWT format M2M token via local verification', async ({ request }) => {
       const jwtToken = await createJwtM2MToken(createMachineClient(), network.scopedSender.secretKey);
 
@@ -418,9 +396,9 @@ export const registerOAuthAuthTests = (adapter: MachineAuthTestAdapter): void =>
     });
 
     test.afterAll(async () => {
-      await fakeOAuth.cleanup();
-      await fakeUser.deleteIfExists();
-      await app.teardown();
+      await fakeOAuth?.cleanup();
+      await fakeUser?.deleteIfExists();
+      await app?.teardown();
     });
 
     test('verifies valid OAuth access token obtained through authorization flow', async ({ page, context }) => {

integration/testUtils/usersService.ts

@@ -1,7 +1,7 @@
 import type { APIKey, ClerkClient, Organization, User } from '@clerk/backend';
 import { faker } from '@faker-js/faker';
 
-import { hash } from '../models/helpers';
+import { fakerPassword, hash } from '../models/helpers';
 
 async function withErrorLogging<T>(operation: string, fn: () => Promise<T>): Promise<T> {
   try {
@@ -133,7 +133,7 @@ export const createUserService = (clerkClient: ClerkClient) => {
         lastName: faker.person.lastName(),
         email: withEmail ? email : undefined,
         username: withUsername ? `${randomHash}_clerk_cookie` : undefined,
-        password: withPassword ? `${email}${randomHash}` : undefined,
+        password: withPassword ? fakerPassword() : undefined,
         phoneNumber: withPhoneNumber ? phoneNumber : undefined,
         deleteIfExists: () => self.deleteIfExists({ email, phoneNumber }),
       };

integration/tests/components.test.ts

@@ -20,8 +20,8 @@ testAgainstRunningApps({ withEnv: [appConfigs.envs.withEmailCodes] })('component
 
   test.afterAll(async () => {
     await app.teardown();
-    await fakeUser.deleteIfExists();
-    await fakeOrganization.delete();
+    await fakeUser?.deleteIfExists();
+    await fakeOrganization?.delete();
   });
 
   const components = [

integration/tests/session-token-cache/multi-session.test.ts

@@ -246,9 +246,7 @@ testAgainstRunningApps({ withEnv: [appConfigs.envs.withSessionTasks] })(
      * this might be something we want to add in the future, but currently it is not
      * deterministic.
      */
-    test('multi-session scheduled refreshes produce one request per session', async ({ context }) => {
-      test.setTimeout(90_000);
-
+    test('cross-session token refreshes do not deduplicate', async ({ context }) => {
       const page1 = await context.newPage();
       await page1.goto(app.serverUrl);
       await page1.waitForFunction(() => (window as any).Clerk?.loaded);
@@ -297,7 +295,7 @@ testAgainstRunningApps({ withEnv: [appConfigs.envs.withSessionTasks] })(
       expect(user2SessionId).not.toBe(user1SessionId);
 
       // Tab1 has user1's active session; tab2 has user2's active session.
-      // Start counting /tokens requests.
+      // Start counting /tokens requests from here on.
       const refreshRequests: Array<{ sessionId: string; url: string }> = [];
       await context.route('**/v1/client/sessions/*/tokens*', async route => {
         const url = route.request().url();
@@ -306,23 +304,26 @@ testAgainstRunningApps({ withEnv: [appConfigs.envs.withSessionTasks] })(
         await route.continue();
       });
 
-      // Wait for proactive refresh timers to fire.
-      // Default token TTL is 60s; onRefresh fires at 60 - 15 - 2 = 43s from iat.
-      // Uses page.evaluate to avoid the global actionTimeout (10s) capping the wait.
-      await page1.evaluate(() => new Promise(resolve => setTimeout(resolve, 50_000)));
+      // Manually trigger a fresh /tokens fetch on each tab. Because the two
+      // tabs hold different sessions (different tokenIds), BroadcastChannel
+      // does NOT deduplicate across them — each tab is expected to make its
+      // own request.
+      const [page1Token, page2Token] = await Promise.all([
+        page1.evaluate(() => (window as any).Clerk.session?.getToken({ skipCache: true })),
+        page2.evaluate(() => (window as any).Clerk.session?.getToken({ skipCache: true })),
+      ]);
+
+      // Allow both broadcasts to settle.
+      // eslint-disable-next-line playwright/no-wait-for-timeout
+      await page1.waitForTimeout(500);
 
-      // Two different sessions should each produce exactly one refresh request.
-      // BroadcastChannel deduplication is per-tokenId, so different sessions refresh independently.
       expect(refreshRequests.length).toBe(2);
 
       const refreshedSessionIds = new Set(refreshRequests.map(r => r.sessionId));
       expect(refreshedSessionIds.has(user1SessionId)).toBe(true);
       expect(refreshedSessionIds.has(user2SessionId)).toBe(true);
 
-      // Both tabs should still have valid tokens after the refresh cycle
-      const page1Token = await page1.evaluate(() => (window as any).Clerk.session?.getToken());
-      const page2Token = await page2.evaluate(() => (window as any).Clerk.session?.getToken());
-
+      // Both tabs should hold valid, distinct tokens (different sessions).
       expect(page1Token).toBeTruthy();
       expect(page2Token).toBeTruthy();
       expect(page1Token).not.toBe(page2Token);

integration/tests/session-token-cache/single-session.test.ts

@@ -129,74 +129,15 @@ testAgainstRunningApps({ withEnv: [appConfigs.envs.withEmailCodes] })(
       expect(tokenRequests.length).toBe(1);
     });
 
-    /**
-     * Test Flow:
-     * 1. Open two tabs with the same browser context (shared cookies)
-     * 2. Sign in on tab1, reload tab2 to pick up the session
-     * 3. Both tabs hydrate their token cache with the session token
-     * 4. Start counting /tokens requests, then wait for the timers to fire
-     * 5. Assert only 1 /tokens request was made (not 2)
-     */
-    test('multi-tab scheduled refreshes are deduped to a single request', async ({ context }) => {
-      test.setTimeout(90_000);
-
-      const page1 = await context.newPage();
-      const page2 = await context.newPage();
-
-      await page1.goto(app.serverUrl);
-      await page2.goto(app.serverUrl);
-
-      await page1.waitForFunction(() => (window as any).Clerk?.loaded);
-      await page2.waitForFunction(() => (window as any).Clerk?.loaded);
-
-      const u1 = createTestUtils({ app, page: page1 });
-      await u1.po.signIn.goTo();
-      await u1.po.signIn.setIdentifier(fakeUser.email);
-      await u1.po.signIn.continue();
-      await u1.po.signIn.setPassword(fakeUser.password);
-      await u1.po.signIn.continue();
-      await u1.po.expect.toBeSignedIn();
-
-      // eslint-disable-next-line playwright/no-wait-for-timeout
-      await page1.waitForTimeout(1000);
-
-      await page2.reload();
-      await page2.waitForFunction(() => (window as any).Clerk?.loaded);
-
-      const u2 = createTestUtils({ app, page: page2 });
-      await u2.po.expect.toBeSignedIn();
-
-      // Both tabs are now signed in and have hydrated their token caches
-      // via Session constructor -> #hydrateCache, each with an independent
-      // onRefresh timer that fires at ~43s (TTL 60s - 15s leeway - 2s lead).
-      // Start counting /tokens requests from this point.
-      const refreshRequests: string[] = [];
-      await context.route('**/v1/client/sessions/*/tokens*', async route => {
-        refreshRequests.push(route.request().url());
-        await route.continue();
-      });
-
-      // Wait for proactive refresh timers to fire.
-      // Default token TTL is 60s; onRefresh fires at 60 - 15 - 2 = 43s from iat.
-      // We wait 50s to give comfortable buffer, this includes the broadcast delay.
-      //
-      // Uses page.evaluate instead of page.waitForTimeout to avoid
-      // the global actionTimeout (10s) silently capping the wait.
-      await page1.evaluate(() => new Promise(resolve => setTimeout(resolve, 50_000)));
-
-      // Only one tab should have made a /tokens request; the other tab should have
-      // received the refreshed token via BroadcastChannel.
-      expect(refreshRequests.length).toBe(1);
-
-      // Both tabs should still have valid tokens after the refresh cycle
-      const [page1Token, page2Token] = await Promise.all([
-        page1.evaluate(() => (window as any).Clerk.session?.getToken()),
-        page2.evaluate(() => (window as any).Clerk.session?.getToken()),
-      ]);
-
-      expect(page1Token).toBeTruthy();
-      expect(page2Token).toBeTruthy();
-      expect(page1Token).toBe(page2Token);
-    });
+    // The previous "multi-tab scheduled refreshes are deduped to a single request"
+    // test relied on the proactive-refresh setTimeout firing within a 50s wall-clock
+    // window, which assumed JWT TTL = 60s. The dev test instance now issues 300s
+    // tokens, so the timer fires at ~283s and the test never reached it. The
+    // BroadcastChannel-based dedup it was checking is already covered by the
+    // "multi-tab token sharing works when clearing the cache" test above, which
+    // explicitly triggers a fetch via `getToken({ skipCache: true })`. The
+    // proactive-refresh timer scheduling itself (the math, the leeway, the
+    // re-registration on success) is best validated by unit tests that mock
+    // `setTimeout` rather than depending on real time in a real browser.
   },
 );