fix(testing): add retry logic for testing token fetch on 429/5xx (#8138)

Author: jacekradko

.changeset/retry-testing-token.md

@@ -0,0 +1,5 @@
+---
+"@clerk/testing": patch
+---
+
+Add retry logic with exponential backoff for testing token fetch on 429 and 5xx responses.

packages/testing/package.json

@@ -72,7 +72,8 @@
     "dev:pub": "pnpm dev -- --env.publish",
     "format": "node ../../scripts/format-package.mjs",
     "format:check": "node ../../scripts/format-package.mjs --check",
-    "lint": "eslint src"
+    "lint": "eslint src",
+    "test": "vitest"
   },
   "dependencies": {
     "@clerk/backend": "workspace:^",

packages/testing/src/common/__tests__/setup.test.ts

@@ -0,0 +1,200 @@
+import { ClerkAPIResponseError } from '@clerk/shared/error';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+// Re-export internals for testing by importing the module and testing through fetchEnvVars
+// Since fetchWithRetry and isNetworkError are not exported, we test them indirectly through fetchEnvVars
+// and also directly by extracting them via a test-specific import approach.
+
+// We need to mock the dependencies before importing the module under test
+vi.mock('@clerk/backend', () => ({
+  createClerkClient: vi.fn(),
+}));
+
+vi.mock('dotenv', () => ({
+  default: { config: vi.fn() },
+}));
+
+vi.mock('@clerk/shared/keys', () => ({
+  parsePublishableKey: vi.fn(() => ({ frontendApi: 'clerk.test.lcl.dev' })),
+}));
+
+import { createClerkClient } from '@clerk/backend';
+
+import { fetchEnvVars } from '../setup';
+
+function createClerkAPIError(status: number, retryAfter?: number) {
+  return new ClerkAPIResponseError('API error', {
+    data: [],
+    status,
+    retryAfter,
+  });
+}
+
+function createNetworkError(code: string) {
+  const err = new Error(`connect ${code}`);
+  (err as NodeJS.ErrnoException).code = code;
+  return err;
+}
+
+describe('fetchWithRetry (via fetchEnvVars)', () => {
+  const mockCreateTestingToken = vi.fn();
+
+  beforeEach(() => {
+    vi.useFakeTimers();
+    vi.stubEnv('CLERK_PUBLISHABLE_KEY', 'pk_test_abc');
+    vi.stubEnv('CLERK_SECRET_KEY', 'sk_test_abc');
+    delete process.env.CLERK_TESTING_TOKEN;
+
+    vi.mocked(createClerkClient).mockReturnValue({
+      testingTokens: { createTestingToken: mockCreateTestingToken },
+    } as any);
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+    vi.unstubAllEnvs();
+    vi.restoreAllMocks();
+  });
+
+  it('returns on first success without retrying', async () => {
+    mockCreateTestingToken.mockResolvedValueOnce({ token: 'test-token' });
+
+    const result = await fetchEnvVars({ dotenv: false });
+
+    expect(result.CLERK_TESTING_TOKEN).toBe('test-token');
+    expect(mockCreateTestingToken).toHaveBeenCalledTimes(1);
+  });
+
+  it('retries on 429 and succeeds', async () => {
+    mockCreateTestingToken
+      .mockRejectedValueOnce(createClerkAPIError(429))
+      .mockResolvedValueOnce({ token: 'test-token' });
+
+    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
+    const promise = fetchEnvVars({ dotenv: false });
+    await vi.advanceTimersByTimeAsync(30_000);
+    const result = await promise;
+
+    expect(result.CLERK_TESTING_TOKEN).toBe('test-token');
+    expect(mockCreateTestingToken).toHaveBeenCalledTimes(2);
+    expect(warnSpy).toHaveBeenCalledTimes(1);
+    expect(warnSpy.mock.calls[0][0]).toContain('[Retry] 429');
+    expect(warnSpy.mock.calls[0][0]).toContain('attempt 1/5');
+  });
+
+  it.each([408, 500, 502, 503, 504])('retries on %i status code', async status => {
+    mockCreateTestingToken
+      .mockRejectedValueOnce(createClerkAPIError(status))
+      .mockResolvedValueOnce({ token: 'test-token' });
+
+    vi.spyOn(console, 'warn').mockImplementation(() => {});
+    const promise = fetchEnvVars({ dotenv: false });
+    await vi.advanceTimersByTimeAsync(30_000);
+    const result = await promise;
+
+    expect(result.CLERK_TESTING_TOKEN).toBe('test-token');
+    expect(mockCreateTestingToken).toHaveBeenCalledTimes(2);
+  });
+
+  it('does not retry on non-retryable status codes', async () => {
+    mockCreateTestingToken.mockRejectedValueOnce(createClerkAPIError(401));
+    vi.spyOn(console, 'error').mockImplementation(() => {});
+
+    await expect(fetchEnvVars({ dotenv: false })).rejects.toThrow('API error');
+    expect(mockCreateTestingToken).toHaveBeenCalledTimes(1);
+  });
+
+  it('throws after max retries exhausted', async () => {
+    mockCreateTestingToken.mockImplementation(() => Promise.reject(createClerkAPIError(429)));
+
+    vi.spyOn(console, 'warn').mockImplementation(() => {});
+    vi.spyOn(console, 'error').mockImplementation(() => {});
+
+    const promise = fetchEnvVars({ dotenv: false }).catch(e => e);
+
+    await vi.runAllTimersAsync();
+
+    const error = await promise;
+    expect(error).toBeInstanceOf(ClerkAPIResponseError);
+    expect(error.status).toBe(429);
+    // 1 initial + 5 retries = 6 total calls
+    expect(mockCreateTestingToken).toHaveBeenCalledTimes(6);
+  });
+
+  it('uses retryAfter from error when available', async () => {
+    mockCreateTestingToken
+      .mockRejectedValueOnce(createClerkAPIError(429, 2))
+      .mockResolvedValueOnce({ token: 'test-token' });
+
+    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
+    const promise = fetchEnvVars({ dotenv: false });
+
+    // retryAfter is 2 seconds = 2000ms
+    await vi.advanceTimersByTimeAsync(2000);
+    const result = await promise;
+
+    expect(result.CLERK_TESTING_TOKEN).toBe('test-token');
+    expect(warnSpy.mock.calls[0][0]).toContain('waiting 2000ms');
+  });
+
+  it('caps retryAfter delay at MAX_RETRY_DELAY_MS', async () => {
+    mockCreateTestingToken
+      .mockRejectedValueOnce(createClerkAPIError(429, 60))
+      .mockResolvedValueOnce({ token: 'test-token' });
+
+    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
+    const promise = fetchEnvVars({ dotenv: false });
+    await vi.advanceTimersByTimeAsync(30_000);
+    const result = await promise;
+
+    expect(result.CLERK_TESTING_TOKEN).toBe('test-token');
+    // 60s * 1000 = 60000ms, capped to 30000ms
+    expect(warnSpy.mock.calls[0][0]).toContain('waiting 30000ms');
+  });
+
+  it.each(['ECONNREFUSED', 'ECONNRESET', 'ENOTFOUND', 'ETIMEDOUT', 'EAI_AGAIN'])(
+    'retries on network error %s',
+    async code => {
+      mockCreateTestingToken
+        .mockRejectedValueOnce(createNetworkError(code))
+        .mockResolvedValueOnce({ token: 'test-token' });
+
+      const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
+      const promise = fetchEnvVars({ dotenv: false });
+      await vi.advanceTimersByTimeAsync(30_000);
+      const result = await promise;
+
+      expect(result.CLERK_TESTING_TOKEN).toBe('test-token');
+      expect(mockCreateTestingToken).toHaveBeenCalledTimes(2);
+      expect(warnSpy.mock.calls[0][0]).toContain(`[Retry] ${code}`);
+    },
+  );
+
+  it('does not retry on non-network errors', async () => {
+    mockCreateTestingToken.mockRejectedValueOnce(new TypeError('unexpected'));
+    vi.spyOn(console, 'error').mockImplementation(() => {});
+
+    await expect(fetchEnvVars({ dotenv: false })).rejects.toThrow('unexpected');
+    expect(mockCreateTestingToken).toHaveBeenCalledTimes(1);
+  });
+
+  it('does not retry when non-retryable error code is present', async () => {
+    const err = new Error('unknown');
+    (err as NodeJS.ErrnoException).code = 'EPERM';
+    vi.spyOn(console, 'error').mockImplementation(() => {});
+
+    mockCreateTestingToken.mockRejectedValueOnce(err);
+
+    await expect(fetchEnvVars({ dotenv: false })).rejects.toThrow('unknown');
+    expect(mockCreateTestingToken).toHaveBeenCalledTimes(1);
+  });
+
+  it('skips retry when CLERK_TESTING_TOKEN is already set', async () => {
+    vi.stubEnv('CLERK_TESTING_TOKEN', 'existing-token');
+
+    const result = await fetchEnvVars({ dotenv: false });
+
+    expect(result.CLERK_TESTING_TOKEN).toBe('existing-token');
+    expect(mockCreateTestingToken).not.toHaveBeenCalled();
+  });
+});

packages/testing/src/common/setup.ts

@@ -1,9 +1,49 @@
 import { createClerkClient } from '@clerk/backend';
+import { isClerkAPIResponseError } from '@clerk/shared/error';
 import { parsePublishableKey } from '@clerk/shared/keys';
 import dotenv from 'dotenv';
 
 import type { ClerkSetupOptions, ClerkSetupReturn } from './types';
 
+const MAX_RETRIES = 5;
+const BASE_DELAY_MS = 1000;
+const JITTER_MAX_MS = 500;
+const MAX_RETRY_DELAY_MS = 30_000;
+const RETRYABLE_STATUS_CODES = new Set([408, 429, 500, 502, 503, 504]);
+const RETRYABLE_NETWORK_ERRORS = new Set(['ECONNREFUSED', 'ECONNRESET', 'ENOTFOUND', 'ETIMEDOUT', 'EAI_AGAIN']);
+
+function isNetworkError(error: unknown): boolean {
+  return (
+    error instanceof Error &&
+    'code' in error &&
+    RETRYABLE_NETWORK_ERRORS.has((error as NodeJS.ErrnoException).code ?? '')
+  );
+}
+
+async function fetchWithRetry<T>(fn: () => Promise<T>, label: string): Promise<T> {
+  for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+    try {
+      return await fn();
+    } catch (error) {
+      const isRetryableApi = isClerkAPIResponseError(error) && RETRYABLE_STATUS_CODES.has(error.status);
+      const isRetryableNetwork = isNetworkError(error);
+      if ((!isRetryableApi && !isRetryableNetwork) || attempt === MAX_RETRIES) {
+        throw error;
+      }
+      const status = isClerkAPIResponseError(error) ? error.status : (error as NodeJS.ErrnoException).code;
+      const delay =
+        isClerkAPIResponseError(error) && typeof error.retryAfter === 'number'
+          ? Math.min(error.retryAfter * 1000, MAX_RETRY_DELAY_MS)
+          : Math.min(BASE_DELAY_MS * Math.pow(2, attempt) + Math.random() * JITTER_MAX_MS, MAX_RETRY_DELAY_MS);
+      console.warn(
+        `[Retry] ${status} for ${label}, attempt ${attempt + 1}/${MAX_RETRIES}, waiting ${Math.round(delay)}ms`,
+      );
+      await new Promise(resolve => setTimeout(resolve, delay));
+    }
+  }
+  throw new Error('Unreachable');
+}
+
 export const fetchEnvVars = async (options?: ClerkSetupOptions): Promise<ClerkSetupReturn> => {
   const { debug = false, dotenv: loadDotEnv = true, ...rest } = options || {};
 
@@ -44,7 +84,10 @@ export const fetchEnvVars = async (options?: ClerkSetupOptions): Promise<ClerkSe
     try {
       const apiUrl = (rest as any)?.apiUrl || process.env.CLERK_API_URL;
       const clerkClient = createClerkClient({ secretKey, apiUrl });
-      const tokenData = await clerkClient.testingTokens.createTestingToken();
+      const tokenData = await fetchWithRetry(
+        () => clerkClient.testingTokens.createTestingToken(),
+        'testingTokens.createTestingToken',
+      );
       testingToken = tokenData.token;
     } catch (err) {
       console.error('Failed to fetch testing token from Clerk API.');

packages/testing/vitest.config.mts

@@ -0,0 +1,8 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    watch: false,
+    include: ['**/*.{test,spec}.{ts,tsx}'],
+  },
+});