this is not related to the metrics change, just trying to get some excessively flaky tests to maybe not need to be run five+ times to pass

Author: chrishagglund-ship-it

src/integration-tests/E2EFiveTaskWorkflow.test.ts

@@ -11,16 +11,16 @@ import {
   WorkflowExecutor,
   clearWorkerRegistry,
   OrkesClients,
-  orkesConductorClient,
   simpleTask,
   worker,
 } from "../sdk";
 import { cleanupWorkflowsAndTasks } from "./utils/cleanup";
 import { waitForWorkflowStatus } from "./utils/waitForWorkflowStatus";
 import { describeForOrkesV5 } from "./utils/customJestDescribe";
+import { createClientWithRetry } from "./utils/createClientWithRetry";
 
 describeForOrkesV5("E2E: 5-task workflow × 50 executions", () => {
-  const clientPromise: Promise<Client> = orkesConductorClient();
+  const clientPromise: Promise<Client> = createClientWithRetry();
   let executor: WorkflowExecutor;
   let handler: TaskHandler | undefined;
 

src/integration-tests/MetadataClient.test.ts

@@ -2,11 +2,11 @@ import { expect, describe, test, jest } from "@jest/globals";
 import {
   MetadataClient,
   taskDefinition,
-  orkesConductorClient,
 } from "../sdk";
+import { createClientWithRetry } from "./utils/createClientWithRetry";
 
 describe("MetadataClient", () => {
-  const clientPromise = orkesConductorClient();
+  const clientPromise = createClientWithRetry();
   const taskName = `jsSdkTest-test_task_definition-${Date.now()}`;
 
   jest.setTimeout(60000);

src/integration-tests/SchemaClient.test.ts

@@ -1,9 +1,9 @@
 import { expect, describe, test, jest, beforeAll, afterAll } from "@jest/globals";
 import {
-  orkesConductorClient,
   OrkesClients,
   SchemaClient,
 } from "../sdk";
+import { createClientWithRetry } from "./utils/createClientWithRetry";
 import { describeForOrkesV5 } from "./utils/customJestDescribe";
 
 /**
@@ -16,7 +16,7 @@ import { describeForOrkesV5 } from "./utils/customJestDescribe";
 describeForOrkesV5("SchemaClient", () => {
   jest.setTimeout(60000);
 
-  const clientPromise = orkesConductorClient();
+  const clientPromise = createClientWithRetry();
   const suffix = Date.now();
 
   let schemaClient: SchemaClient;

src/integration-tests/ServiceRegistryClient.test.ts

@@ -1,6 +1,6 @@
 import { beforeAll, afterEach, expect, jest, test } from "@jest/globals";
 import { ServiceRegistryClient } from "../sdk/clients/service-registry";
-import { orkesConductorClient } from "../sdk/createConductorClient";
+import { createClientWithRetry } from "./utils/createClientWithRetry";
 import { ServiceType } from "../open-api";
 import * as fs from "fs";
 import * as path from "path";
@@ -13,7 +13,7 @@ const TEST_SERVICE_URI =
   "http://httpbin-server:8081/api-docs";
 
 describeForOrkesV5("ServiceRegistryClient", () => {
-  const clientPromise = orkesConductorClient();
+  const clientPromise = createClientWithRetry();
   let serviceRegistryClient: ServiceRegistryClient;
 
   const testServicesToCleanup: string[] = [];

src/integration-tests/WorkflowExecutor.test.ts

@@ -22,9 +22,9 @@ import {
   TaskClient,
   MetadataClient,
   WorkflowExecutor,
-  orkesConductorClient,
 } from "../sdk";
 import { cleanupWorkflowsAndTasks } from "./utils/cleanup";
+import { createClientWithRetry } from "./utils/createClientWithRetry";
 import { waitForWorkflowStatus } from "./utils/waitForWorkflowStatus";
 import { getComplexSignalTestWfDef } from "./metadata/complex_wf_signal_test";
 import { getComplexSignalTestSubWf1Def } from "./metadata/complex_wf_signal_test_subworkflow_1";
@@ -33,7 +33,7 @@ import { getWaitSignalTestWfDef } from "./metadata/wait_signal_test";
 import { describeForOrkesV4, describeForOrkesV5 } from "./utils/customJestDescribe";
 
 describe("WorkflowExecutor", () => {
-  const clientPromise = orkesConductorClient();
+  const clientPromise = createClientWithRetry();
 
   jest.setTimeout(60000);
 
@@ -291,7 +291,7 @@ describe("WorkflowExecutor", () => {
       WAIT_SIGNAL_TEST: getWaitSignalTestWfDef(now),
     };
 
-    const clientPromise = orkesConductorClient();
+    const clientPromise = createClientWithRetry();
     jest.setTimeout(300000);
 
     let client: Client;

src/integration-tests/readme.test.ts

@@ -1,17 +1,17 @@
 import { expect, describe, test, jest, afterAll } from "@jest/globals";
 import {
-  orkesConductorClient,
   TaskRunner,
   WorkflowExecutor,
   simpleTask,
   generate,
   MetadataClient,
 } from "../sdk";
+import { createClientWithRetry } from "./utils/createClientWithRetry";
 import { TaskType } from "../open-api";
 import { waitForWorkflowStatus } from "./utils/waitForWorkflowStatus";
 
 describe("TaskManager", () => {
-  const clientPromise = orkesConductorClient();
+  const clientPromise = createClientWithRetry();
   const workflowsToCleanup: { name: string; version: number }[] = [];
 
   jest.setTimeout(60000);

src/sdk/createConductorClient/__tests__/createConductorClient.test.ts

@@ -1,4 +1,10 @@
 import { jest, expect, describe, it, beforeEach } from "@jest/globals";
+import { MAX_AUTH_BACKOFF_MS, MAX_INITIAL_TOKEN_RETRIES } from "../constants";
+
+const INITIAL_AUTH_TOTAL_BACKOFF_MS = Array.from(
+  { length: MAX_INITIAL_TOKEN_RETRIES - 1 },
+  (_, i) => Math.min(Math.pow(2, i) * 1000, MAX_AUTH_BACKOFF_MS)
+).reduce((a, b) => a + b, 0);
 
 // Mock undici to avoid ts-jest compilation failure on missing module
 jest.mock("../helpers/getUndiciHttp2FetchFn", () => ({
@@ -176,9 +182,10 @@ describe("createConductorClient integration", () => {
     });
 
     it("should throw when initial auth fails", async () => {
+      jest.useFakeTimers();
       mockedGenerateToken.mockResolvedValue(tokenFailure("Bad credentials"));
 
-      await expect(
+      const assertion = expect(
         createConductorClient(
           {
             serverUrl: "http://localhost:8080",
@@ -188,6 +195,12 @@ describe("createConductorClient integration", () => {
           async () => jsonResponse({})
         )
       ).rejects.toThrow("Failed to generate authorization token");
+
+      // Advance past all initial-auth retry backoff delays
+      await jest.advanceTimersByTimeAsync(INITIAL_AUTH_TOTAL_BACKOFF_MS);
+      await assertion;
+
+      jest.useRealTimers();
     });
   });
 

src/sdk/createConductorClient/constants.ts

@@ -5,7 +5,7 @@ export const MAX_HTTP2_CONNECTIONS = 10;
 export const TOKEN_TTL_MS = 2_700_000; // 45 minutes - refresh token before it expires
 export const MAX_AUTH_FAILURES = 5; // stop logging errors after this many consecutive failures
 export const MAX_AUTH_BACKOFF_MS = 60_000; // 60s cap on exponential backoff
-export const MAX_INITIAL_TOKEN_RETRIES = 3; // retry initial token request on transient failures
+export const MAX_INITIAL_TOKEN_RETRIES = 8; // retry initial token request on transient failures (covers ~3min of 503s)
 
 // Fetch retry
 export const MAX_TRANSPORT_RETRIES = 3;

src/sdk/createConductorClient/helpers/__tests__/handleAuth.test.ts

@@ -4,7 +4,13 @@ import { handleAuth } from "../handleAuth";
 import { TokenResource } from "../../../../open-api/generated";
 import type { Client } from "../../../../open-api/generated/client/types.gen";
 import type { ConductorLogger } from "../../../helpers/logger";
-import { TOKEN_TTL_MS, MAX_AUTH_FAILURES, MAX_INITIAL_TOKEN_RETRIES } from "../../constants";
+import { TOKEN_TTL_MS, MAX_AUTH_FAILURES, MAX_AUTH_BACKOFF_MS, MAX_INITIAL_TOKEN_RETRIES } from "../../constants";
+
+/** Total fake-time needed to exhaust all initial-auth retry backoffs. */
+const INITIAL_AUTH_TOTAL_BACKOFF_MS = Array.from(
+  { length: MAX_INITIAL_TOKEN_RETRIES - 1 },
+  (_, i) => Math.min(Math.pow(2, i) * 1000, MAX_AUTH_BACKOFF_MS)
+).reduce((a, b) => a + b, 0);
 
 // Mock TokenResource.generateToken
 jest.mock("../../../../open-api/generated", () => ({
@@ -117,8 +123,7 @@ describe("handleAuth", () => {
         handleAuth(mockClient as unknown as Client, "key-id", "key-secret", 3600000, mockLogger)
       ).rejects.toThrow("Failed to generate authorization token");
 
-      // Advance past all retry backoff delays (1s + 2s = 3s)
-      await jest.advanceTimersByTimeAsync(3000);
+      await jest.advanceTimersByTimeAsync(INITIAL_AUTH_TOTAL_BACKOFF_MS);
       await assertion;
 
       expect(mockedGenerateToken).toHaveBeenCalledTimes(MAX_INITIAL_TOKEN_RETRIES);
@@ -141,8 +146,7 @@ describe("handleAuth", () => {
 
       const p = handleAuth(mockClient as unknown as Client, "key-id", "key-secret", 3600000, mockLogger);
 
-      // Advance past retry backoff delays
-      await jest.advanceTimersByTimeAsync(3000);
+      await jest.advanceTimersByTimeAsync(INITIAL_AUTH_TOTAL_BACKOFF_MS);
 
       const result = await p;
       expect(result).toBeDefined();
@@ -164,7 +168,7 @@ describe("handleAuth", () => {
         handleAuth(mockClient as unknown as Client, "key-id", "key-secret", 3600000, mockLogger)
       ).rejects.toThrow("Failed to generate authorization token");
 
-      await jest.advanceTimersByTimeAsync(3000);
+      await jest.advanceTimersByTimeAsync(INITIAL_AUTH_TOTAL_BACKOFF_MS);
       await assertion;
 
       expect(mockLogger.debug).toHaveBeenCalledWith(
@@ -184,7 +188,7 @@ describe("handleAuth", () => {
         handleAuth(mockClient as unknown as Client, "key-id", "key-secret", 3600000, mockLogger)
       ).rejects.toThrow("Failed to generate authorization token");
 
-      await jest.advanceTimersByTimeAsync(3000);
+      await jest.advanceTimersByTimeAsync(INITIAL_AUTH_TOTAL_BACKOFF_MS);
       await assertion;
 
       expect(mockLogger.debug).toHaveBeenCalledWith(