fix(deploy): route test failures through api path

Author: wyattjoh

packages/cli-core/src/cli-program.ts

@@ -910,43 +910,40 @@ Tutorial — enable completions for your shell:
       createOption(
         "--test-force-production-instance",
         "Force deploy to use a mocked production instance",
-      ).hideHelp(),
+      ),
     )
     .addOption(
       createOption(
         "--test-fail-production-instance-check",
         "Simulate a deploy failure while checking for a production instance",
-      ).hideHelp(),
+      ),
     )
     .addOption(
       createOption(
         "--test-fail-domain-lookup",
         "Simulate a deploy failure while loading the production domain",
-      ).hideHelp(),
+      ),
     )
     .addOption(
       createOption(
         "--test-fail-validate-cloning",
         "Simulate a deploy failure while validating cloning",
-      ).hideHelp(),
+      ),
     )
     .addOption(
       createOption(
         "--test-fail-create-production-instance",
         "Simulate a deploy failure while creating the production instance",
-      ).hideHelp(),
+      ),
     )
     .addOption(
-      createOption(
-        "--test-fail-dns-verification",
-        "Simulate a deploy failure while verifying DNS",
-      ).hideHelp(),
+      createOption("--test-fail-dns-verification", "Simulate a deploy failure while verifying DNS"),
     )
     .addOption(
       createOption(
         "--test-fail-oauth-save",
         "Simulate a deploy failure while saving OAuth credentials",
-      ).hideHelp(),
+      ),
     )
     .action(deploy);
 

packages/cli-core/src/commands/deploy/index.test.ts

@@ -3,7 +3,7 @@ import { mkdtemp, rm } from "node:fs/promises";
 import { join, relative } from "node:path";
 import { tmpdir } from "node:os";
 import { captureLog, promptsStubs, listageStubs } from "../../test/lib/stubs.ts";
-import { EXIT_CODE, UserAbortError, type CliError } from "../../lib/errors.ts";
+import { CliError, EXIT_CODE, UserAbortError } from "../../lib/errors.ts";
 
 const mockIsAgent = mock();
 let _modeOverride: string | undefined;
@@ -197,6 +197,20 @@ describe("deploy", () => {
     return captured.run(() => deploy(options));
   }
 
+  async function expectTestApiFailure(promise: Promise<unknown>, message: string): Promise<Error> {
+    let error: Error | undefined;
+    try {
+      await promise;
+    } catch (caught) {
+      error = caught as Error;
+    }
+
+    expect(error).toBeInstanceOf(Error);
+    expect(error).not.toBeInstanceOf(CliError);
+    expect(error?.message).toContain(message);
+    return error!;
+  }
+
   async function linkedProject(profile: Record<string, unknown> = {}) {
     tempDir = await mkdtemp(join(tmpdir(), "clerk-deploy-test-"));
     _setConfigDir(tempDir);
@@ -857,14 +871,47 @@ describe("deploy", () => {
       await linkedProject();
       mockIsAgent.mockReturnValue(false);
 
-      await expect(runDeploy({ testFailProductionInstanceCheck: true })).rejects.toThrow(
+      await expectTestApiFailure(
+        runDeploy({ testFailProductionInstanceCheck: true }),
         "Simulated deploy failure: production instance check.",
       );
 
-      expect(mockFetchApplication).not.toHaveBeenCalled();
+      expect(mockFetchApplication).toHaveBeenCalledWith("app_xyz789");
       expect(mockFetchInstanceConfig).not.toHaveBeenCalled();
     });
 
+    test("--test-fail-production-instance-check prints one Failed status in interactive output", async () => {
+      await linkedProject();
+      mockIsAgent.mockReturnValue(false);
+      stderrSpy = spyOn(process.stderr, "write").mockImplementation(() => true);
+      const originalCi = process.env.CI;
+      const originalIsTty = process.stderr.isTTY;
+      Object.defineProperty(process.stderr, "isTTY", { configurable: true, value: true });
+      delete process.env.CI;
+
+      try {
+        await expectTestApiFailure(
+          runDeploy({ testFailProductionInstanceCheck: true }),
+          "Simulated deploy failure: production instance check.",
+        );
+      } finally {
+        Object.defineProperty(process.stderr, "isTTY", {
+          configurable: true,
+          value: originalIsTty,
+        });
+        if (originalCi === undefined) {
+          delete process.env.CI;
+        } else {
+          process.env.CI = originalCi;
+        }
+      }
+
+      const terminalOutput = stripAnsi(
+        stderrSpy.mock.calls.map((call: unknown[]) => String(call[0])).join(""),
+      );
+      expect(terminalOutput.match(/\bFailed\b/g) ?? []).toHaveLength(1);
+    });
+
     test("--test-fail-domain-lookup simulates production domain lookup failure", async () => {
       await linkedProject();
       mockLiveProduction({
@@ -873,22 +920,26 @@ describe("deploy", () => {
       });
       mockIsAgent.mockReturnValue(false);
 
-      await expect(runDeploy({ testFailDomainLookup: true })).rejects.toThrow(
+      await expectTestApiFailure(
+        runDeploy({ testFailDomainLookup: true }),
         "Simulated deploy failure: production domain lookup.",
       );
 
-      expect(mockListApplicationDomains).not.toHaveBeenCalled();
+      expect(mockListApplicationDomains).toHaveBeenCalledWith("app_xyz789");
     });
 
     test("--test-fail-validate-cloning simulates cloning validation failure", async () => {
       await linkedProject();
       mockIsAgent.mockReturnValue(false);
 
-      await expect(runDeploy({ testFailValidateCloning: true })).rejects.toThrow(
+      await expectTestApiFailure(
+        runDeploy({ testFailValidateCloning: true }),
         "Simulated deploy failure: cloning validation.",
       );
 
-      expect(mockValidateCloning).not.toHaveBeenCalled();
+      expect(mockValidateCloning).toHaveBeenCalledWith("app_xyz789", {
+        clone_instance_id: "ins_dev_123",
+      });
       expect(mockCreateProductionInstance).not.toHaveBeenCalled();
     });
 
@@ -898,11 +949,15 @@ describe("deploy", () => {
       mockConfirm.mockResolvedValueOnce(true).mockResolvedValueOnce(true);
       mockInput.mockResolvedValueOnce("example.com");
 
-      await expect(runDeploy({ testFailCreateProductionInstance: true })).rejects.toThrow(
+      await expectTestApiFailure(
+        runDeploy({ testFailCreateProductionInstance: true }),
         "Simulated deploy failure: production instance creation.",
       );
 
-      expect(mockCreateProductionInstance).not.toHaveBeenCalled();
+      expect(mockCreateProductionInstance).toHaveBeenCalledWith("app_xyz789", {
+        home_url: "example.com",
+        clone_instance_id: "ins_dev_123",
+      });
     });
 
     test("--test-fail-dns-verification simulates DNS verification failure", async () => {
@@ -920,23 +975,13 @@ describe("deploy", () => {
       mockPassword.mockResolvedValueOnce("google-secret");
       mockPatchInstanceConfig.mockResolvedValueOnce({});
 
-      await runDeploy({ testFailDnsVerification: true });
-      const err = stripAnsi(captured.err);
+      await expectTestApiFailure(
+        runDeploy({ testFailDnsVerification: true }),
+        "Simulated deploy failure: DNS verification.",
+      );
 
-      expect(mockGetDeployStatus).not.toHaveBeenCalled();
-      expect(err).toContain("DNS propagation can take time");
-      expect(err).toContain("Add the following records at your DNS provider:");
-      expect(err).toContain("Host:  clerk.example.com");
-      expect(err).toContain("Value: frontend-api.clerk.services");
-      expect(err).toContain("Skipping DNS verification for now.");
-      expect(err).toContain("Saved Google OAuth credentials");
-      expect(mockPatchInstanceConfig).toHaveBeenCalledWith("app_xyz789", "ins_prod_mock", {
-        connection_oauth_google: {
-          enabled: true,
-          client_id: "google-client-id.apps.googleusercontent.com",
-          client_secret: "google-secret",
-        },
-      });
+      expect(mockGetDeployStatus).toHaveBeenCalledWith("app_xyz789", "ins_prod_mock");
+      expect(mockPatchInstanceConfig).not.toHaveBeenCalled();
     });
 
     test("--test-fail-oauth-save simulates OAuth credential save failure", async () => {
@@ -953,11 +998,18 @@ describe("deploy", () => {
       mockInput.mockResolvedValueOnce("google-client-id.apps.googleusercontent.com");
       mockPassword.mockResolvedValueOnce("google-secret");
 
-      await expect(runDeploy({ testFailOAuthSave: true })).rejects.toThrow(
+      await expectTestApiFailure(
+        runDeploy({ testFailOAuthSave: true }),
         "Simulated deploy failure: OAuth credential save.",
       );
 
-      expect(mockPatchInstanceConfig).not.toHaveBeenCalled();
+      expect(mockPatchInstanceConfig).toHaveBeenCalledWith("app_xyz789", "ins_prod_123", {
+        connection_oauth_google: {
+          enabled: true,
+          client_id: "google-client-id.apps.googleusercontent.com",
+          client_secret: "google-secret",
+        },
+      });
     });
 
     test("plain deploy resumes DNS verification from live API state", async () => {

packages/cli-core/src/commands/deploy/index.ts

@@ -3,7 +3,12 @@ import { NEXT_STEPS } from "../../lib/next-steps.ts";
 import { isInsideGutter, log, setPrefixTone, type PrefixTone } from "../../lib/log.ts";
 import { sleep } from "../../lib/sleep.ts";
 import { bar, intro, outro, withSpinner } from "../../lib/spinner.ts";
-import { CliError, UserAbortError, isPromptExitError, throwUsageError } from "../../lib/errors.ts";
+import {
+  PlapiError,
+  UserAbortError,
+  isPromptExitError,
+  throwUsageError,
+} from "../../lib/errors.ts";
 import { resolveProfile, setProfile } from "../../lib/config.ts";
 import {
   type Application,
@@ -211,14 +216,15 @@ async function resolveDeployContext(options: DeployOptions): Promise<DeployConte
     profileKey: resolved.path,
     profile: resolved.profile,
     ...testFlags,
-    ...(await withSpinner("Checking for production instance...", () => {
-      if (testFlags.testFailProductionInstanceCheck) {
-        throw testDeployFailure("production instance check");
-      }
-      return resolveLiveApplicationContext(resolved.profile, {
-        forceMockProductionInstance: testFlags.testForceProductionInstance,
-      });
-    })),
+    ...(await withSpinner("Checking for production instance...", () =>
+      withTestFailureAfterApiCall(
+        resolveLiveApplicationContext(resolved.profile, {
+          forceMockProductionInstance: testFlags.testForceProductionInstance,
+        }),
+        testFlags.testFailProductionInstanceCheck,
+        "production instance check",
+      ),
+    )),
   };
 }
 
@@ -245,8 +251,24 @@ function resolveTestDeployFlags(
   };
 }
 
-function testDeployFailure(step: string): CliError {
-  return new CliError(`Simulated deploy failure: ${step}.`);
+function simulatedDeployApiFailure(step: string): PlapiError {
+  return new PlapiError(
+    500,
+    JSON.stringify({ errors: [{ message: `Simulated deploy failure: ${step}.` }] }),
+    "clerk deploy test flag",
+  );
+}
+
+async function withTestFailureAfterApiCall<T>(
+  promise: Promise<T>,
+  shouldFail: boolean | undefined,
+  step: string,
+): Promise<T> {
+  const result = await promise;
+  if (shouldFail) {
+    throw simulatedDeployApiFailure(step);
+  }
+  return result;
 }
 
 async function resolveLiveApplicationContext(
@@ -516,13 +538,13 @@ async function resolveLiveDeploySnapshot(
 }
 
 async function loadProductionDomain(ctx: DeployContext): Promise<ApplicationDomain | undefined> {
-  if (ctx.testFailDomainLookup) {
-    throw testDeployFailure("production domain lookup");
-  }
   if (ctx.testForceProductionInstance) {
     return mockProductionDomain();
   }
   const domains = await listApplicationDomains(ctx.appId);
+  if (ctx.testFailDomainLookup) {
+    throw simulatedDeployApiFailure("production domain lookup");
+  }
   return domains.data.find((domain) => !domain.is_satellite) ?? domains.data[0];
 }
 
@@ -635,10 +657,11 @@ function discoverEnabledOAuthProviders(config: Record<string, unknown>): Discove
 
 async function runValidateCloning(ctx: DeployContext): Promise<void> {
   await withSpinner("Validating subscription compatibility...", async () => {
-    if (ctx.testFailValidateCloning) {
-      throw testDeployFailure("cloning validation");
-    }
-    await validateCloning(ctx.appId, { clone_instance_id: ctx.developmentInstanceId });
+    await withTestFailureAfterApiCall(
+      validateCloning(ctx.appId, { clone_instance_id: ctx.developmentInstanceId }),
+      ctx.testFailValidateCloning,
+      "cloning validation",
+    );
   });
 }
 
@@ -647,13 +670,14 @@ async function createProductionInstance(
   domain: string,
 ): Promise<ProductionInstanceResponse> {
   return withSpinner("Creating production instance...", async () => {
-    if (ctx.testFailCreateProductionInstance) {
-      throw testDeployFailure("production instance creation");
-    }
-    return apiCreateProductionInstance(ctx.appId, {
-      home_url: domain,
-      clone_instance_id: ctx.developmentInstanceId,
-    });
+    return withTestFailureAfterApiCall(
+      apiCreateProductionInstance(ctx.appId, {
+        home_url: domain,
+        clone_instance_id: ctx.developmentInstanceId,
+      }),
+      ctx.testFailCreateProductionInstance,
+      "production instance creation",
+    );
   });
 }
 
@@ -757,11 +781,11 @@ async function runDnsVerification(
   }
 
   const verified = await withSpinner(`Verifying DNS for ${state.domain}...`, async () => {
-    if (ctx.testFailDnsVerification) {
-      return false;
-    }
     for (let attempt = 0; attempt < DEPLOY_STATUS_MAX_POLLS; attempt++) {
       const result = await getDeployStatus(ctx.appId, productionInstanceId);
+      if (ctx.testFailDnsVerification) {
+        throw simulatedDeployApiFailure("DNS verification");
+      }
       if (result.status === "complete") return true;
       await sleep(DEPLOY_STATUS_POLL_INTERVAL_MS);
     }
@@ -882,15 +906,16 @@ async function collectAndSaveOAuthCredentials(
   );
 
   await withSpinner(`Saving ${label} OAuth credentials...`, async () => {
-    if (ctx.testFailOAuthSave) {
-      throw testDeployFailure("OAuth credential save");
-    }
-    await patchInstanceConfig(ctx.appId, productionInstanceId, {
-      [`connection_oauth_${provider}`]: {
-        enabled: true,
-        ...credentials,
-      },
-    });
+    await withTestFailureAfterApiCall(
+      patchInstanceConfig(ctx.appId, productionInstanceId, {
+        [`connection_oauth_${provider}`]: {
+          enabled: true,
+          ...credentials,
+        },
+      }),
+      ctx.testFailOAuthSave,
+      "OAuth credential save",
+    );
   });
   log.success(`Saved ${label} OAuth credentials`);
   return true;