[pull] main from firecrawl:main

apps/api/src/lib/exchange.test.ts

@@ -550,11 +550,13 @@ describe("Exchange routing", () => {
       status: 200,
     } as Awaited<ReturnType<typeof fetch>>);
 
-    await reportExchangeBilling({
-      accessEventId: "6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60",
-      status: "confirmed",
-      billingReference: "bill-1",
-    });
+    await expect(
+      reportExchangeBilling({
+        accessEventId: "6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60",
+        status: "confirmed",
+        billingReference: "bill-1",
+      }),
+    ).resolves.toBe(true);
 
     expect(fetch).toHaveBeenCalledWith(
       "https://exchange.example/v1/access-events/6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60/billing",
@@ -564,10 +566,12 @@ describe("Exchange routing", () => {
       }),
     );
 
-    await reportExchangeBilling({
-      accessEventId: "6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60",
-      status: "void",
-    });
+    await expect(
+      reportExchangeBilling({
+        accessEventId: "6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60",
+        status: "void",
+      }),
+    ).resolves.toBe(true);
 
     expect(fetch).toHaveBeenLastCalledWith(
       "https://exchange.example/v1/access-events/6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60/billing",
@@ -578,12 +582,84 @@ describe("Exchange routing", () => {
     );
 
     vi.mocked(fetch).mockRejectedValue(new Error("connect timeout"));
-    await expect(
-      reportExchangeBilling({
+    vi.useFakeTimers();
+    try {
+      const report = reportExchangeBilling({
         accessEventId: "6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60",
         status: "confirmed",
         billingReference: "bill-1",
+      });
+      await vi.runAllTimersAsync();
+      await expect(report).resolves.toBe(false);
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+
+  it("retries transient billing report failures and stops on definitive rejections", async () => {
+    vi.mocked(fetch)
+      .mockResolvedValueOnce({
+        ok: false,
+        status: 503,
+      } as Awaited<ReturnType<typeof fetch>>)
+      .mockResolvedValueOnce({
+        ok: true,
+        status: 200,
+      } as Awaited<ReturnType<typeof fetch>>);
+
+    vi.useFakeTimers();
+    try {
+      const report = reportExchangeBilling({
+        accessEventId: "6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60",
+        status: "confirmed",
+      });
+      await vi.runAllTimersAsync();
+      await expect(report).resolves.toBe(true);
+    } finally {
+      vi.useRealTimers();
+    }
+    expect(fetch).toHaveBeenCalledTimes(2);
+
+    // 429 is transient rate limiting, not a final answer: it retries.
+    vi.mocked(fetch).mockClear();
+    vi.mocked(fetch)
+      .mockResolvedValueOnce({
+        ok: false,
+        status: 429,
+        headers: new Headers({ "retry-after": "1" }),
+      } as Awaited<ReturnType<typeof fetch>>)
+      .mockResolvedValueOnce({
+        ok: true,
+        status: 200,
+      } as Awaited<ReturnType<typeof fetch>>);
+
+    vi.useFakeTimers();
+    try {
+      const report = reportExchangeBilling({
+        accessEventId: "6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60",
+        status: "confirmed",
+      });
+      await vi.runAllTimersAsync();
+      await expect(report).resolves.toBe(true);
+    } finally {
+      vi.useRealTimers();
+    }
+    expect(fetch).toHaveBeenCalledTimes(2);
+
+    // Any other 4xx is the Exchange's final answer (conflict, unknown
+    // event): no retry, report failure to the caller.
+    vi.mocked(fetch).mockClear();
+    vi.mocked(fetch).mockResolvedValue({
+      ok: false,
+      status: 409,
+    } as Awaited<ReturnType<typeof fetch>>);
+
+    await expect(
+      reportExchangeBilling({
+        accessEventId: "6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60",
+        status: "void",
       }),
-    ).resolves.toBeUndefined();
+    ).resolves.toBe(false);
+    expect(fetch).toHaveBeenCalledTimes(1);
   });
 });

apps/api/src/lib/exchange.ts

@@ -547,53 +547,119 @@ export function getExchangeSuccessCredits(input: {
 }
 
 const EXCHANGE_BILLING_TIMEOUT_MS = 5_000;
+const EXCHANGE_BILLING_ATTEMPTS = 3;
+const EXCHANGE_BILLING_RETRY_DELAY_MS = 2_000;
+const EXCHANGE_BILLING_RETRY_MAX_DELAY_MS = 15_000;
+
+// Retry-After from a 429, in milliseconds, when present and sane.
+// Accepts both delta-seconds and HTTP-date forms.
+function getRetryAfterMs(response: {
+  headers?: { get?: (name: string) => string | null };
+}): number | undefined {
+  const header = response.headers?.get?.("retry-after");
+  if (!header) {
+    return undefined;
+  }
+
+  const seconds = Number(header);
+  if (Number.isFinite(seconds)) {
+    return seconds > 0 ? seconds * 1_000 : undefined;
+  }
+
+  const resetAt = Date.parse(header);
+  if (Number.isNaN(resetAt)) {
+    return undefined;
+  }
+  const delayMs = resetAt - Date.now();
+  return delayMs > 0 ? delayMs : undefined;
+}
 
 /**
  * Report the billing outcome of a delivered Exchange access so the service
  * can reconcile its ledger: "confirmed" once the customer was billed, "void"
  * when the delivered access was ultimately discarded and never billed.
- * Failures only log - the service flags unresolved events for follow-up.
+ * Retries transient failures with a short backoff; never throws. Returns
+ * whether the report was accepted - a sustained failure leaves the event
+ * pending on the Exchange, which flags unresolved events for follow-up.
  */
 export async function reportExchangeBilling(input: {
   accessEventId: string;
   status: "confirmed" | "void";
   billingReference?: string;
-}): Promise<void> {
+}): Promise<boolean> {
   const baseUrl = getExchangeBaseUrl();
   if (!baseUrl) {
-    return;
+    return false;
   }
 
-  try {
-    const response = await fetch(
-      `${baseUrl}/v1/access-events/${encodeURIComponent(input.accessEventId)}/billing`,
-      {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({
+  for (let attempt = 1; attempt <= EXCHANGE_BILLING_ATTEMPTS; attempt++) {
+    let retryAfterMs: number | undefined;
+
+    try {
+      const response = await fetch(
+        `${baseUrl}/v1/access-events/${encodeURIComponent(input.accessEventId)}/billing`,
+        {
+          method: "POST",
+          headers: { "Content-Type": "application/json" },
+          body: JSON.stringify({
+            status: input.status,
+            ...(input.billingReference === undefined
+              ? {}
+              : { billingReference: input.billingReference }),
+          }),
+          signal: AbortSignal.timeout(EXCHANGE_BILLING_TIMEOUT_MS),
+        },
+      );
+
+      if (response.ok) {
+        return true;
+      }
+
+      // 4xx responses other than 429 are definitive (conflict, unknown
+      // event) - the Exchange has spoken and a retry cannot change the
+      // answer. 429 is transient rate limiting and retries.
+      if (response.status < 500 && response.status !== 429) {
+        rootLogger.warn("Exchange billing report rejected", {
+          accessEventId: input.accessEventId,
           status: input.status,
-          ...(input.billingReference === undefined
-            ? {}
-            : { billingReference: input.billingReference }),
-        }),
-        signal: AbortSignal.timeout(EXCHANGE_BILLING_TIMEOUT_MS),
-      },
-    );
+          statusCode: response.status,
+        });
+        return false;
+      }
+
+      if (response.status === 429) {
+        retryAfterMs = getRetryAfterMs(response);
+      }
 
-    if (!response.ok) {
       rootLogger.warn("Exchange billing report failed", {
         accessEventId: input.accessEventId,
         status: input.status,
         statusCode: response.status,
+        attempt,
+      });
+    } catch (error) {
+      rootLogger.warn("Exchange billing report errored", {
+        accessEventId: input.accessEventId,
+        status: input.status,
+        attempt,
+        error,
       });
     }
-  } catch (error) {
-    rootLogger.warn("Exchange billing report errored", {
-      accessEventId: input.accessEventId,
-      status: input.status,
-      error,
-    });
+
+    if (attempt < EXCHANGE_BILLING_ATTEMPTS) {
+      // Full jitter on the backoff so a batch of reports failing together
+      // does not retry against a degraded Exchange in synchronized bursts.
+      // Retry-After, when given, is the lower bound.
+      const backoff = EXCHANGE_BILLING_RETRY_DELAY_MS * attempt;
+      const delay = Math.min(
+        Math.max(retryAfterMs ?? 0, Math.random() * backoff),
+        EXCHANGE_BILLING_RETRY_MAX_DELAY_MS,
+      );
+      await new Promise(resolve => setTimeout(resolve, delay));
+    }
   }
+
+  return false;
 }
 
 /**