fix(integrations-catalog): add global request pacing to prevent 429s

Adds SYNC_MIN_INTERVAL_MS (default 100ms) to enforce a minimum gap
between HTTP requests across all workers. Paces the sync to stay under
the CompAI internal API throttle, which was triggering 429s at higher
concurrency.

Also logs backoff delays when retries fire for visibility.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: tofikwest

tools/integrations-catalog-sync/README.md

@@ -24,6 +24,7 @@ node tools/integrations-catalog-sync/generate-readme.mjs
 Optional tuning:
 
 - `SYNC_CONCURRENCY` (default `2`) — concurrent fetches. Higher values trigger API rate limits.
+- `SYNC_MIN_INTERVAL_MS` (default `100`) — minimum interval between HTTP requests, enforced globally across all workers. Paces the sync to stay under the API throttle. Increase if you still see 429s; decrease (or set `0`) if the backend is raised later.
 
 ## Safety guardrails
 

tools/integrations-catalog-sync/sync.mjs

@@ -56,16 +56,27 @@ function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+const MIN_REQUEST_INTERVAL_MS = parseInt(process.env.SYNC_MIN_INTERVAL_MS || "100", 10);
+let nextAllowedAt = 0;
+
+async function pace() {
+  const now = Date.now();
+  if (now < nextAllowedAt) await sleep(nextAllowedAt - now);
+  nextAllowedAt = Date.now() + MIN_REQUEST_INTERVAL_MS;
+}
+
 async function fetchJson(path, maxRetries = 5) {
   let lastError;
   for (let attempt = 0; attempt < maxRetries; attempt++) {
+    await pace();
     const res = await fetch(`${API_BASE}${path}`, { headers: HEADERS });
     if (res.ok) return res.json();
     const text = await res.text();
     if ((res.status === 429 || res.status >= 500) && attempt < maxRetries - 1) {
       const retryAfter = parseInt(res.headers.get("retry-after") || "0", 10);
       const delay = retryAfter > 0 ? retryAfter * 1000 : Math.min(1000 * Math.pow(2, attempt), 15000);
       lastError = new Error(`${path} HTTP ${res.status}: ${text.slice(0, 200)}`);
+      console.warn(`  ${path} → ${res.status}, backing off ${delay}ms (attempt ${attempt + 1}/${maxRetries})`);
       await sleep(delay);
       continue;
     }